389-commits August 2016

389-commits@lists.fedoraproject.org

6 participants
70 discussions

ldap/admin
by Mark Reynolds 25 Aug '16

25 Aug '16

ldap/admin/src/base-initconfig.in | 67 ++++++++++++++++++-------------------- 1 file changed, 33 insertions(+), 34 deletions(-) New commits: commit 1e44c1f321d03dcb30615bcacfc5c099612b1c66 Author: Mark Reynolds <mreynolds(a)redhat.com> Date: Thu Aug 25 13:39:25 2016 -0400 Ticket 48950 - Add systemd warning to the LD_PRELOAD example in /etc/sysconfig/dirsrv Description: systemd warning about omitting the PORT was missing from the LD_PRELOAD example. Also fixed up file format/layout.. https://fedorahosted.org/389/ticket/48950 Reviewed by: nhosoi, amsharma, lkrispen (Thanks!!!) diff --git a/ldap/admin/src/base-initconfig.in b/ldap/admin/src/base-initconfig.in index 0481c3e..8507296 100644 --- a/ldap/admin/src/base-initconfig.in +++ b/ldap/admin/src/base-initconfig.in @@ -7,44 +7,43 @@ @preamble@ -# In order to make more file descriptors available -# to the directory server, first make sure the system -# hard limits are raised, then use ulimit - uncomment -# out the following line and change the value to the -# desired value -# ulimit -n 8192 -# note - if using systemd, ulimit won't work - you must edit -# the systemd unit file for directory server to add the -# LimitNOFILE option - see man systemd.exec for more info - -# A per instance keytab does not make much sense for servers. -# Kerberos clients use the machine FQDN to obtain a ticket like ldap/FQDN, there -# is nothing that can make a client understand how to get a per-instance ticket. -# Therefore by default a keytab should be considered a per server option. - -# Also this file is sourced for all instances, so again all -# instances would ultimately get the same keytab. - +# In order to make more file descriptors available to the directory server, +# first make sure the system hard limits are raised, then use ulimit - +# uncomment out the following line and change the value to the desired value +#ulimit -n 8192 +# note - if using systemd, ulimit won't work - you must edit the systemd unit +# file for directory server to add the LimitNOFILE option - see "man +# systemd.exec" for more info + +# A per instance keytab does not make much sense for servers. Kerberos clients +# use the machine FQDN to obtain a ticket like ldap/FQDN, there is nothing that +# can make a client understand how to get a per-instance ticket. Therefore by +# default a keytab should be considered a per server option. +# +# Also this file is sourced for all instances, so again all instances would +# ultimately get the same keytab. +# # Finally a keytab is normally named either krb5.keytab or <service>.keytab - -# In order to use SASL/GSSAPI (Kerberos) the directory -# server needs to know where to find its keytab -# file - uncomment the following line and set -# the path and filename appropriately -# if using systemd, omit the "; export VARNAME" at the end -# KRB5_KTNAME=@instconfigdir@/myname.keytab ; export KRB5_KTNAME - -# how many seconds to wait for the startpid file to show -# up before we assume there is a problem and fail to start -# if using systemd, omit the "; export VARNAME" at the end +# +# In order to use SASL/GSSAPI (Kerberos) the directory server needs to know +# where to find its keytab file - uncomment the following line and set the +# path and filename appropriately. +# If using systemd, omit the "; export VARNAME" at the end. +#KRB5_KTNAME=@instconfigdir@/myname.keytab ; export KRB5_KTNAME + +# How many seconds to wait for the startpid file to show up before we assume +# there is a problem and fail to start. +# If using systemd, omit the "; export STARTPID_TIME" at the end. #STARTPID_TIME=10 ; export STARTPID_TIME -# how many seconds to wait for the pid file to show -# up before we assume there is a problem and fail to start -# if using systemd, omit the "; export VARNAME" at the end + +# How many seconds to wait for the pid file to show up before we assume there +# is a problem and fail to start. +# If using systemd, omit the "; export PID_TIME" at the end. #PID_TIME=600 ; export PID_TIME # The tcmalloc memory allocator has been shown to have a positive impact on # the Directory Server's virtual & resident memory size/growth. tcmalloc is -# available on RHEL/Fedora in the gperftools package (this could be different -# on other platforms). Here is an example of preloading tcmalloc: +# available on RHEL/Fedora in the gperftools-libs package (this could be +# different on other platforms). +# If using systemd, omit the "; export LD_PRELOAD" at the end. #LD_PRELOAD=@libdir@/libtcmalloc.so.4 ; export LD_PRELOAD

1 0

ldap/ldif
by Ludwig Krispenz 25 Aug '16

25 Aug '16

ldap/ldif/template-suffix-db.ldif.in | 1 + 1 file changed, 1 insertion(+) New commits: commit 2fb3c270c73cdab9511cf20452b68bfc38357659 Author: Ludwig Krispenz <lkrispen(a)redhat.com> Date: Thu Aug 25 13:50:15 2016 +0200 provide backend dir in suffix template this fixes a regression by the fix for #48402 Reviewed by: diff --git a/ldap/ldif/template-suffix-db.ldif.in b/ldap/ldif/template-suffix-db.ldif.in index 1bcc5c2..9f8042c 100644 --- a/ldap/ldif/template-suffix-db.ldif.in +++ b/ldap/ldif/template-suffix-db.ldif.in @@ -5,6 +5,7 @@ objectclass: top objectclass: extensibleObject objectclass: nsBackendInstance nsslapd-suffix: %ds_suffix% +nsslapd-directory: %db_dir%/%ds_bename% cn: %ds_bename% dn: cn=encrypted attribute keys,cn=%ds_bename%,cn=ldbm database,cn=plugins,cn=config

1 0

ldap/admin
by Noriko Hosoi 24 Aug '16

24 Aug '16

ldap/admin/src/scripts/DSCreate.pm.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) New commits: commit e9ae515f6f6abfbd6a5c1220ca54fb51657246b9 Author: Viktor Ashirov <vashirov(a)redhat.com> Date: Wed Aug 24 20:25:16 2016 +0200 Ticket 48953 - Skip labelling and unlabelling ports during the test Description: In CI creating and removing instances takes most of the time, because of labelling/unlabelling ports via semanage. Fix description: Skip labelling and unlabelling if special environment variables are present: DS_SKIP_LABEL and DS_SKIP_UNLABEL respectively. https://fedorahosted.org/389/ticket/48953 Reviewed by: nhosoi(a)redhat.com (Thanks!) diff --git a/ldap/admin/src/scripts/DSCreate.pm.in b/ldap/admin/src/scripts/DSCreate.pm.in index bf5fc5c..7e98803 100644 --- a/ldap/admin/src/scripts/DSCreate.pm.in +++ b/ldap/admin/src/scripts/DSCreate.pm.in @@ -1039,7 +1039,7 @@ sub updateSelinuxPolicy { # label the selected port as ldap_port_t # We should be doing this for secure port too ..... - if ($inf->{slapd}->{ServerPort} != 0) { + if ($inf->{slapd}->{ServerPort} != 0 and not $ENV{DS_SKIP_LABEL}) { my $port_query_cmd = ("$inf->{slapd}->{sbindir}/ds_selinux_port_query $inf->{slapd}->{ServerPort} ldap_port_t 2> $mydevnull"); my $need_label = 0; my $result = system($port_query_cmd); @@ -1462,7 +1462,7 @@ sub removeDSInstance { # remove the selinux label from the ports if needed my $mydevnull = (-c "/dev/null" ? " /dev/null " : " NUL "); - if ((getLogin() eq 'root') and "@with_selinux@" and system("@sbindir@/ds_selinux_enabled") == 256 ) { + if ((getLogin() eq 'root') and "@with_selinux@" and system("@sbindir@/ds_selinux_enabled") == 256 and not $ENV{DS_SKIP_UNLABEL}) { foreach my $port (@{$entry->{"nsslapd-port"}}) {

1 0

dirsrvtests/tests
by Simon Pichugin 24 Aug '16

24 Aug '16

dirsrvtests/tests/suites/password/pwdPolicy_attribute_test.py | 339 ++++++++++ dirsrvtests/tests/suites/password/pwdPolicy_syntax_test.py | 224 ++++++ dirsrvtests/tests/suites/password/pwdPolicy_test.py | 224 ------ dirsrvtests/tests/suites/password/pwd_change_policytest.py | 240 ------- 4 files changed, 563 insertions(+), 464 deletions(-) New commits: commit 6abd5f49d8e32732a97794e68f642045f16f23e3 Author: Simon Pichugin <spichugi(a)redhat.com> Date: Wed Aug 24 10:08:29 2016 +0200 Ticket 48967 - Add CI test and refactor test suite Description: Add a test case to password policy test suite. Refactor structure of password test suite so it would become more logical. https://fedorahosted.org/389/ticket/48967 Reviewed by: nhosoi (Thank you, Noriko!) diff --git a/dirsrvtests/tests/suites/password/pwdPolicy_attribute_test.py b/dirsrvtests/tests/suites/password/pwdPolicy_attribute_test.py new file mode 100644 index 0000000..d3be7e2 --- /dev/null +++ b/dirsrvtests/tests/suites/password/pwdPolicy_attribute_test.py @@ -0,0 +1,339 @@ +import os +import sys +import time +import subprocess +import ldap +import logging +import pytest +from lib389 import DirSrv, Entry, tools, tasks +from lib389.tools import DirSrvTools +from lib389._constants import * +from lib389.properties import * +from lib389.tasks import * +from lib389.utils import * + +DEBUGGING = False +OU_PEOPLE = 'ou=people,{}'.format(DEFAULT_SUFFIX) +TEST_USER_NAME = 'simplepaged_test' +TEST_USER_DN = 'uid={},{}'.format(TEST_USER_NAME, OU_PEOPLE) +TEST_USER_PWD = 'simplepaged_test' +PW_POLICY_CONT_USER = 'cn="cn=nsPwPolicyEntry,uid=simplepaged_test,'\ + 'ou=people,dc=example,dc=com",'\ + 'cn=nsPwPolicyContainer,ou=people,dc=example,dc=com' +PW_POLICY_CONT_PEOPLE = 'cn="cn=nsPwPolicyEntry,'\ + 'ou=people,dc=example,dc=com",'\ + 'cn=nsPwPolicyContainer,ou=people,dc=example,dc=com' + +if DEBUGGING: + logging.getLogger(__name__).setLevel(logging.DEBUG) +else: + logging.getLogger(__name__).setLevel(logging.INFO) + +log = logging.getLogger(__name__) + + +class TopologyStandalone(object): + """The DS Topology Class""" + def __init__(self, standalone): + """Init""" + standalone.open() + self.standalone = standalone + + +(a)pytest.fixture(scope="module") +def topology(request): + """Create DS Deployment""" + + # Creating standalone instance ... + if DEBUGGING: + standalone = DirSrv(verbose=True) + else: + standalone = DirSrv(verbose=False) + args_instance[SER_HOST] = HOST_STANDALONE + args_instance[SER_PORT] = PORT_STANDALONE + args_instance[SER_SERVERID_PROP] = SERVERID_STANDALONE + args_instance[SER_CREATION_SUFFIX] = DEFAULT_SUFFIX + args_standalone = args_instance.copy() + standalone.allocate(args_standalone) + instance_standalone = standalone.exists() + if instance_standalone: + standalone.delete() + standalone.create() + standalone.open() + + def fin(): + """If we are debugging just stop the instances, otherwise remove + them + """ + if DEBUGGING: + standalone.stop() + else: + standalone.delete() + + request.addfinalizer(fin) + + + return TopologyStandalone(standalone) + + +(a)pytest.fixture(scope="module") +def test_user(topology, request): + """User for binding operation""" + + log.info('Adding user {}'.format(TEST_USER_DN)) + try: + topology.standalone.add_s(Entry((TEST_USER_DN, { + 'objectclass': 'top person'.split(), + 'objectclass': 'organizationalPerson', + 'objectclass': 'inetorgperson', + 'cn': TEST_USER_NAME, + 'sn': TEST_USER_NAME, + 'userpassword': TEST_USER_PWD, + 'mail': '%s(a)redhat.com' % TEST_USER_NAME, + 'uid': TEST_USER_NAME + }))) + except ldap.LDAPError as e: + log.error('Failed to add user (%s): error (%s)' % (TEST_USER_DN, + e.message['desc'])) + raise e + + def fin(): + log.info('Deleting user {}'.format(TEST_USER_DN)) + topology.standalone.delete_s(TEST_USER_DN) + request.addfinalizer(fin) + + +(a)pytest.fixture(scope="module") +def password_policy(topology, test_user): + """Set up password policy for subtree and user""" + + log.info('Enable fine-grained policy') + try: + topology.standalone.modify_s(DN_CONFIG, [(ldap.MOD_REPLACE, + 'nsslapd-pwpolicy-local', + 'on')]) + except ldap.LDAPError as e: + log.error('Failed to set fine-grained policy: error {}'.format( + e.message['desc'])) + raise e + + log.info('Create password policy for subtree {}'.format(OU_PEOPLE)) + try: + subprocess.call(['ns-newpwpolicy.pl', '-D', DN_DM, '-w', PASSWORD, + '-p', str(PORT_STANDALONE), '-h', HOST_STANDALONE, + '-S', OU_PEOPLE, '-Z', SERVERID_STANDALONE]) + except subprocess.CalledProcessError as e: + log.error('Failed to create pw policy policy for {}: error {}'.format( + OU_PEOPLE, e.message['desc'])) + raise e + + log.info('Add pwdpolicysubentry attribute to {}'.format(OU_PEOPLE)) + try: + topology.standalone.modify_s(OU_PEOPLE, [(ldap.MOD_REPLACE, + 'pwdpolicysubentry', + PW_POLICY_CONT_PEOPLE)]) + except ldap.LDAPError as e: + log.error('Failed to pwdpolicysubentry pw policy '\ + 'policy for {}: error {}'.format(OU_PEOPLE, + e.message['desc'])) + raise e + + log.info('Create password policy for subtree {}'.format(TEST_USER_DN)) + try: + subprocess.call(['ns-newpwpolicy.pl', '-D', DN_DM, '-w', PASSWORD, + '-p', str(PORT_STANDALONE), '-h', HOST_STANDALONE, + '-U', TEST_USER_DN, '-Z', SERVERID_STANDALONE]) + except subprocess.CalledProcessError as e: + log.error('Failed to create pw policy policy for {}: error {}'.format( + TEST_USER_DN, e.message['desc'])) + raise e + + log.info('Add pwdpolicysubentry attribute to {}'.format(TEST_USER_DN)) + try: + topology.standalone.modify_s(TEST_USER_DN, [(ldap.MOD_REPLACE, + 'pwdpolicysubentry', + PW_POLICY_CONT_USER)]) + except ldap.LDAPError as e: + log.error('Failed to pwdpolicysubentry pw policy '\ + 'policy for {}: error {}'.format(TEST_USER_DN, + e.message['desc'])) + raise e + + +(a)pytest.mark.parametrize('subtree_pwchange,user_pwchange,exception', + [('on', 'off', ldap.UNWILLING_TO_PERFORM), + ('off', 'off', ldap.UNWILLING_TO_PERFORM), + ('off', 'on', None), ('on', 'on', None)]) +def test_change_pwd(topology, test_user, password_policy, + subtree_pwchange, user_pwchange, exception): + """Verify that 'passwordChange' attr works as expected + User should have a priority over a subtree. + + :Feature: Password policy + + :Setup: Standalone instance, test user, + password policy entries for a user and a subtree + + :Steps: 1. Set passwordChange on the user and the subtree + to various combinations + 2. Bind as test user + 3. Try to change password + + :Assert: Subtree/User passwordChange - result + off/on, on/on - success + on/off, off/off - UNWILLING_TO_PERFORM + """ + + log.info('Set passwordChange to "{}" - {}'.format(subtree_pwchange, + PW_POLICY_CONT_PEOPLE)) + try: + topology.standalone.modify_s(PW_POLICY_CONT_PEOPLE, [(ldap.MOD_REPLACE, + 'passwordChange', + subtree_pwchange)]) + except ldap.LDAPError as e: + log.error('Failed to set passwordChange '\ + 'policy for {}: error {}'.format(PW_POLICY_CONT_PEOPLE, + e.message['desc'])) + raise e + + + log.info('Set passwordChange to "{}" - {}'.format(user_pwchange, + PW_POLICY_CONT_USER)) + try: + topology.standalone.modify_s(PW_POLICY_CONT_USER, [(ldap.MOD_REPLACE, + 'passwordChange', + user_pwchange)]) + except ldap.LDAPError as e: + log.error('Failed to set passwordChange '\ + 'policy for {}: error {}'.format(PW_POLICY_CONT_USER, + e.message['desc'])) + raise e + + try: + log.info('Bind as user and modify userPassword') + topology.standalone.simple_bind_s(TEST_USER_DN, TEST_USER_PWD) + if exception: + with pytest.raises(exception): + topology.standalone.modify_s(TEST_USER_DN, [(ldap.MOD_REPLACE, + 'userPassword', + 'new_pass')]) + else: + topology.standalone.modify_s(TEST_USER_DN, [(ldap.MOD_REPLACE, + 'userPassword', + 'new_pass')]) + except ldap.LDAPError as e: + log.error('Failed to change userpassword for {}: error {}'.format( + TEST_USER_DN, e.message['info'])) + raise e + finally: + log.info('Bind as DM') + topology.standalone.simple_bind_s(DN_DM, PASSWORD) + topology.standalone.modify_s(TEST_USER_DN, [(ldap.MOD_REPLACE, + 'userPassword', + TEST_USER_PWD)]) + + +def test_pwd_min_age(topology, test_user, password_policy): + """If we set passwordMinAge to some value, for example to 10, then it + should not allow the user to change the password within 10 seconds after + his previous change. + + :Feature: Password policy + + :Setup: Standalone instance, test user, + password policy entries for a user and a subtree + + :Steps: 1. Set passwordMinAge to 10 on the user pwpolicy entry + 2. Set passwordMinAge to 10 on the subtree pwpolicy entry + 3. Set passwordMinAge to 10 on the cn=config entry + 4. Bind as test user + 5. Try to change password two times in a row + 6. Wait 12 seconds + 7. Try to change password + + :Assert: User should be not allowed to change the password + right after previous change - CONSTRAINT_VIOLATION + User should be not allowed to change the password + after 12 seconds passed + """ + + num_seconds = '10' + + log.info('Set passwordminage to "{}" - {}'.format(num_seconds, PW_POLICY_CONT_PEOPLE)) + try: + topology.standalone.modify_s(PW_POLICY_CONT_PEOPLE, [(ldap.MOD_REPLACE, + 'passwordminage', + num_seconds)]) + except ldap.LDAPError as e: + log.error('Failed to set passwordminage '\ + 'policy for {}: error {}'.format(PW_POLICY_CONT_PEOPLE, + e.message['desc'])) + raise e + + log.info('Set passwordminage to "{}" - {}'.format(num_seconds, PW_POLICY_CONT_USER)) + try: + topology.standalone.modify_s(PW_POLICY_CONT_USER, [(ldap.MOD_REPLACE, + 'passwordminage', + num_seconds)]) + except ldap.LDAPError as e: + log.error('Failed to set passwordminage '\ + 'policy for {}: error {}'.format(PW_POLICY_CONT_USER, + e.message['desc'])) + raise e + + log.info('Set passwordminage to "{}" - {}'.format(num_seconds, DN_CONFIG)) + try: + topology.standalone.modify_s(DN_CONFIG, [(ldap.MOD_REPLACE, + 'passwordminage', + num_seconds)]) + except ldap.LDAPError as e: + log.error('Failed to set passwordminage '\ + 'policy for {}: error {}'.format(DN_CONFIG, + e.message['desc'])) + raise e + + try: + log.info('Bind as user and modify userPassword') + topology.standalone.simple_bind_s(TEST_USER_DN, TEST_USER_PWD) + topology.standalone.modify_s(TEST_USER_DN, [(ldap.MOD_REPLACE, + 'userPassword', + 'new_pass')]) + except ldap.LDAPError as e: + log.error('Failed to change userpassword for {}: error {}'.format( + TEST_USER_DN, e.message['info'])) + raise e + + + log.info('Bind as user and modify userPassword straight away after previous change') + topology.standalone.simple_bind_s(TEST_USER_DN, 'new_pass') + with pytest.raises(ldap.CONSTRAINT_VIOLATION): + topology.standalone.modify_s(TEST_USER_DN, [(ldap.MOD_REPLACE, + 'userPassword', + 'new_new_pass')]) + + log.info('Wait {} second'.format(int(num_seconds) + 2)) + time.sleep(int(num_seconds) + 2) + + try: + log.info('Bind as user and modify userPassword') + topology.standalone.simple_bind_s(TEST_USER_DN, 'new_pass') + topology.standalone.modify_s(TEST_USER_DN, [(ldap.MOD_REPLACE, + 'userPassword', + TEST_USER_PWD)]) + except ldap.LDAPError as e: + log.error('Failed to change userpassword for {}: error {}'.format( + TEST_USER_DN, e.message['info'])) + raise e + finally: + log.info('Bind as DM') + topology.standalone.simple_bind_s(DN_DM, PASSWORD) + topology.standalone.modify_s(TEST_USER_DN, [(ldap.MOD_REPLACE, + 'userPassword', + TEST_USER_PWD)]) + + +if __name__ == '__main__': + # Run isolated + # -s for DEBUG mode + CURRENT_FILE = os.path.realpath(__file__) + pytest.main("-s %s" % CURRENT_FILE) diff --git a/dirsrvtests/tests/suites/password/pwdPolicy_syntax_test.py b/dirsrvtests/tests/suites/password/pwdPolicy_syntax_test.py new file mode 100644 index 0000000..aad4e40 --- /dev/null +++ b/dirsrvtests/tests/suites/password/pwdPolicy_syntax_test.py @@ -0,0 +1,224 @@ +# --- BEGIN COPYRIGHT BLOCK --- +# Copyright (C) 2015 Red Hat, Inc. +# All rights reserved. +# +# License: GPL (version 3 or any later version). +# See LICENSE for details. +# --- END COPYRIGHT BLOCK --- +# +import os +import time +import ldap +import logging +import pytest +from lib389 import DirSrv, Entry +from lib389._constants import * +from lib389.properties import * +from lib389.tasks import * + +logging.getLogger(__name__).setLevel(logging.DEBUG) +log = logging.getLogger(__name__) + +DEBUGGING = False + +USER_DN = 'uid=user,ou=People,%s' % DEFAULT_SUFFIX + +if DEBUGGING: + logging.getLogger(__name__).setLevel(logging.DEBUG) +else: + logging.getLogger(__name__).setLevel(logging.INFO) + + +log = logging.getLogger(__name__) + + +class TopologyStandalone(object): + """The DS Topology Class""" + def __init__(self, standalone): + """Init""" + standalone.open() + self.standalone = standalone + + +(a)pytest.fixture(scope="module") +def topology(request): + """Create DS Deployment""" + + # Creating standalone instance ... + if DEBUGGING: + standalone = DirSrv(verbose=True) + else: + standalone = DirSrv(verbose=False) + args_instance[SER_HOST] = HOST_STANDALONE + args_instance[SER_PORT] = PORT_STANDALONE + args_instance[SER_SERVERID_PROP] = SERVERID_STANDALONE + args_instance[SER_CREATION_SUFFIX] = DEFAULT_SUFFIX + args_standalone = args_instance.copy() + standalone.allocate(args_standalone) + instance_standalone = standalone.exists() + if instance_standalone: + standalone.delete() + standalone.create() + standalone.open() + + def fin(): + """If we are debugging just stop the instances, otherwise remove + them + """ + if DEBUGGING: + standalone.stop() + else: + standalone.delete() + + request.addfinalizer(fin) + + return TopologyStandalone(standalone) + + +def _create_user(inst): + """Create the test user.""" + inst.add_s(Entry(( + USER_DN, { + 'objectClass': 'top account simplesecurityobject'.split(), + 'uid': 'user', + 'userpassword': PASSWORD + }))) + + +def setPolicy(inst, attr, value): + """Bind as ROot DN, set polcy, and then bind as user""" + try: + inst.simple_bind_s(DN_DM, PASSWORD) + except ldap.LDAPError as e: + log.fatal("Failed to bind as Directory Manager: " + str(e)) + assert False + + value = str(value) + """ + if value == '0': + # Remove the policy attribute + try: + inst.modify_s("cn=config", + [(ldap.MOD_DELETE, attr, None)]) + except ldap.LDAPError as e: + log.fatal("Failed to rmeove password policy %s: %s" % + (attr, str(e))) + assert False + else: + """ + # Set the policy value + inst.config.set(attr, value) + + try: + inst.simple_bind_s(USER_DN, PASSWORD) + except ldap.LDAPError as e: + log.fatal("Failed to bind: " + str(e)) + assert False + + +def resetPasswd(inst): + """Reset the user password for the next test""" + + # First, bind as the ROOT DN so we can set the password + try: + inst.simple_bind_s(DN_DM, PASSWORD) + except ldap.LDAPError as e: + log.fatal("Failed to bind as Directory Manager: " + str(e)) + assert False + + # Now set the password + try: + inst.modify_s(USER_DN, + [(ldap.MOD_REPLACE, 'userpassword', PASSWORD)]) + except ldap.LDAPError as e: + log.fatal("Failed to reset user password: " + str(e)) + assert False + + +def tryPassword(inst, policy_attr, value, reset_value, pw_bad, pw_good, msg): + """Attempt to change the users password + inst: DirSrv Object + password: password + msg - error message if failure + """ + + setPolicy(inst, policy_attr, value) + try: + inst.modify_s(USER_DN, + [(ldap.MOD_REPLACE, 'userpassword', pw_bad)]) + log.fatal('Invalid password was unexpectedly accepted (%s)' % + (policy_attr)) + assert False + except ldap.CONSTRAINT_VIOLATION: + log.info('Invalid password correctly rejected by %s: %s' % + (policy_attr, msg)) + pass + except ldap.LDAPError as e: + log.fatal("Failed to change password: " + str(e)) + assert False + + # Change password that is allowed + try: + inst.modify_s(USER_DN, + [(ldap.MOD_REPLACE, 'userpassword', pw_good)]) + except ldap.LDAPError as e: + log.fatal("Failed to change password: " + str(e)) + assert False + + # Reset for the next test + resetPasswd(inst) + setPolicy(inst, policy_attr, reset_value) + + +def test_pwdPolicy_syntax(topology): + ''' + Password policy test: Ensure that on a password change, the policy syntax + is enforced correctly. + ''' + + # Create a user + _create_user(topology.standalone) + + # Set the password policy globally + topology.standalone.config.set('passwordCheckSyntax', 'on') + topology.standalone.config.set('nsslapd-pwpolicy-local', 'off') + topology.standalone.config.set('passwordMinCategories', '1') + + # + # Test each syntax catagory + # + + # Min Length + tryPassword(topology.standalone, 'passwordMinLength', 10, 2, 'passwd', + 'password123', 'length too short') + # Min Digit + tryPassword(topology.standalone, 'passwordMinDigits', 2, 0, 'passwd', + 'password123', 'does not contain minimum number of digits') + # Min Alphas + tryPassword(topology.standalone, 'passwordMinAlphas', 2, 0, 'p123456789', + 'password123', 'does not contain minimum number of alphas') + # Max Repeats + tryPassword(topology.standalone, 'passwordMaxRepeats', 2, 0, 'passsword', + 'pasword123', 'too many repeating characters') + # Min Specials + tryPassword(topology.standalone, 'passwordMinSpecials', 2, 0, 'passwd', + 'password_#$', + 'does not contain minimum number of special characters') + # Min Lowers + tryPassword(topology.standalone, 'passwordMinLowers', 2, 0, 'PASSWORD123', + 'password123', + 'does not contain minimum number of lowercase characters') + # Min Uppers + tryPassword(topology.standalone, 'passwordMinUppers', 2, 0, 'password', + 'PASSWORD', + 'does not contain minimum number of lowercase characters') + # Min 8-bits - "ldap" package only accepts ascii strings at the moment + + log.info('pwdPolicy tests PASSED') + + +if __name__ == '__main__': + # Run isolated + # -s for DEBUG mode + CURRENT_FILE = os.path.realpath(__file__) + pytest.main("-s %s" % CURRENT_FILE) diff --git a/dirsrvtests/tests/suites/password/pwdPolicy_test.py b/dirsrvtests/tests/suites/password/pwdPolicy_test.py deleted file mode 100644 index aad4e40..0000000 --- a/dirsrvtests/tests/suites/password/pwdPolicy_test.py +++ /dev/null @@ -1,224 +0,0 @@ -# --- BEGIN COPYRIGHT BLOCK --- -# Copyright (C) 2015 Red Hat, Inc. -# All rights reserved. -# -# License: GPL (version 3 or any later version). -# See LICENSE for details. -# --- END COPYRIGHT BLOCK --- -# -import os -import time -import ldap -import logging -import pytest -from lib389 import DirSrv, Entry -from lib389._constants import * -from lib389.properties import * -from lib389.tasks import * - -logging.getLogger(__name__).setLevel(logging.DEBUG) -log = logging.getLogger(__name__) - -DEBUGGING = False - -USER_DN = 'uid=user,ou=People,%s' % DEFAULT_SUFFIX - -if DEBUGGING: - logging.getLogger(__name__).setLevel(logging.DEBUG) -else: - logging.getLogger(__name__).setLevel(logging.INFO) - - -log = logging.getLogger(__name__) - - -class TopologyStandalone(object): - """The DS Topology Class""" - def __init__(self, standalone): - """Init""" - standalone.open() - self.standalone = standalone - - -(a)pytest.fixture(scope="module") -def topology(request): - """Create DS Deployment""" - - # Creating standalone instance ... - if DEBUGGING: - standalone = DirSrv(verbose=True) - else: - standalone = DirSrv(verbose=False) - args_instance[SER_HOST] = HOST_STANDALONE - args_instance[SER_PORT] = PORT_STANDALONE - args_instance[SER_SERVERID_PROP] = SERVERID_STANDALONE - args_instance[SER_CREATION_SUFFIX] = DEFAULT_SUFFIX - args_standalone = args_instance.copy() - standalone.allocate(args_standalone) - instance_standalone = standalone.exists() - if instance_standalone: - standalone.delete() - standalone.create() - standalone.open() - - def fin(): - """If we are debugging just stop the instances, otherwise remove - them - """ - if DEBUGGING: - standalone.stop() - else: - standalone.delete() - - request.addfinalizer(fin) - - return TopologyStandalone(standalone) - - -def _create_user(inst): - """Create the test user.""" - inst.add_s(Entry(( - USER_DN, { - 'objectClass': 'top account simplesecurityobject'.split(), - 'uid': 'user', - 'userpassword': PASSWORD - }))) - - -def setPolicy(inst, attr, value): - """Bind as ROot DN, set polcy, and then bind as user""" - try: - inst.simple_bind_s(DN_DM, PASSWORD) - except ldap.LDAPError as e: - log.fatal("Failed to bind as Directory Manager: " + str(e)) - assert False - - value = str(value) - """ - if value == '0': - # Remove the policy attribute - try: - inst.modify_s("cn=config", - [(ldap.MOD_DELETE, attr, None)]) - except ldap.LDAPError as e: - log.fatal("Failed to rmeove password policy %s: %s" % - (attr, str(e))) - assert False - else: - """ - # Set the policy value - inst.config.set(attr, value) - - try: - inst.simple_bind_s(USER_DN, PASSWORD) - except ldap.LDAPError as e: - log.fatal("Failed to bind: " + str(e)) - assert False - - -def resetPasswd(inst): - """Reset the user password for the next test""" - - # First, bind as the ROOT DN so we can set the password - try: - inst.simple_bind_s(DN_DM, PASSWORD) - except ldap.LDAPError as e: - log.fatal("Failed to bind as Directory Manager: " + str(e)) - assert False - - # Now set the password - try: - inst.modify_s(USER_DN, - [(ldap.MOD_REPLACE, 'userpassword', PASSWORD)]) - except ldap.LDAPError as e: - log.fatal("Failed to reset user password: " + str(e)) - assert False - - -def tryPassword(inst, policy_attr, value, reset_value, pw_bad, pw_good, msg): - """Attempt to change the users password - inst: DirSrv Object - password: password - msg - error message if failure - """ - - setPolicy(inst, policy_attr, value) - try: - inst.modify_s(USER_DN, - [(ldap.MOD_REPLACE, 'userpassword', pw_bad)]) - log.fatal('Invalid password was unexpectedly accepted (%s)' % - (policy_attr)) - assert False - except ldap.CONSTRAINT_VIOLATION: - log.info('Invalid password correctly rejected by %s: %s' % - (policy_attr, msg)) - pass - except ldap.LDAPError as e: - log.fatal("Failed to change password: " + str(e)) - assert False - - # Change password that is allowed - try: - inst.modify_s(USER_DN, - [(ldap.MOD_REPLACE, 'userpassword', pw_good)]) - except ldap.LDAPError as e: - log.fatal("Failed to change password: " + str(e)) - assert False - - # Reset for the next test - resetPasswd(inst) - setPolicy(inst, policy_attr, reset_value) - - -def test_pwdPolicy_syntax(topology): - ''' - Password policy test: Ensure that on a password change, the policy syntax - is enforced correctly. - ''' - - # Create a user - _create_user(topology.standalone) - - # Set the password policy globally - topology.standalone.config.set('passwordCheckSyntax', 'on') - topology.standalone.config.set('nsslapd-pwpolicy-local', 'off') - topology.standalone.config.set('passwordMinCategories', '1') - - # - # Test each syntax catagory - # - - # Min Length - tryPassword(topology.standalone, 'passwordMinLength', 10, 2, 'passwd', - 'password123', 'length too short') - # Min Digit - tryPassword(topology.standalone, 'passwordMinDigits', 2, 0, 'passwd', - 'password123', 'does not contain minimum number of digits') - # Min Alphas - tryPassword(topology.standalone, 'passwordMinAlphas', 2, 0, 'p123456789', - 'password123', 'does not contain minimum number of alphas') - # Max Repeats - tryPassword(topology.standalone, 'passwordMaxRepeats', 2, 0, 'passsword', - 'pasword123', 'too many repeating characters') - # Min Specials - tryPassword(topology.standalone, 'passwordMinSpecials', 2, 0, 'passwd', - 'password_#$', - 'does not contain minimum number of special characters') - # Min Lowers - tryPassword(topology.standalone, 'passwordMinLowers', 2, 0, 'PASSWORD123', - 'password123', - 'does not contain minimum number of lowercase characters') - # Min Uppers - tryPassword(topology.standalone, 'passwordMinUppers', 2, 0, 'password', - 'PASSWORD', - 'does not contain minimum number of lowercase characters') - # Min 8-bits - "ldap" package only accepts ascii strings at the moment - - log.info('pwdPolicy tests PASSED') - - -if __name__ == '__main__': - # Run isolated - # -s for DEBUG mode - CURRENT_FILE = os.path.realpath(__file__) - pytest.main("-s %s" % CURRENT_FILE) diff --git a/dirsrvtests/tests/suites/password/pwd_change_policytest.py b/dirsrvtests/tests/suites/password/pwd_change_policytest.py deleted file mode 100644 index 1d48c65..0000000 --- a/dirsrvtests/tests/suites/password/pwd_change_policytest.py +++ /dev/null @@ -1,240 +0,0 @@ -import os -import sys -import time -import subprocess -import ldap -import logging -import pytest -from lib389 import DirSrv, Entry, tools, tasks -from lib389.tools import DirSrvTools -from lib389._constants import * -from lib389.properties import * -from lib389.tasks import * -from lib389.utils import * - -DEBUGGING = False -OU_PEOPLE = 'ou=people,{}'.format(DEFAULT_SUFFIX) -TEST_USER_NAME = 'simplepaged_test' -TEST_USER_DN = 'uid={},{}'.format(TEST_USER_NAME, OU_PEOPLE) -TEST_USER_PWD = 'simplepaged_test' -PW_POLICY_CONT_USER = 'cn="cn=nsPwPolicyEntry,uid=simplepaged_test,'\ - 'ou=people,dc=example,dc=com",'\ - 'cn=nsPwPolicyContainer,ou=people,dc=example,dc=com' -PW_POLICY_CONT_PEOPLE = 'cn="cn=nsPwPolicyEntry,'\ - 'ou=people,dc=example,dc=com",'\ - 'cn=nsPwPolicyContainer,ou=people,dc=example,dc=com' - -if DEBUGGING: - logging.getLogger(__name__).setLevel(logging.DEBUG) -else: - logging.getLogger(__name__).setLevel(logging.INFO) - -log = logging.getLogger(__name__) - - -class TopologyStandalone(object): - """The DS Topology Class""" - def __init__(self, standalone): - """Init""" - standalone.open() - self.standalone = standalone - - -(a)pytest.fixture(scope="module") -def topology(request): - """Create DS Deployment""" - - # Creating standalone instance ... - if DEBUGGING: - standalone = DirSrv(verbose=True) - else: - standalone = DirSrv(verbose=False) - args_instance[SER_HOST] = HOST_STANDALONE - args_instance[SER_PORT] = PORT_STANDALONE - args_instance[SER_SERVERID_PROP] = SERVERID_STANDALONE - args_instance[SER_CREATION_SUFFIX] = DEFAULT_SUFFIX - args_standalone = args_instance.copy() - standalone.allocate(args_standalone) - instance_standalone = standalone.exists() - if instance_standalone: - standalone.delete() - standalone.create() - standalone.open() - - def fin(): - """If we are debugging just stop the instances, otherwise remove - them - """ - if DEBUGGING: - standalone.stop() - else: - standalone.delete() - - request.addfinalizer(fin) - - - return TopologyStandalone(standalone) - - -(a)pytest.fixture(scope="module") -def test_user(topology, request): - """User for binding operation""" - - log.info('Adding user {}'.format(TEST_USER_DN)) - try: - topology.standalone.add_s(Entry((TEST_USER_DN, { - 'objectclass': 'top person'.split(), - 'objectclass': 'organizationalPerson', - 'objectclass': 'inetorgperson', - 'cn': TEST_USER_NAME, - 'sn': TEST_USER_NAME, - 'userpassword': TEST_USER_PWD, - 'mail': '%s(a)redhat.com' % TEST_USER_NAME, - 'uid': TEST_USER_NAME - }))) - except ldap.LDAPError as e: - log.error('Failed to add user (%s): error (%s)' % (TEST_USER_DN, - e.message['desc'])) - raise e - - def fin(): - log.info('Deleting user {}'.format(TEST_USER_DN)) - topology.standalone.delete_s(TEST_USER_DN) - request.addfinalizer(fin) - - -(a)pytest.fixture(scope="module") -def password_policy(topology, test_user): - """Set up password policy for subtree and user""" - - log.info('Enable fine-grained policy') - try: - topology.standalone.modify_s(DN_CONFIG, [(ldap.MOD_REPLACE, - 'nsslapd-pwpolicy-local', - 'on')]) - except ldap.LDAPError as e: - log.error('Failed to set fine-grained policy: error {}'.format( - e.message['desc'])) - raise e - - log.info('Create password policy for subtree {}'.format(OU_PEOPLE)) - try: - subprocess.call(['ns-newpwpolicy.pl', '-D', DN_DM, '-w', PASSWORD, - '-p', str(PORT_STANDALONE), '-h', HOST_STANDALONE, - '-S', OU_PEOPLE, '-Z', SERVERID_STANDALONE]) - except subprocess.CalledProcessError as e: - log.error('Failed to create pw policy policy for {}: error {}'.format( - OU_PEOPLE, e.message['desc'])) - raise e - - log.info('Add pwdpolicysubentry attribute to {}'.format(OU_PEOPLE)) - try: - topology.standalone.modify_s(OU_PEOPLE, [(ldap.MOD_REPLACE, - 'pwdpolicysubentry', - PW_POLICY_CONT_PEOPLE)]) - except ldap.LDAPError as e: - log.error('Failed to pwdpolicysubentry pw policy '\ - 'policy for {}: error {}'.format(OU_PEOPLE, - e.message['desc'])) - raise e - - log.info('Create password policy for subtree {}'.format(TEST_USER_DN)) - try: - subprocess.call(['ns-newpwpolicy.pl', '-D', DN_DM, '-w', PASSWORD, - '-p', str(PORT_STANDALONE), '-h', HOST_STANDALONE, - '-U', TEST_USER_DN, '-Z', SERVERID_STANDALONE]) - except subprocess.CalledProcessError as e: - log.error('Failed to create pw policy policy for {}: error {}'.format( - TEST_USER_DN, e.message['desc'])) - raise e - - log.info('Add pwdpolicysubentry attribute to {}'.format(TEST_USER_DN)) - try: - topology.standalone.modify_s(TEST_USER_DN, [(ldap.MOD_REPLACE, - 'pwdpolicysubentry', - PW_POLICY_CONT_USER)]) - except ldap.LDAPError as e: - log.error('Failed to pwdpolicysubentry pw policy '\ - 'policy for {}: error {}'.format(TEST_USER_DN, - e.message['desc'])) - raise e - - -(a)pytest.mark.parametrize('subtree_pwchange,user_pwchange,exception', - [('off', 'on', None), ('on', 'on', None), - ('on', 'off', ldap.UNWILLING_TO_PERFORM), - ('off', 'off', ldap.UNWILLING_TO_PERFORM)]) -def test_change_pwd(topology, test_user, password_policy, - subtree_pwchange, user_pwchange, exception): - """Verify that 'passwordChange' attr works as expected - User should have a priority over a subtree. - - :Feature: Password policy - - :Setup: Standalone instance, test user, - password policy entries for a user and a subtree - - :Steps: 1. Set passwordChange on the user and the subtree - to various combinations - 2. Bind as test user - 3. Try to change password - - :Assert: Subtree/User passwordChange - result - off/on, on/on - success - on/off, off/off - UNWILLING_TO_PERFORM - """ - - log.info('Set passwordChange to "{}" - {}'.format(subtree_pwchange, - PW_POLICY_CONT_PEOPLE)) - try: - topology.standalone.modify_s(PW_POLICY_CONT_PEOPLE, [(ldap.MOD_REPLACE, - 'passwordChange', - subtree_pwchange)]) - except ldap.LDAPError as e: - log.error('Failed to set passwordChange '\ - 'policy for {}: error {}'.format(PW_POLICY_CONT_PEOPLE, - e.message['desc'])) - raise e - - - log.info('Set passwordChange to "{}" - {}'.format(user_pwchange, - PW_POLICY_CONT_USER)) - try: - topology.standalone.modify_s(PW_POLICY_CONT_USER, [(ldap.MOD_REPLACE, - 'passwordChange', - user_pwchange)]) - except ldap.LDAPError as e: - log.error('Failed to set passwordChange '\ - 'policy for {}: error {}'.format(PW_POLICY_CONT_USER, - e.message['desc'])) - raise e - - try: - log.info('Bind as user and modify userPassword') - topology.standalone.simple_bind_s(TEST_USER_DN, TEST_USER_PWD) - if exception: - with pytest.raises(exception): - topology.standalone.modify_s(TEST_USER_DN, [(ldap.MOD_REPLACE, - 'userPassword', - 'new_pass')]) - else: - topology.standalone.modify_s(TEST_USER_DN, [(ldap.MOD_REPLACE, - 'userPassword', - 'new_pass')]) - except ldap.LDAPError as e: - log.error('Failed to change userpassword for {}: error {}'.format( - TEST_USER_DN, e.message['info'])) - raise e - finally: - log.info('Bind as DM') - topology.standalone.simple_bind_s(DN_DM, PASSWORD) - topology.standalone.modify_s(TEST_USER_DN, [(ldap.MOD_REPLACE, - 'userPassword', - TEST_USER_PWD)]) - - -if __name__ == '__main__': - # Run isolated - # -s for DEBUG mode - CURRENT_FILE = os.path.realpath(__file__) - pytest.main("-s %s" % CURRENT_FILE)

1 0

ldap/servers
by Noriko Hosoi 24 Aug '16

24 Aug '16

ldap/servers/slapd/pw.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) New commits: commit 790e723e4f30b3d245535ce7a9d5d00477878341 Author: Noriko Hosoi <nhosoi(a)redhat.com> Date: Tue Aug 23 14:18:32 2016 -0700 Ticket #48967 - passwordMinAge attribute doesn't limit the minimum age of the password Description: There was a logic error in check_pw_minage. Password- MinAge was ignored by the error. This patch fixes the logic error. https://fedorahosted.org/389/ticket/48967 Reviewed and tested by wibrown(a)redhat.com and spichugi(a)redhat.com. (Thank you, William and Simon!) diff --git a/ldap/servers/slapd/pw.c b/ldap/servers/slapd/pw.c index 6b865ec..7469b9e 100644 --- a/ldap/servers/slapd/pw.c +++ b/ldap/servers/slapd/pw.c @@ -729,7 +729,7 @@ check_pw_minage ( Slapi_PBlock *pb, const Slapi_DN *sdn, struct berval **vals) pwpolicy = new_passwdPolicy(pb, dn); slapi_pblock_get ( pb, SLAPI_PWPOLICY, &pwresponse_req ); - if (!pb->pb_op->o_isroot && !pwpolicy->pw_minage) { + if (!pb->pb_op->o_isroot && pwpolicy->pw_minage) { Slapi_Entry *e; char *passwordAllowChangeTime;

1 0

ldap/servers
by Ludwig Krispenz 24 Aug '16

24 Aug '16

ldap/servers/slapd/back-ldbm/dblayer.c | 2 -- 1 file changed, 2 deletions(-) New commits: commit 3a08be376a640fbbfc4ea3e2d29a6def265e0614 Author: Ludwig Krispenz <lkrispen(a)redhat.com> Date: Wed Aug 24 15:03:50 2016 +0200 fix jenkins warnings about unused vars diff --git a/ldap/servers/slapd/back-ldbm/dblayer.c b/ldap/servers/slapd/back-ldbm/dblayer.c index c8d242e..c84196a 100644 --- a/ldap/servers/slapd/back-ldbm/dblayer.c +++ b/ldap/servers/slapd/back-ldbm/dblayer.c @@ -7124,7 +7124,6 @@ dblayer_restore_file_init(struct ldbminfo *li) void dblayer_import_file_update(ldbm_instance *inst) { - int rc; PRFileDesc *prfd; char *fname = dblayer_import_file_name(inst); dblayer_file_open(fname, PR_RDWR, inst->inst_li->li_mode, &prfd); @@ -7196,7 +7195,6 @@ dblayer_restore_file_check(struct ldbminfo *li) void dblayer_restore_file_update(struct ldbminfo *li, char *directory) { - int rc; PRFileDesc *prfd; char *fname = dblayer_restore_file_name(li); dblayer_file_open(fname, PR_RDWR, li->li_mode, &prfd);

1 0

ldap/servers
by Ludwig Krispenz 24 Aug '16

24 Aug '16

ldap/servers/slapd/back-ldbm/archive.c | 9 + ldap/servers/slapd/back-ldbm/dblayer.c | 166 +++++++++++++++++++++++++ ldap/servers/slapd/back-ldbm/instance.c | 12 + ldap/servers/slapd/back-ldbm/ldif2ldbm.c | 18 ++ ldap/servers/slapd/back-ldbm/proto-back-ldbm.h | 9 + ldap/servers/slapd/bulk_import.c | 3 ldap/servers/slapd/slapi-plugin.h | 2 7 files changed, 217 insertions(+), 2 deletions(-) New commits: commit 900c32f3b0610c3daf31023769fa541e03351b84 Author: Ludwig Krispenz <lkrispen(a)redhat.com> Date: Tue Jun 28 17:07:34 2016 +0200 Ticket 48402 - v3 allow plugins to detect a restore or import Bug Description: RFE plugins do not know if the database was imported or restored, it could require different behaviour Fix Description: set flags in the backends whenever an import or restore was detected. see: http://www.port389.org/docs/389ds/design/detect-startup-after-import-or-res… https://fedorahosted.org/389/ticket/48402 Reviewed by: Noriko, thanks diff --git a/ldap/servers/slapd/back-ldbm/archive.c b/ldap/servers/slapd/back-ldbm/archive.c index 6b56937..64d558c 100644 --- a/ldap/servers/slapd/back-ldbm/archive.c +++ b/ldap/servers/slapd/back-ldbm/archive.c @@ -69,6 +69,12 @@ int ldbm_back_archive2ldbm( Slapi_PBlock *pb ) if (run_from_cmdline) { mapping_tree_init(); ldbm_config_load_dse_info(li); + + /* initialize a restore file to be able to detect a startup after restore */ + if (dblayer_restore_file_init(li)) { + LDAPDebug0Args(LDAP_DEBUG_ANY, "archive2db: failed to write restore file.\n"); + return -1; + } } if (backendname) { inst = ldbm_instance_find_by_name(li, backendname); @@ -249,6 +255,9 @@ int ldbm_back_archive2ldbm( Slapi_PBlock *pb ) } } out: + if (run_from_cmdline && ( 0 == return_value )) { + dblayer_restore_file_update(li, directory); + } slapi_ch_free_string(&directory); return return_value; } diff --git a/ldap/servers/slapd/back-ldbm/dblayer.c b/ldap/servers/slapd/back-ldbm/dblayer.c index 783d104..c8d242e 100644 --- a/ldap/servers/slapd/back-ldbm/dblayer.c +++ b/ldap/servers/slapd/back-ldbm/dblayer.c @@ -203,6 +203,8 @@ static void dblayer_push_pvt_txn(back_txn *txn); static back_txn *dblayer_get_pvt_txn(); static void dblayer_pop_pvt_txn(); +static int dblayer_post_restore = 0; + #define MEGABYTE (1024 * 1024) #define GIGABYTE (1024 * MEGABYTE) @@ -1404,6 +1406,9 @@ dblayer_start(struct ldbminfo *li, int dbmode) /* The error message was output by read_metadata() */ return -1; } + if (dblayer_restore_file_check(li)) { + dblayer_set_restored(); + } } dblayer_free_env(&priv->dblayer_env); @@ -7054,6 +7059,167 @@ error_out: return return_value; } +static char * +dblayer_import_file_name(ldbm_instance *inst) +{ + char *fname = slapi_ch_smprintf("%s/.import_%s", + inst->inst_parent_dir_name, + inst->inst_dir_name); + return fname; +} + +static char * +dblayer_restore_file_name(struct ldbminfo *li) +{ + char *fname = slapi_ch_smprintf("%s/../.restore", li->li_directory); + + return fname; +} + +static int +dblayer_file_open(char *fname, int flags, int mode, PRFileDesc **prfd) +{ + int rc = 0; + *prfd = PR_Open(fname, flags, mode ); + + if (NULL == *prfd) rc = PR_GetError(); + if (rc && rc != PR_FILE_NOT_FOUND_ERROR ) { + LDAPDebug(LDAP_DEBUG_ANY, + "Failed to open file: %s, error: (%d) %s\n", + fname, rc, slapd_pr_strerror(rc)); + } + return rc; +} + +int +dblayer_import_file_init(ldbm_instance *inst) +{ + int rc = -1; + PRFileDesc *prfd = NULL; + char *fname = dblayer_import_file_name(inst); + rc = dblayer_file_open(fname, PR_RDWR | PR_CREATE_FILE | PR_TRUNCATE, inst->inst_li->li_mode, &prfd); + if (prfd) { + PR_Close(prfd); + rc = 0; + } + slapi_ch_free_string(&fname); + return rc; +} + +int +dblayer_restore_file_init(struct ldbminfo *li) +{ + int rc = -1; + PRFileDesc *prfd; + char *fname = dblayer_restore_file_name(li); + rc = dblayer_file_open(fname, PR_RDWR | PR_CREATE_FILE | PR_TRUNCATE, li->li_mode, &prfd); + if (prfd) { + PR_Close(prfd); + rc = 0; + } + slapi_ch_free_string(&fname); + return rc; + +} +void +dblayer_import_file_update(ldbm_instance *inst) +{ + int rc; + PRFileDesc *prfd; + char *fname = dblayer_import_file_name(inst); + dblayer_file_open(fname, PR_RDWR, inst->inst_li->li_mode, &prfd); + + if (prfd) { + char *line = slapi_ch_smprintf("import of %s succeeded",inst->inst_dir_name); + slapi_write_buffer(prfd, line, strlen(line)); + slapi_ch_free_string(&line); + PR_Close(prfd); + } + slapi_ch_free_string(&fname); +} + +int +dblayer_file_check(char *fname, int mode) +{ + int rc = 0; + int err; + PRFileDesc *prfd; + err = dblayer_file_open(fname, PR_RDWR, mode, &prfd); + + if (prfd) { + /* file exists, additional check on size */ + PRFileInfo64 prfinfo; + rc = 1; + /* read it */ + err = PR_GetOpenFileInfo64(prfd, &prfinfo); + if (err == PR_SUCCESS && 0 == prfinfo.size) { + /* it is empty restore or import has failed */ + LDAPDebug1Arg(LDAP_DEBUG_ANY, + "Previous import or restore failed, file: %s is empty\n", fname); + } + PR_Close(prfd); + PR_Delete(fname); + } else { + if (PR_FILE_NOT_FOUND_ERROR == err) { + rc = 0; + } else { + /* file exists, but we cannot open it */ + rc = 1; + /* error is already looged try to delete it*/ + PR_Delete(fname); + } + } + + return rc; + +} +int +dblayer_import_file_check(ldbm_instance *inst) +{ + int rc; + char *fname = dblayer_import_file_name(inst); + rc = dblayer_file_check(fname, inst->inst_li->li_mode); + slapi_ch_free_string(&fname); + return rc; +} + +int +dblayer_restore_file_check(struct ldbminfo *li) +{ + int rc; + char *fname = dblayer_restore_file_name(li); + rc = dblayer_file_check(fname, li->li_mode); + slapi_ch_free_string(&fname); + return rc; +} + +void +dblayer_restore_file_update(struct ldbminfo *li, char *directory) +{ + int rc; + PRFileDesc *prfd; + char *fname = dblayer_restore_file_name(li); + dblayer_file_open(fname, PR_RDWR, li->li_mode, &prfd); + if (prfd) { + char *line = slapi_ch_smprintf("restore of %s succeeded", directory); + slapi_write_buffer(prfd, line, strlen(line)); + slapi_ch_free_string(&line); + PR_Close(prfd); + } +} + +int +dblayer_is_restored(void) +{ + return dblayer_post_restore; +} + +void +dblayer_set_restored(void) +{ + dblayer_post_restore = 1; +} + /* * inst_dir_name is a relative path (from 6.21) * ==> txn log stores relative paths and becomes relocatable diff --git a/ldap/servers/slapd/back-ldbm/instance.c b/ldap/servers/slapd/back-ldbm/instance.c index 53d922e..f059e8a 100644 --- a/ldap/servers/slapd/back-ldbm/instance.c +++ b/ldap/servers/slapd/back-ldbm/instance.c @@ -299,7 +299,16 @@ ldbm_instance_stop(backend *be) return rc; } - +static void +ldbm_instance_set_flags(ldbm_instance *inst) +{ + if (dblayer_is_restored()) { + slapi_be_set_flag(inst->inst_be, SLAPI_BE_FLAG_POST_RESTORE); + } + if (dblayer_import_file_check(inst)) { + slapi_be_set_flag(inst->inst_be, SLAPI_BE_FLAG_POST_IMPORT); + } +} /* Walks down the set of instances, starting each one. */ int @@ -313,6 +322,7 @@ ldbm_instance_startall(struct ldbminfo *li) while (inst_obj != NULL) { int rc1; inst = (ldbm_instance *) object_get_data(inst_obj); + ldbm_instance_set_flags(inst); rc1 = ldbm_instance_start(inst->inst_be); if (rc1 != 0) { rc = rc1; diff --git a/ldap/servers/slapd/back-ldbm/ldif2ldbm.c b/ldap/servers/slapd/back-ldbm/ldif2ldbm.c index 5898361..7aae93d 100644 --- a/ldap/servers/slapd/back-ldbm/ldif2ldbm.c +++ b/ldap/servers/slapd/back-ldbm/ldif2ldbm.c @@ -690,6 +690,14 @@ int ldbm_back_ldif2ldbm( Slapi_PBlock *pb ) return -1; } + if ((task_flags & SLAPI_TASK_RUNNING_FROM_COMMANDLINE)) { + if (dblayer_import_file_init(inst)) { + LDAPDebug0Args(LDAP_DEBUG_ANY, + "ldbm_back_ldif2ldbm: failed to write import file\n"); + return -1; + } + } + /***** prepare & init libdb and dblayer *****/ if (! (task_flags & SLAPI_TASK_RUNNING_FROM_COMMANDLINE)) { @@ -766,7 +774,15 @@ int ldbm_back_ldif2ldbm( Slapi_PBlock *pb ) /* always use "new" import code now */ slapi_pblock_set(pb, SLAPI_BACKEND, inst->inst_be); - return ldbm_back_ldif2ldbm_deluxe(pb); + ret = ldbm_back_ldif2ldbm_deluxe(pb); + if (ret == 0) { + if (task_flags & SLAPI_TASK_RUNNING_FROM_COMMANDLINE) { + dblayer_import_file_update(inst); + } else { + slapi_be_set_flag(inst->inst_be, SLAPI_BE_FLAG_POST_IMPORT); + } + } + return ret; fail: /* DON'T enable the backend -- leave it offline */ diff --git a/ldap/servers/slapd/back-ldbm/proto-back-ldbm.h b/ldap/servers/slapd/back-ldbm/proto-back-ldbm.h index 8c813dd..1647163 100644 --- a/ldap/servers/slapd/back-ldbm/proto-back-ldbm.h +++ b/ldap/servers/slapd/back-ldbm/proto-back-ldbm.h @@ -168,6 +168,15 @@ int ldbm_back_get_info(Slapi_Backend *be, int cmd, void **info); int ldbm_back_set_info(Slapi_Backend *be, int cmd, void *info); int ldbm_back_ctrl_info(Slapi_Backend *be, int cmd, void *info); +int dblayer_is_restored(void); +void dblayer_set_restored(void); +int dblayer_restore_file_check(struct ldbminfo *li); +int dblayer_restore_file_init(struct ldbminfo *li); +void dblayer_restore_file_update(struct ldbminfo *li, char *directory); +int dblayer_import_file_init(ldbm_instance *inst); +void dblayer_import_file_update(ldbm_instance *inst); +int dblayer_import_file_check(ldbm_instance *inst); + /* * dn2entry.c */ diff --git a/ldap/servers/slapd/bulk_import.c b/ldap/servers/slapd/bulk_import.c index c63faa1..35f9855 100644 --- a/ldap/servers/slapd/bulk_import.c +++ b/ldap/servers/slapd/bulk_import.c @@ -146,6 +146,9 @@ process_bulk_import_op (Slapi_PBlock *pb, int state, Slapi_Entry *e) "failed; error = %d\n", rc); return LDAP_OPERATIONS_ERROR; } + if (state == SLAPI_BI_STATE_DONE) { + slapi_be_set_flag(be, SLAPI_BE_FLAG_POST_IMPORT); + } return LDAP_SUCCESS; } diff --git a/ldap/servers/slapd/slapi-plugin.h b/ldap/servers/slapd/slapi-plugin.h index a7e544a..89853c0 100644 --- a/ldap/servers/slapd/slapi-plugin.h +++ b/ldap/servers/slapd/slapi-plugin.h @@ -6460,6 +6460,8 @@ void slapi_be_set_flag(Slapi_Backend * be, int flag); void slapi_be_unset_flag(Slapi_Backend * be, int flag); #define SLAPI_BE_FLAG_REMOTE_DATA 0x1 /* entries held by backend are remote */ #define SLAPI_BE_FLAG_DONT_BYPASS_FILTERTEST 0x10 /* force to call filter_test (search only) */ +#define SLAPI_BE_FLAG_POST_IMPORT 0x100 /* backend was imported */ +#define SLAPI_BE_FLAG_POST_RESTORE 0x200 /* startup after restore */ /* These functions allow a plugin to register for callback when

1 0

ldap/ldif ldap/servers
by Noriko Hosoi 23 Aug '16

23 Aug '16

ldap/ldif/template-dse.ldif.in | 1 + ldap/servers/slapd/auditlog.c | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) New commits: commit ef2c3c4cc6f966935dbe367dd0d882ae81de3cc4 Author: Noriko Hosoi <nhosoi(a)redhat.com> Date: Mon Aug 22 22:24:51 2016 -0700 Ticket #48969 - nsslapd-auditfaillog always has an explicit path Bug Description: In the current implementation, nsslapd-auditfaillog is not set, by default. Internally, the value is NULL, which let audit fail log share the same audit log file with nsslapd-auditlog. Once, some path is set to nsslapd-auditfaillog, it is not allowed to delete or set NULL or empty to it. That is, there is no way to go back to the default behaviour. There is another issue for the default value. Since search for nsslapd-auditfaillog under cn=config does not return anything, it is hard to find out where the failed logs are written. Fix Description: To solve the 2 issues, this patch changes the default value to an explicit path /path/to/logdir/audit. https://fedorahosted.org/389/ticket/48969 Reviewed by wibrown(a)redhat.com (Thank you, William!) diff --git a/ldap/ldif/template-dse.ldif.in b/ldap/ldif/template-dse.ldif.in index 46b416b..8258b70 100644 --- a/ldap/ldif/template-dse.ldif.in +++ b/ldap/ldif/template-dse.ldif.in @@ -53,6 +53,7 @@ nsslapd-auditlog-maxlogsize: 100 nsslapd-auditlog-logrotationtime: 1 nsslapd-auditlog-logrotationtimeunit: day nsslapd-auditlog-logging-enabled: off +nsslapd-auditfaillog: %log_dir%/audit nsslapd-auditfaillog-logging-enabled: off nsslapd-rootdn: %rootdn% nsslapd-rootpw: %ds_passwd% diff --git a/ldap/servers/slapd/auditlog.c b/ldap/servers/slapd/auditlog.c index ec7111b..85d136c 100644 --- a/ldap/servers/slapd/auditlog.c +++ b/ldap/servers/slapd/auditlog.c @@ -112,6 +112,7 @@ write_auditfail_log_entry( Slapi_PBlock *pb ) Operation *op; int pbrc = 0; char *auditfail_config = NULL; + char *audit_config = NULL; /* if the audit log is not enabled, just skip all of this stuff */ @@ -167,8 +168,9 @@ write_auditfail_log_entry( Slapi_PBlock *pb ) /* log the raw, unnormalized DN */ dn = slapi_sdn_get_udn(sdn); auditfail_config = config_get_auditfaillog(); - if (auditfail_config == NULL || strlen(auditfail_config) == 0) { - /* If no auditfail log write to audit log */ + audit_config = config_get_auditlog(); + if (auditfail_config == NULL || strlen(auditfail_config) == 0 || PL_strcasecmp(auditfail_config, audit_config) == 0) { + /* If no auditfail log or "auditfaillog" == "auditlog", write to audit log */ write_audit_file(SLAPD_AUDIT_LOG, operation_get_type(op), dn, change, flag, curtime, pbrc, SLAPD_AUDITFAIL_LOG); } else { /* If we have our own auditfail log path */

1 0

Branch '389-ds-base-1.2.11' - ldap/servers
by Mark Reynolds 23 Aug '16

23 Aug '16

ldap/servers/plugins/replication/cl5_api.c | 163 ++++++++++------ ldap/servers/plugins/replication/cl5_api.h | 2 ldap/servers/plugins/replication/repl5_replica_config.c | 2 3 files changed, 106 insertions(+), 61 deletions(-) New commits: commit 0d5afb2bbd6d639ec59e6bdf075bd61d3d72ef79 Author: Mark Reynolds <mreynolds(a)redhat.com> Date: Tue Aug 23 12:06:30 2016 -0400 Ticket 48964 - cleanAllRUV changelog purging incorrectly processes all backends Bug Description: When the changelog was being purged of "cleaned" rids it was checking all the backend changelogs, and not the one from which the cleanAllRUV task originated from. This could corrupt a different backend's changelog if both backends used the same RID. Fix Description: Purge the changelog associated with the backend that is specified in the cleanAllRUV task. Also moved the "purging" to its own function, and fixed a few compiler warnings. https://fedorahosted.org/389/ticket/48965 Reviewed by: nhosoi(Thanks!) (cherry picked from commit fda00435a7536c1ded72bb78a975f3370d09a3be) diff --git a/ldap/servers/plugins/replication/cl5_api.c b/ldap/servers/plugins/replication/cl5_api.c index 259c054..d91574f 100644 --- a/ldap/servers/plugins/replication/cl5_api.c +++ b/ldap/servers/plugins/replication/cl5_api.c @@ -344,10 +344,9 @@ static int _cl5CheckMissingCSN (const CSN *minCsn, const RUV *supplierRUV, CL5DB static int _cl5TrimInit (); static void _cl5TrimCleanup (); static int _cl5TrimMain (void *param); -static void _cl5DoTrimming (ReplicaId rid); +static void _cl5DoTrimming (); static PRBool _cl5CanTrim (time_t time, long *numToTrim); static void _cl5TrimFile (Object *obj, long *numToTrim); - static void _cl5PurgeRID(Object *obj, ReplicaId cleaned_rid); static int _cl5PurgeGetFirstEntry (Object *obj, CL5Entry *entry, void **iterator, DB_TXN *txnid, int rid, DBT *key); static int _cl5PurgeGetNextEntry (CL5Entry *entry, void *iterator, DBT *key); @@ -3445,43 +3444,37 @@ static int _cl5TrimMain (void *param) return 0; } -/* We remove an entry if it has been replayed to all consumers and - and the number of entries in the changelog is larger than maxEntries - or age of the entry is larger than maxAge. - Also we can't purge entries which correspond to max csns in the - supplier's ruv. Here is a example where we can get into trouble: - The server is setup with time based trimming and no consumer's - At some point all the entries are trimmed from the changelog. - At a later point a consumer is added and initialized online - Then a change is made on the supplier. - To update the consumer, the supplier would attempt to locate - the last change sent to the consumer in the changelog and will - fail because the change was removed. - +/* + * We remove an entry if it has been replayed to all consumers and the number + * of entries in the changelog is larger than maxEntries or age of the entry + * is larger than maxAge. Also we can't purge entries which correspond to max + * csns in the supplier's ruv. Here is a example where we can get into trouble: + * + * The server is setup with time based trimming and no consumer's + * At some point all the entries are trimmed from the changelog. + * At a later point a consumer is added and initialized online. + * Then a change is made on the supplier. + * To update the consumer, the supplier would attempt to locate the last + * change sent to the consumer in the changelog and will fail because the + * change was removed. */ - -static void _cl5DoTrimming (ReplicaId rid) +static void _cl5DoTrimming () { Object *obj; long numToTrim; PR_Lock (s_cl5Desc.dbTrim.lock); - /* ONREPL We trim file by file which means that some files will be - trimmed more often than other. We might have to fix that by, for - example, randomizing starting point */ + /* + * We are trimming all the changelogs. We trim file by file which + * means that some files will be trimmed more often than other. We + * might have to fix that by, for example, randomizing the starting + * point. + */ obj = objset_first_obj (s_cl5Desc.dbFiles); - while (obj && (_cl5CanTrim ((time_t)0, &numToTrim) || rid)) + while (obj && _cl5CanTrim ((time_t)0, &numToTrim)) { - if (rid){ - /* - * We are cleaning an invalid rid, and need to strip it - * from the changelog. - */ - _cl5PurgeRID (obj, rid); - } else { - _cl5TrimFile (obj, &numToTrim); - } + _cl5TrimFile (obj, &numToTrim); obj = objset_next_obj (s_cl5Desc.dbFiles, obj); } @@ -3494,6 +3487,43 @@ static void _cl5DoTrimming (ReplicaId rid) } /* + * We are purging a changelog after a cleanAllRUV task. Find the specific + * changelog for the backend that is being cleaned, and purge all the records + * with the cleaned rid. + */ +static void _cl5DoPurging (Replica *replica) +{ + ReplicaId rid = replica_get_rid(replica); + const Slapi_DN *sdn = replica_get_root(replica); + const char *replName = replica_get_name(replica); + char *replGen = replica_get_generation(replica); + char *fileName; + Object *obj; + + PR_Lock (s_cl5Desc.dbTrim.lock); + fileName = _cl5MakeFileName (replName, replGen); + obj = objset_find(s_cl5Desc.dbFiles, _cl5CompareDBFile, fileName); + if (obj) { + /* We found our changelog, now purge it */ + _cl5PurgeRID (obj, rid); + object_release (obj); + slapi_log_error(SLAPI_LOG_REPL, repl_plugin_name_cl, + "Purged rid (%d) from suffix (%s)\n", + rid, slapi_sdn_get_dn(sdn)); + } else { + slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name_cl, + "Purge rid (%d) failed to find changelog file (%s) for suffix (%s)\n", + rid, fileName, slapi_sdn_get_dn(sdn)); + } + PR_Unlock (s_cl5Desc.dbTrim.lock); + + slapi_ch_free_string(&replGen); + slapi_ch_free_string(&fileName); + + return; +} + +/* * If the rid is not set it is the very first iteration of the changelog. * If the rid is set, we are doing another pass, and we have a key as our * starting point. @@ -4005,23 +4035,25 @@ static PRBool _cl5CanTrim (time_t time, long *numToTrim) { *numToTrim = 0; - if (s_cl5Desc.dbTrim.maxAge == 0 && s_cl5Desc.dbTrim.maxEntries == 0) + if (s_cl5Desc.dbTrim.maxAge == 0 && s_cl5Desc.dbTrim.maxEntries == 0) { return PR_FALSE; - + } if (s_cl5Desc.dbTrim.maxAge == 0) { *numToTrim = cl5GetOperationCount (NULL) - s_cl5Desc.dbTrim.maxEntries; return ( *numToTrim > 0 ); } - if (s_cl5Desc.dbTrim.maxEntries > 0 && - (*numToTrim = cl5GetOperationCount (NULL) - s_cl5Desc.dbTrim.maxEntries) > 0) - return PR_TRUE; + if (s_cl5Desc.dbTrim.maxEntries > 0 && + (*numToTrim = cl5GetOperationCount (NULL) - s_cl5Desc.dbTrim.maxEntries) > 0) { + return PR_TRUE; + } - if (time) + if (time) { return (current_time () - time > s_cl5Desc.dbTrim.maxAge); - else - return PR_TRUE; + } else { + return PR_TRUE; + } } static int _cl5ReadRUV (const char *replGen, Object *obj, PRBool purge) @@ -4034,7 +4066,6 @@ static int _cl5ReadRUV (const char *replGen, Object *obj, PRBool purge) char *pos; char *agmt_name; - PR_ASSERT (replGen && obj); file = (CL5DBFile*)object_get_data (obj); @@ -4042,13 +4073,12 @@ static int _cl5ReadRUV (const char *replGen, Object *obj, PRBool purge) agmt_name = get_thread_private_agmtname(); - if (purge) /* read purge vector entry */ - key.data = _cl5GetHelperEntryKey (PURGE_RUV_TIME, csnStr); - else /* read upper bound vector */ - key.data = _cl5GetHelperEntryKey (MAX_RUV_TIME, csnStr); - + if (purge) { /* read purge vector entry */ + key.data = _cl5GetHelperEntryKey (PURGE_RUV_TIME, csnStr); + } else { /* read upper bound vector */ + key.data = _cl5GetHelperEntryKey (MAX_RUV_TIME, csnStr); + } key.size = CSN_STRSIZE; - data.flags = DB_DBT_MALLOC; rc = file->db->get(file->db, NULL/*txn*/, &key, &data, 0); @@ -4058,13 +4088,13 @@ static int _cl5ReadRUV (const char *replGen, Object *obj, PRBool purge) rc = _cl5ReadBervals (&vals, &pos, data.size); slapi_ch_free (&(data.data)); if (rc != CL5_SUCCESS) - goto done; + goto done; - if (purge) + if (purge) { rc = ruv_init_from_bervals(vals, &file->purgeRUV); - else + } else { rc = ruv_init_from_bervals(vals, &file->maxRUV); - + } if (rc != RUV_SUCCESS) { slapi_log_error(SLAPI_LOG_REPL, repl_plugin_name_cl, @@ -4072,7 +4102,7 @@ static int _cl5ReadRUV (const char *replGen, Object *obj, PRBool purge) "RUV error %d\n", agmt_name, purge? "purge" : "upper bound", rc); rc = CL5_RUV_ERROR; - goto done; + goto done; } /* delete the entry; it is re-added when file @@ -4084,7 +4114,7 @@ static int _cl5ReadRUV (const char *replGen, Object *obj, PRBool purge) case DB_NOTFOUND: /* RUV is lost - need to construct */ rc = _cl5ConstructRUV (replGen, obj, purge); - goto done; + goto done; default: slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name_cl, "%s: _cl5ReadRUV: failed to get purge RUV; " @@ -6858,12 +6888,14 @@ cl5CleanRUV(ReplicaId rid){ slapi_rwlock_unlock (s_cl5Desc.stLock); } -void trigger_cl_purging(ReplicaId rid){ +/* + * Create a thread to purge a changelog of cleaned RIDs + */ +void trigger_cl_purging(Replica *replica){ PRThread *trim_tid = NULL; - slapi_log_error(SLAPI_LOG_REPL, repl_plugin_name_cl, "trigger_cl_purging: rid (%d)\n",(int)rid); trim_tid = PR_CreateThread(PR_USER_THREAD, (VFP)(void*)trigger_cl_purging_thread, - (void *)&rid, PR_PRIORITY_NORMAL, PR_GLOBAL_THREAD, + (void *)replica, PR_PRIORITY_NORMAL, PR_GLOBAL_THREAD, PR_UNJOINABLE_THREAD, DEFAULT_THREAD_STACKSIZE); if (NULL == trim_tid){ slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name_cl, @@ -6875,19 +6907,32 @@ void trigger_cl_purging(ReplicaId rid){ } } +/* + * Purge a changelog of entries that originated from a particular replica(rid) + */ void trigger_cl_purging_thread(void *arg){ - ReplicaId rid = *(ReplicaId *)arg; + Replica *replica = (Replica *)arg; - /* make sure we have a change log, and we aren't closing it */ - if(s_cl5Desc.dbState == CL5_STATE_CLOSED || s_cl5Desc.dbState == CL5_STATE_CLOSING){ + /* Make sure we have a change log, and we aren't closing it */ + if (replica == NULL || + s_cl5Desc.dbState == CL5_STATE_CLOSED || + s_cl5Desc.dbState == CL5_STATE_CLOSING) { return; } + + /* Bump the changelog thread count */ if (CL5_SUCCESS != _cl5AddThread()) { slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name_cl, - "trigger_cl_purging: failed to increment thread count " + "trigger_cl_purging: Abort - failed to increment thread count " "NSPR error - %d\n", PR_GetError ()); + return; } - _cl5DoTrimming(rid); + + /* Purge the changelog */ + _cl5DoPurging(replica); _cl5RemoveThread(); + slapi_log_error(SLAPI_LOG_REPL, repl_plugin_name_cl, + "trigger_cl_purging: purged changelog for (%s) rid (%d)\n", + slapi_sdn_get_dn(replica_get_root(replica)), replica_get_rid(replica)); } diff --git a/ldap/servers/plugins/replication/cl5_api.h b/ldap/servers/plugins/replication/cl5_api.h index b46a691..83683f5 100644 --- a/ldap/servers/plugins/replication/cl5_api.h +++ b/ldap/servers/plugins/replication/cl5_api.h @@ -493,6 +493,6 @@ int cl5WriteRUV(); int cl5DeleteRUV(); void cl5CleanRUV(ReplicaId rid); void cl5NotifyCleanup(int rid); -void trigger_cl_purging(ReplicaId rid); +void trigger_cl_purging(Replica *replica); #endif diff --git a/ldap/servers/plugins/replication/repl5_replica_config.c b/ldap/servers/plugins/replication/repl5_replica_config.c index a1dceae..010e517 100644 --- a/ldap/servers/plugins/replication/repl5_replica_config.c +++ b/ldap/servers/plugins/replication/repl5_replica_config.c @@ -1244,7 +1244,7 @@ replica_execute_cleanruv_task (Object *r, ReplicaId rid, char *returntext /* not /* * Now purge the changelog */ - trigger_cl_purging(rid); + trigger_cl_purging(replica); if (rc != RUV_SUCCESS){ slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name, "cleanruv_task: task failed(%d)\n",rc);

1 0

Branch '389-ds-base-1.3.3' - ldap/servers
by Mark Reynolds 23 Aug '16

23 Aug '16

ldap/servers/plugins/replication/cl5_api.c | 162 ++++++++++------ ldap/servers/plugins/replication/cl5_api.h | 2 ldap/servers/plugins/replication/repl5_replica_config.c | 2 3 files changed, 106 insertions(+), 60 deletions(-) New commits: commit f4df06b4d50edbf4a8e3c0a8a8ecc08a174ce870 Author: Mark Reynolds <mreynolds(a)redhat.com> Date: Tue Aug 23 12:06:30 2016 -0400 Ticket 48964 - cleanAllRUV changelog purging incorrectly processes all backends Bug Description: When the changelog was being purged of "cleaned" rids it was checking all the backend changelogs, and not the one from which the cleanAllRUV task originated from. This could corrupt a different backend's changelog if both backends used the same RID. Fix Description: Purge the changelog associated with the backend that is specified in the cleanAllRUV task. Also moved the "purging" to its own function, and fixed a few compiler warnings. https://fedorahosted.org/389/ticket/48965 Reviewed by: nhosoi(Thanks!) (cherry picked from commit fda00435a7536c1ded72bb78a975f3370d09a3be) diff --git a/ldap/servers/plugins/replication/cl5_api.c b/ldap/servers/plugins/replication/cl5_api.c index 2cf5a3f..782d05f 100644 --- a/ldap/servers/plugins/replication/cl5_api.c +++ b/ldap/servers/plugins/replication/cl5_api.c @@ -351,7 +351,7 @@ static int _cl5CheckMissingCSN (const CSN *minCsn, const RUV *supplierRUV, CL5DB static int _cl5TrimInit (); static void _cl5TrimCleanup (); static int _cl5TrimMain (void *param); -static void _cl5DoTrimming (ReplicaId rid); +static void _cl5DoTrimming (); static void _cl5CompactDBs(); static void _cl5PurgeRID(Object *obj, ReplicaId cleaned_rid); static int _cl5PurgeGetFirstEntry (Object *obj, CL5Entry *entry, void **iterator, DB_TXN *txnid, int rid, DBT *key); @@ -3481,43 +3481,37 @@ static int _cl5TrimMain (void *param) return 0; } -/* We remove an entry if it has been replayed to all consumers and - and the number of entries in the changelog is larger than maxEntries - or age of the entry is larger than maxAge. - Also we can't purge entries which correspond to max csns in the - supplier's ruv. Here is a example where we can get into trouble: - The server is setup with time based trimming and no consumer's - At some point all the entries are trimmed from the changelog. - At a later point a consumer is added and initialized online - Then a change is made on the supplier. - To update the consumer, the supplier would attempt to locate - the last change sent to the consumer in the changelog and will - fail because the change was removed. - +/* + * We remove an entry if it has been replayed to all consumers and the number + * of entries in the changelog is larger than maxEntries or age of the entry + * is larger than maxAge. Also we can't purge entries which correspond to max + * csns in the supplier's ruv. Here is a example where we can get into trouble: + * + * The server is setup with time based trimming and no consumer's + * At some point all the entries are trimmed from the changelog. + * At a later point a consumer is added and initialized online. + * Then a change is made on the supplier. + * To update the consumer, the supplier would attempt to locate the last + * change sent to the consumer in the changelog and will fail because the + * change was removed. */ - -static void _cl5DoTrimming (ReplicaId rid) +static void _cl5DoTrimming () { Object *obj; long numToTrim; PR_Lock (s_cl5Desc.dbTrim.lock); - /* ONREPL We trim file by file which means that some files will be - trimmed more often than other. We might have to fix that by, for - example, randomizing starting point */ + /* + * We are trimming all the changelogs. We trim file by file which + * means that some files will be trimmed more often than other. We + * might have to fix that by, for example, randomizing the starting + * point. + */ obj = objset_first_obj (s_cl5Desc.dbFiles); - while (obj && (_cl5CanTrim ((time_t)0, &numToTrim) || rid)) + while (obj && _cl5CanTrim ((time_t)0, &numToTrim)) { - if (rid){ - /* - * We are cleaning an invalid rid, and need to strip it - * from the changelog. - */ - _cl5PurgeRID (obj, rid); - } else { - _cl5TrimFile (obj, &numToTrim); - } + _cl5TrimFile (obj, &numToTrim); obj = objset_next_obj (s_cl5Desc.dbFiles, obj); } @@ -3529,6 +3523,43 @@ static void _cl5DoTrimming (ReplicaId rid) return; } +/* + * We are purging a changelog after a cleanAllRUV task. Find the specific + * changelog for the backend that is being cleaned, and purge all the records + * with the cleaned rid. + */ +static void _cl5DoPurging (Replica *replica) +{ + ReplicaId rid = replica_get_rid(replica); + const Slapi_DN *sdn = replica_get_root(replica); + const char *replName = replica_get_name(replica); + char *replGen = replica_get_generation(replica); + char *fileName; + Object *obj; + + PR_Lock (s_cl5Desc.dbTrim.lock); + fileName = _cl5MakeFileName (replName, replGen); + obj = objset_find(s_cl5Desc.dbFiles, _cl5CompareDBFile, fileName); + if (obj) { + /* We found our changelog, now purge it */ + _cl5PurgeRID (obj, rid); + object_release (obj); + slapi_log_error(SLAPI_LOG_REPL, repl_plugin_name_cl, + "Purged rid (%d) from suffix (%s)\n", + rid, slapi_sdn_get_dn(sdn)); + } else { + slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name_cl, + "Purge rid (%d) failed to find changelog file (%s) for suffix (%s)\n", + rid, fileName, slapi_sdn_get_dn(sdn)); + } + PR_Unlock (s_cl5Desc.dbTrim.lock); + + slapi_ch_free_string(&replGen); + slapi_ch_free_string(&fileName); + + return; +} + /* clear free page files to reduce changelog */ static void _cl5CompactDBs() @@ -4106,23 +4137,25 @@ static PRBool _cl5CanTrim (time_t time, long *numToTrim) { *numToTrim = 0; - if (s_cl5Desc.dbTrim.maxAge == 0 && s_cl5Desc.dbTrim.maxEntries == 0) + if (s_cl5Desc.dbTrim.maxAge == 0 && s_cl5Desc.dbTrim.maxEntries == 0) { return PR_FALSE; - + } if (s_cl5Desc.dbTrim.maxAge == 0) { *numToTrim = cl5GetOperationCount (NULL) - s_cl5Desc.dbTrim.maxEntries; return ( *numToTrim > 0 ); } - if (s_cl5Desc.dbTrim.maxEntries > 0 && - (*numToTrim = cl5GetOperationCount (NULL) - s_cl5Desc.dbTrim.maxEntries) > 0) - return PR_TRUE; + if (s_cl5Desc.dbTrim.maxEntries > 0 && + (*numToTrim = cl5GetOperationCount (NULL) - s_cl5Desc.dbTrim.maxEntries) > 0) { + return PR_TRUE; + } - if (time) + if (time) { return (current_time () - time > s_cl5Desc.dbTrim.maxAge); - else - return PR_TRUE; + } else { + return PR_TRUE; + } } static int _cl5ReadRUV (const char *replGen, Object *obj, PRBool purge) @@ -4135,7 +4168,6 @@ static int _cl5ReadRUV (const char *replGen, Object *obj, PRBool purge) char *pos; char *agmt_name; - PR_ASSERT (replGen && obj); file = (CL5DBFile*)object_get_data (obj); @@ -4143,13 +4175,12 @@ static int _cl5ReadRUV (const char *replGen, Object *obj, PRBool purge) agmt_name = get_thread_private_agmtname(); - if (purge) /* read purge vector entry */ - key.data = _cl5GetHelperEntryKey (PURGE_RUV_TIME, csnStr); - else /* read upper bound vector */ - key.data = _cl5GetHelperEntryKey (MAX_RUV_TIME, csnStr); - + if (purge) { /* read purge vector entry */ + key.data = _cl5GetHelperEntryKey (PURGE_RUV_TIME, csnStr); + } else { /* read upper bound vector */ + key.data = _cl5GetHelperEntryKey (MAX_RUV_TIME, csnStr); + } key.size = CSN_STRSIZE; - data.flags = DB_DBT_MALLOC; rc = file->db->get(file->db, NULL/*txn*/, &key, &data, 0); @@ -4159,13 +4190,13 @@ static int _cl5ReadRUV (const char *replGen, Object *obj, PRBool purge) rc = _cl5ReadBervals (&vals, &pos, data.size); slapi_ch_free (&(data.data)); if (rc != CL5_SUCCESS) - goto done; + goto done; - if (purge) + if (purge) { rc = ruv_init_from_bervals(vals, &file->purgeRUV); - else + } else { rc = ruv_init_from_bervals(vals, &file->maxRUV); - + } if (rc != RUV_SUCCESS) { slapi_log_error(SLAPI_LOG_REPL, repl_plugin_name_cl, @@ -4173,7 +4204,7 @@ static int _cl5ReadRUV (const char *replGen, Object *obj, PRBool purge) "RUV error %d\n", agmt_name, purge? "purge" : "upper bound", rc); rc = CL5_RUV_ERROR; - goto done; + goto done; } /* delete the entry; it is re-added when file @@ -4185,7 +4216,7 @@ static int _cl5ReadRUV (const char *replGen, Object *obj, PRBool purge) case DB_NOTFOUND: /* RUV is lost - need to construct */ rc = _cl5ConstructRUV (replGen, obj, purge); - goto done; + goto done; default: slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name_cl, "%s: _cl5ReadRUV: failed to get purge RUV; " @@ -7083,12 +7114,14 @@ cl5CleanRUV(ReplicaId rid){ slapi_rwlock_unlock (s_cl5Desc.stLock); } -void trigger_cl_purging(ReplicaId rid){ +/* + * Create a thread to purge a changelog of cleaned RIDs + */ +void trigger_cl_purging(Replica *replica){ PRThread *trim_tid = NULL; - slapi_log_error(SLAPI_LOG_REPL, repl_plugin_name_cl, "trigger_cl_purging: rid (%d)\n",(int)rid); trim_tid = PR_CreateThread(PR_USER_THREAD, (VFP)(void*)trigger_cl_purging_thread, - (void *)&rid, PR_PRIORITY_NORMAL, PR_GLOBAL_THREAD, + (void *)replica, PR_PRIORITY_NORMAL, PR_GLOBAL_THREAD, PR_UNJOINABLE_THREAD, DEFAULT_THREAD_STACKSIZE); if (NULL == trim_tid){ slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name_cl, @@ -7100,19 +7133,32 @@ void trigger_cl_purging(ReplicaId rid){ } } +/* + * Purge a changelog of entries that originated from a particular replica(rid) + */ void trigger_cl_purging_thread(void *arg){ - ReplicaId rid = *(ReplicaId *)arg; + Replica *replica = (Replica *)arg; - /* make sure we have a change log, and we aren't closing it */ - if(s_cl5Desc.dbState == CL5_STATE_CLOSED || s_cl5Desc.dbState == CL5_STATE_CLOSING){ + /* Make sure we have a change log, and we aren't closing it */ + if (replica == NULL || + s_cl5Desc.dbState == CL5_STATE_CLOSED || + s_cl5Desc.dbState == CL5_STATE_CLOSING) { return; } + + /* Bump the changelog thread count */ if (CL5_SUCCESS != _cl5AddThread()) { slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name_cl, - "trigger_cl_purging: failed to increment thread count " + "trigger_cl_purging: Abort - failed to increment thread count " "NSPR error - %d\n", PR_GetError ()); + return; } - _cl5DoTrimming(rid); + + /* Purge the changelog */ + _cl5DoPurging(replica); _cl5RemoveThread(); + slapi_log_error(SLAPI_LOG_REPL, repl_plugin_name_cl, + "trigger_cl_purging: purged changelog for (%s) rid (%d)\n", + slapi_sdn_get_dn(replica_get_root(replica)), replica_get_rid(replica)); } diff --git a/ldap/servers/plugins/replication/cl5_api.h b/ldap/servers/plugins/replication/cl5_api.h index 0abd46d..d69a23e 100644 --- a/ldap/servers/plugins/replication/cl5_api.h +++ b/ldap/servers/plugins/replication/cl5_api.h @@ -496,6 +496,6 @@ int cl5WriteRUV(); int cl5DeleteRUV(); void cl5CleanRUV(ReplicaId rid); void cl5NotifyCleanup(int rid); -void trigger_cl_purging(ReplicaId rid); +void trigger_cl_purging(Replica *replica); #endif diff --git a/ldap/servers/plugins/replication/repl5_replica_config.c b/ldap/servers/plugins/replication/repl5_replica_config.c index a013ebf..3c4f68c 100644 --- a/ldap/servers/plugins/replication/repl5_replica_config.c +++ b/ldap/servers/plugins/replication/repl5_replica_config.c @@ -1497,7 +1497,7 @@ replica_execute_cleanruv_task (Object *r, ReplicaId rid, char *returntext /* not /* * Now purge the changelog */ - trigger_cl_purging(rid); + trigger_cl_purging(replica); if (rc != RUV_SUCCESS){ slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name, "cleanruv_task: task failed(%d)\n",rc);

1 0

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

2009

2008

2007

2006

2005

389-commits August 2016