389-commits April 2013

389-commits@lists.fedoraproject.org

6 participants
77 discussions

Branch '389-ds-base-1.2.11' - VERSION.sh
by Mark Reynolds 09 Apr '13

09 Apr '13

VERSION.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) New commits: commit 9a7ba7d6cb9d84a2638bf12ae7d912f997790ff2 Author: Mark Reynolds <mreynolds(a)redhat.com> Date: Tue Apr 9 14:54:19 2013 -0400 bump verison to 1.2.11.21 diff --git a/VERSION.sh b/VERSION.sh index f2bac77..8989684 100644 --- a/VERSION.sh +++ b/VERSION.sh @@ -10,7 +10,7 @@ vendor="389 Project" # PACKAGE_VERSION is constructed from these VERSION_MAJOR=1 VERSION_MINOR=2 -VERSION_MAINT=11.20 +VERSION_MAINT=11.21 # if this is a PRERELEASE, set VERSION_PREREL # otherwise, comment it out # be sure to include the dot prefix in the prerel

1 0

Branch '389-ds-base-1.3.1' - ldap/admin ldap/servers
by Noriko Hosoi 09 Apr '13

09 Apr '13

ldap/admin/src/scripts/80upgradednformat.pl | 206 ------- ldap/admin/src/scripts/80upgradednformat.pl.in | 307 ++++++++++ ldap/servers/slapd/back-ldbm/back-ldbm.h | 1 ldap/servers/slapd/back-ldbm/dbversion.c | 3 ldap/servers/slapd/back-ldbm/import-threads.c | 714 ++++++++++++++++++------- ldap/servers/slapd/back-ldbm/import.c | 103 ++- ldap/servers/slapd/back-ldbm/import.h | 23 ldap/servers/slapd/back-ldbm/ldif2ldbm.c | 43 + ldap/servers/slapd/slapi-private.h | 1 9 files changed, 974 insertions(+), 427 deletions(-) New commits: commit 36696aac383ea9de73ff15f255677dfe1f975acf Author: Noriko Hosoi <nhosoi(a)redhat.com> Date: Tue Apr 9 10:27:39 2013 -0700 Ticket #529 - dn normalization must handle multiple space characters in attributes Bug description: This is the second half of the fix for #529. The first half fixed the DN normalization which used to allow DNs where only the number of spaces are different. Now it is rejucted as expected. But it breaks the backward compatibility. Fix description: The upgrade script 80upgradednformat.pl called from "setup-ds.pl -u" checks the duplicated DNs and rename them if necessary. For instance, if there are 2 DNs: cn=test user0,dc=example,dc=com (entryid: N) cn=test user0,dc=example,dc=com (entryid: M) then the upgrade script/tool modifies the second one as follows: cn=test user0 M,dc=example,dc=com (entryid: M) and the original "cn: test user0" is kept in the attribute. The modified result is reported in "setup-ds.pl -u" as follows: !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Duplicated DN(s) were found and renamed. !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Renamed entry IDs are listed in /var/lib/dirsrv/slapd-ID/ldif/userRoot_conflict.txt. Contents of the conflict.txt: prinary entry ID: duplicated entry IDs 13:16 18 14:17 https://fedorahosted.org/389/ticket/529 Reviewed and commentted by Rich and Mark (Thanks!!) (cherry picked from commit 69ff83598d517bed84922b1c7dd67cab023b4d99) diff --git a/ldap/admin/src/scripts/80upgradednformat.pl b/ldap/admin/src/scripts/80upgradednformat.pl deleted file mode 100644 index 092b7cd..0000000 --- a/ldap/admin/src/scripts/80upgradednformat.pl +++ /dev/null @@ -1,206 +0,0 @@ -use Mozilla::LDAP::Conn; -use Mozilla::LDAP::Utils qw(normalizeDN); -use Mozilla::LDAP::API qw(:constant ldap_url_parse ldap_explode_dn); -use File::Basename; -use File::Copy; -use DSUpdate qw(isOffline); - -# Upgrade DN format if needed. -# For each backend instance, -# run upgradednformat with -N (dryrun mode), -# if it returns 0 (Upgrade candidates are found), -# recursively copy the instance dir to the work dir (dnupgrade) -# run upgradednformat w/o -N against the DB in the work dir -# if it went ok, replace the original instance dir with the work dir. -# Note: This script does nothing if the server is up. -sub runinst { - my ($inf, $inst, $dseldif, $conn) = @_; - - my @errs; - - my $config = "cn=config"; - my $config_entry = $conn->search($config, "base", "(cn=*)"); - if (!$config_entry) { - return ("error_no_configuration_entry", $!); - } - - # Check if the server is up or not - my $rc; - ($rc, @errs) = isOffline($inf, $inst, $conn); - if (!$rc) { - return @errs; - } - my $mappingtree = "cn=mapping tree,cn=config"; - my $ldbmbase = "cn=ldbm database,cn=plugins,cn=config"; - - my $backend_entry; - my $mtentry = $conn->search($mappingtree, "onelevel", "(cn=*)", 0, @attr); - if (!$mtentry) { - return ("error_no_mapping_tree_entries", $!); - } - - my $db_config_entry = - $conn->search("cn=config,cn=ldbm database,cn=plugins,cn=config", - "base", "(objectclass=*)"); - if (!$db_config_entry) { - return ('error_finding_config_entry', - 'cn=config,cn=ldbm database,cn=plugins,cn=config', - $conn->getErrorString()); - } - # If subtree rename swich is not found in the config file, - # set off to the switch and upgrade dn format using entrydn. - my $switch = $db_config_entry->getValues('nsslapd-subtree-rename-switch'); - if ("" eq $switch) { - $db_config_entry->addValue('nsslapd-subtree-rename-switch', "off"); - $conn->update($db_config_entry); - } - - # If a suffix in the mapping tree is doube-quoted and - # the cn value has only the double-quoted value, e.g. - # dn: cn="dc=example,dc=com",cn=mapping tree,cn=config - # cn: "dc=example,dc=com" - # the following code adds non-quoted value: - # cn: dc=example,dc=com - while ($mtentry) { - my $numvals = $mtentry->size("cn"); - my $i; - my $withquotes = -1; - my $noquotes = -1; - for ($i = 0; $i < $numvals; $i++) { - if ($mtentry->{"cn"}[$i] =~ /^".*"$/) { - $withquotes = $i; - } else { - $noquotes = $i; - } - } - if ($withquotes >= 0 && $noquotes == -1) { - # Has only cn: "<suffix>" - # Adding cn: <suffix> - my $stripped = $mtentry->{"cn"}[$withquotes]; - $stripped =~ s/^"(.*)"$/$1/; - $mtentry->addValue("cn", $stripped); - $conn->update($mtentry); - } - $mtentry = $conn->nextEntry(); - } - - my $instancedir = $config_entry->{"nsslapd-instancedir"}[0]; - my $upgradednformat = $instancedir . "/upgradednformat"; - - # Scan through all of the backends to see if any of them - # contain escape characters in the DNs. If we find any - # escapes, we need to run the conversion tool on that - # backend. - $backend_entry = $conn->search($ldbmbase, "onelevel", "(objectClass=nsBackendInstance)", 0, @attr); - if (!$backend_entry) { - return ("error_no_backend_entries", $!); - } - - while ($backend_entry) { - my $backend = $backend_entry->{"cn"}[0]; - my $dbinstdir = $backend_entry->{"nsslapd-directory"}[0]; - my $workdir = $dbinstdir . "/dnupgrade"; - my $dbdir = dirname($dbinstdir); - my $pdbdir = dirname($dbdir); - my $instname = basename($dbinstdir); - - if ("$dbdir" eq "" || "$instname" eq "") { - push @errs, ["error_invalid_dbinst_dir", $dbinstdir]; - return @errs; - } - - # clean up db region files, which might contain the old pages - if ( -d $dbdir && -f $dbdir."/__db.001") { - unlink <$dbdir/__db.*>; - } - - if (-e "$dbinstdir/id2entry.db4") { - # Check if any DNs contain escape characters with dbscan. - # dryrun mode - # return values: 0 -- need to upgrade dn format - # 1 -- no need to upgrade dn format - # -1 -- error - my $escapes = system("$upgradednformat -n $backend -a $dbinstdir -N"); - if (0 == $escapes) { - # need to upgrade dn format - my $rc = 0; - - if (system("cd $pdbdir; tar cf - db/DBVERSION | (cd $dbinstdir; tar xf -)") || - system("cd $pdbdir; tar cf - db/$instname/DBVERSION | (cd $dbinstdir; tar xf -)") || - system("cd $pdbdir; tar cf - db/$instname/*.db4 | (cd $dbinstdir; tar xf -)")) { - push @errs, ["error_cant_backup_db", $backend, $!]; - return @errs; - } - my @stat = stat("$dbdir"); - my $mode = $stat[2]; - my $uid = $stat[4]; - my $gid = $stat[5]; - - move("$dbinstdir/db", "$workdir"); - chmod($mode, $workdir); - chown($uid, $gid, $workdir); - - @stat = stat("$dbinstdir"); - $mode = $stat[2]; - $uid = $stat[4]; - $gid = $stat[5]; - - chmod($mode, "$workdir/$instname"); - chown($uid, $gid, "$workdir/$instname"); - - # call conversion tool here and get return status. - $rc = system("$upgradednformat -n $backend -a $workdir/$instname"); - if ($rc == 0) { # success - move("$dbinstdir", "$dbinstdir.orig"); - move("$dbinstdir.orig/dnupgrade/$instname", "$dbinstdir"); - copy("$dbinstdir.orig/dnupgrade/DBVERSION", "$dbdir"); - } else { - # Conversion failed. Cleanup and bail. - unlink <$dbinstdir/dnupgrade/$backend/*>; - rmdir("$dbinstdir/dnupgrade/$backend"); - unlink <$dbinstdir/dnupgrade/*>; - rmdir("$dbinstdir/dnupgrade"); - return ("error_cant_convert_db", $backend, $rc); - } - } else { - # already upgraded or an error occurred. - # check ancestorid to see if it has not-sorted ID list or not. - my $ancestorid = $dbinstdir . "/ancestorid.db4"; - if (-e "$ancestorid") { - my $disorder = 0; - open(ANCESTOR, "/usr/bin/dbscan -f $ancestorid -r |"); - while (<ANCESTOR>) { - if (!/^=[0-9]*/) { - chomp($_); - my @IDs = split(/ | /, $_); - # print "ID count: $#IDs\n"; - my $lasti = $#IDs; - for (my $i = 1; $i < $lasti; $i++) { - if ($IDs[$i] >= $IDs[$i + 1]) { - $disorder = 1; - last; - } - } - # print "Result: $disorder \n"; - if ($disorder) { - last; - } - } - } - close(ANCESTOR); - - # ancestorid index is in disorder; need to reindex it. - if ($disorder) { - print "The ancestorid index in $backend is in disorder; Reindexing $ancestorid.\n"; - my $reindex = $instancedir . "/db2index"; - my $rc = system("$reindex -n $backend -t ancestorid"); - } - } - } - } - - $backend_entry = $conn->nextEntry(); - } - - return (); -} diff --git a/ldap/admin/src/scripts/80upgradednformat.pl.in b/ldap/admin/src/scripts/80upgradednformat.pl.in new file mode 100644 index 0000000..6d14099 --- /dev/null +++ b/ldap/admin/src/scripts/80upgradednformat.pl.in @@ -0,0 +1,307 @@ +use Mozilla::LDAP::Conn; +use Mozilla::LDAP::Utils qw(normalizeDN); +use Mozilla::LDAP::API qw(:constant ldap_url_parse ldap_explode_dn); +use File::Basename; +use File::Copy; +use DSUpdate qw(isOffline); + +# Upgrade DN format if needed. +# For each backend instance, +# run upgradednformat with -N (dryrun mode), +# if it returns 0 (Upgrade candidates are found), +# recursively copy the instance dir to the work dir (dnupgrade) +# run upgradednformat w/o -N against the DB in the work dir +# if it went ok, replace the original instance dir with the work dir. +# Note: This script does nothing if the server is up. +sub runinst { + my ($inf, $inst, $dseldif, $conn) = @_; + + my @errs; + + my $config = "cn=config"; + my $config_entry = $conn->search($config, "base", "(cn=*)"); + if (!$config_entry) { + return ("error_no_configuration_entry", $!); + } + + # Check if the server is up or not + my $rc; + ($rc, @errs) = isOffline($inf, $inst, $conn); + if (!$rc) { + return @errs; + } + my $mappingtree = "cn=mapping tree,cn=config"; + my $ldbmbase = "cn=ldbm database,cn=plugins,cn=config"; + + my $backend_entry; + my $mtentry = $conn->search($mappingtree, "onelevel", "(cn=*)", 0, @attr); + if (!$mtentry) { + return ("error_no_mapping_tree_entries", $!); + } + + my $db_config_entry = + $conn->search("cn=config,cn=ldbm database,cn=plugins,cn=config", + "base", "(objectclass=*)"); + if (!$db_config_entry) { + return ('error_finding_config_entry', + 'cn=config,cn=ldbm database,cn=plugins,cn=config', + $conn->getErrorString()); + } + # If subtree rename swich is not found in the config file, + # set off to the switch and upgrade dn format using entrydn. + my $switch = $db_config_entry->getValues('nsslapd-subtree-rename-switch'); + if ("" eq $switch) { + $db_config_entry->addValue('nsslapd-subtree-rename-switch', "off"); + $conn->update($db_config_entry); + } + + # If a suffix in the mapping tree is doube-quoted and + # the cn value has only the double-quoted value, e.g. + # dn: cn="dc=example,dc=com",cn=mapping tree,cn=config + # cn: "dc=example,dc=com" + # the following code adds non-quoted value: + # cn: dc=example,dc=com + while ($mtentry) { + my $numvals = $mtentry->size("cn"); + my $i; + my $withquotes = -1; + my $noquotes = -1; + for ($i = 0; $i < $numvals; $i++) { + if ($mtentry->{"cn"}[$i] =~ /^".*"$/) { + $withquotes = $i; + } else { + $noquotes = $i; + } + } + if ($withquotes >= 0 && $noquotes == -1) { + # Has only cn: "<suffix>" + # Adding cn: <suffix> + my $stripped = $mtentry->{"cn"}[$withquotes]; + $stripped =~ s/^"(.*)"$/$1/; + $mtentry->addValue("cn", $stripped); + $conn->update($mtentry); + } + $mtentry = $conn->nextEntry(); + } + + my $ldifdir = $config_entry->{"nsslapd-ldifdir"}[0]; + my $instancedir = $config_entry->{"nsslapd-instancedir"}[0]; + my ($slapd, $serverID) = split(/-/, $instancedir); + my $upgradednformat = "@sbindir@/upgradednformat"; + my $reindex = "@sbindir@/db2index"; + + # Scan through all of the backends to see if any of them + # contain escape characters in the DNs. If we find any + # escapes, we need to run the conversion tool on that + # backend. + $backend_entry = $conn->search($ldbmbase, "onelevel", "(objectClass=nsBackendInstance)", 0, @attr); + if (!$backend_entry) { + return ("error_no_backend_entries", $!); + } + + while ($backend_entry) { + my $backend = $backend_entry->{"cn"}[0]; + my $dbinstdir = $backend_entry->{"nsslapd-directory"}[0]; + my $workdir = $dbinstdir . "/dnupgrade"; + my $dbdir = dirname($dbinstdir); + my $pdbdir = dirname($dbdir); + my $instname = basename($dbinstdir); + my $dn_norm_sp_txt = $ldifdir . "/" . $instname . "_dn_norm_sp.txt"; + + if ("$dbdir" eq "" || "$instname" eq "") { + push @errs, ["error_invalid_dbinst_dir", $dbinstdir]; + return @errs; + } + + # clean up db region files, which might contain the old pages + if ( -d $dbdir && -f $dbdir."/__db.001") { + unlink <$dbdir/__db.*>; + } + + my $escapes = 0; + my $rc = 0; + my $cmd = 0; + if ((-e "$dbinstdir/id2entry.db") || (-e "$dbinstdir/id2entry.db4")) { + # Check if any DNs contain escape characters with dbscan. + # dryrun mode + # return values: 0 -- need to upgrade dn format + # 1 -- no need to upgrade dn format + # -1 -- error + $cmd = "$upgradednformat -n $backend -a $dbinstdir -N"; + $rc = system("$cmd"); + if ($rc & 127) { + push @errs, [ 'error_running_command', $cmd, $rc, $! ]; + return @errs; + } + $escapes = $rc >> 8; +# $escapes == 0 ==> no need to do dn upgrade +# $escapes == 1 ==> need to do dn upgrade (both dn upgrade and spaces) +# $escapes == 2 ==> need to do dn upgrade (dn upgrade only) +# $escapes == 3 ==> need to do dn upgrade (dn upgrade spaces only) + if ($escapes <= 0) { + # already upgraded or an error occurred. + # check ancestorid to see if it has not-sorted ID list or not. + my $ancestorid = $dbinstdir . "/ancestorid.db4"; + if (!(-e "$ancestorid")) { + $ancestorid = $dbinstdir . "/ancestorid.db"; + } + if (-e "$ancestorid") { + my $disorder = 0; + open(ANCESTOR, "/usr/bin/dbscan -f $ancestorid -r |"); + while (<ANCESTOR>) { + if (!/^=[0-9]*/) { + chomp($_); + my @IDs = split(/ | /, $_); + # print "ID count: $#IDs\n"; + my $lasti = $#IDs; + for (my $i = 1; $i < $lasti; $i++) { + if ($IDs[$i] >= $IDs[$i + 1]) { + $disorder = 1; + last; + } + } + # print "Result: $disorder \n"; + if ($disorder) { + last; + } + } + } + close(ANCESTOR); + + # ancestorid index is in disorder; need to reindex it. + if ($disorder) { + print "The ancestorid index in $backend is in disorder; Reindexing $ancestorid.\n"; + $cmd = "$reindex -Z $serverID -n $backend -t ancestorid"; + $rc = system("$cmd"); + if ($rc & 127) { + push @errs, [ 'error_running_command', $cmd, $rc, $! ]; + return @errs; + } + } + } + } else { + # need to upgrade dn format + $rc = 0; + + if (system("cd $pdbdir; tar cf - db/DBVERSION | (cd $dbinstdir; tar xf -)") || + system("cd $pdbdir; tar cf - db/$instname/DBVERSION | (cd $dbinstdir; tar xf -)") || + system("cd $pdbdir; tar cf - db/$instname/*.db* | (cd $dbinstdir; tar xf -)")) { + push @errs, [ "error_cant_backup_db", $backend, $! ]; + return @errs; + } + my @stat = stat("$dbdir"); + my $mode = $stat[2]; + my $uid = $stat[4]; + my $gid = $stat[5]; + + move("$dbinstdir/db", "$workdir"); + chmod($mode, $workdir); + chown($uid, $gid, $workdir); + + @stat = stat("$dbinstdir"); + $mode = $stat[2]; + $uid = $stat[4]; + $gid = $stat[5]; + + chmod($mode, "$workdir/$instname"); + chown($uid, $gid, "$workdir/$instname"); + + my $do_dn_norm_sp = 0; + if ((1 == $escapes) || (3 == $escapes)) { + # We are taking care of spaces in DN. + my $sorted = $dn_norm_sp_txt . ".sorted"; + $cmd = "sort $dn_norm_sp_txt"; + $rc = system("$cmd > $sorted"); + if ($rc) { + debug(1, "Error: $cmd failed - output $sorted: $!\n"); + push @errs, [ 'error_running_command', $cmd, $rc, $! ]; + return @errs; + } + # Create a work file from $sorted. + my $workfile = $sorted . ".work"; + # print "$sorted -> $workfile\n"; + + open(SORTEDFILE, "$sorted"); + open(WORKFILE, "> $workfile"); + my $prev_dn = ""; + my $prev_id = 0; + my $new = 1; + for (my $line = <SORTEDFILE>; $line; $line = <SORTEDFILE>) { + chomp($line); + my ($dn, $id) = split(":", $line, 2); + if ($dn eq $prev_dn) { + if ($new == 1) { + print WORKFILE "$prev_id:$id"; + $new = 0; + $do_dn_norm_sp = 1; # go ahead and fix it. + } else { + print WORKFILE " $id"; + } + } else { + if (0 == $new) { + print WORKFILE "\n"; + $new = 1; + } + } + $prev_dn = $dn; + $prev_id = $id; + } + close(WORKFILE); + close(SORTEDFILE); + unlink <$sorted>; + move("$dn_norm_sp_txt", "$dn_norm_sp_txt.orig"); + if ($do_dn_norm_sp) { + move("$workfile", "$dn_norm_sp_txt"); + # otherwise, we don't need $dn_norm_sp_txt + } + } else { + unlink <$dn_norm_sp_txt>; + } + + if ((1 == $escapes) || (2 == $escapes) || (3 == $escapes)) { + # call conversion tool here and get return status. + $cmd = "$upgradednformat -Z $serverID -n $backend -a $workdir/$instname"; + $rc = system("$cmd"); + if ($rc & 127) { + push @errs, [ 'error_running_command', $cmd, $rc, $! ]; + return @errs; + } + $escapes = $rc >> 8; + } + if ((0 == $rc) || (1 == $escapes) || (3 == $escapes)) { + # success + move("$dbinstdir", "$dbinstdir.orig"); + move("$dbinstdir.orig/dnupgrade/$instname", "$dbinstdir"); + copy("$dbinstdir.orig/dnupgrade/DBVERSION", "$dbdir"); + if ((1 == $escapes) || (3 == $escapes)) { + $cmd = "$reindex -Z $serverID -n $backend -t entryrdn"; + $rc = system("$cmd"); + if ($rc & 127) { + push @errs, [ 'error_running_command', $cmd, $rc, $! ]; + return @errs; + } + } + my $dn_norm_sp_txt = $ldifdir . "/" . $instname . "_dn_norm_sp.txt"; + my $conflict = $ldifdir . "/" . $instname . "_conflict.txt"; + system("echo prinary entry ID: duplicated entry IDs > $conflict"); + system("cat $dn_norm_sp_txt >> $conflict"); + print "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"; + print "Duplicated DN(s) were found and renamed.\n"; + print "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"; + print "Renamed entry IDs are listed in $conflict.\n"; + } else { + # Conversion failed. Cleanup and bail. + unlink <$dbinstdir/dnupgrade/$backend/*>; + rmdir("$dbinstdir/dnupgrade/$backend"); + unlink <$dbinstdir/dnupgrade/*>; + rmdir("$dbinstdir/dnupgrade"); + return ("error_cant_convert_db", $backend, $rc); + } + } + } + + $backend_entry = $conn->nextEntry(); + } + + return (); +} diff --git a/ldap/servers/slapd/back-ldbm/back-ldbm.h b/ldap/servers/slapd/back-ldbm/back-ldbm.h index 726ac40..871d489 100644 --- a/ldap/servers/slapd/back-ldbm/back-ldbm.h +++ b/ldap/servers/slapd/back-ldbm/back-ldbm.h @@ -177,6 +177,7 @@ typedef unsigned short u_int16_t; #define BDB_RDNFORMAT "rdn-format" /* Subtree rename enabled */ #define BDB_RDNFORMAT_VERSION "2" /* rdn-format version (by default, 0) */ #define BDB_DNFORMAT "dn-4514" /* DN format RFC 4514 compliant */ +#define BDB_DNFORMAT_VERSION "1" /* DN format version */ #define DBVERSION_NEWIDL 0x1 #define DBVERSION_RDNFORMAT 0x2 diff --git a/ldap/servers/slapd/back-ldbm/dbversion.c b/ldap/servers/slapd/back-ldbm/dbversion.c index 32a60e5..e4eee32 100644 --- a/ldap/servers/slapd/back-ldbm/dbversion.c +++ b/ldap/servers/slapd/back-ldbm/dbversion.c @@ -119,7 +119,8 @@ dbversion_write(struct ldbminfo *li, const char *directory, ptr = buf + len; } if (flags & DBVERSION_DNFORMAT) { - PR_snprintf(ptr, sizeof(buf) - len, "/%s", BDB_DNFORMAT); + PR_snprintf(ptr, sizeof(buf) - len, "/%s-%s", + BDB_DNFORMAT, BDB_DNFORMAT_VERSION); len = strlen(buf); ptr = buf + len; } diff --git a/ldap/servers/slapd/back-ldbm/import-threads.c b/ldap/servers/slapd/back-ldbm/import-threads.c index 54f8159..1c526ef 100644 --- a/ldap/servers/slapd/back-ldbm/import-threads.c +++ b/ldap/servers/slapd/back-ldbm/import-threads.c @@ -1241,6 +1241,134 @@ upgradedn_add_to_list(struct upgradedn_attr **ud_list, } /* + * Return value: count of max consecutive spaces + */ +static int +has_spaces(const char *str) +{ + char *p = (char *)str; + char *np; + char *endp = p + strlen(str); + int wcnt; + int maxwcnt = 0; +next: + if ((np = strchr(p, ' ')) || (np = strchr(p, '\t'))) { + wcnt = 0; + while ((np < endp) && isspace(*np)) { + wcnt++; + np++; + } + if (maxwcnt < wcnt) { + maxwcnt = wcnt; + } + p = np; + goto next; + } else { + goto bail; + } +bail: + return maxwcnt; +} + +static int +add_IDs_to_IDarray(ID ***dn_norm_sp_conflict, int *max, int i, char *strids) +{ + char *p, *next, *start; + ID my_id; + ID **conflict; + ID *idp; + ID *endp; + size_t len; + + if ((NULL == dn_norm_sp_conflict) || (NULL == max) || (0 == *max)) { + return 1; + } + p = PL_strchr(strids, ':'); + if (NULL == p) { + LDAPDebug1Arg(LDAP_DEBUG_ANY, + "Format error: no ':' in %s\n", strids); + return 1; + } + *p = '\0'; + my_id = (ID)strtol(strids, (char **)NULL, 10); + if (!my_id) { + LDAPDebug1Arg(LDAP_DEBUG_ANY, "Invalid ID in %s\n", strids); + return 1; + } + + if (NULL == *dn_norm_sp_conflict) { + *dn_norm_sp_conflict = (ID **)slapi_ch_malloc(sizeof(ID *) * *max); + } else if (*max == i + 1) { + *max *= 2; + *dn_norm_sp_conflict = (ID **)slapi_ch_realloc((char *)*dn_norm_sp_conflict, + sizeof(ID *) * *max); + } + conflict = *dn_norm_sp_conflict; + + len = strlen(strids); + while (isspace(*(++p))); + start = p; + /* since the format is "ID ID ID ...", max count is len/2 + 1 and +1 for my_id */ + conflict[i] = (ID *)slapi_ch_calloc((len/2 + 2), sizeof(ID)); + idp = conflict[i]; + endp = idp + len/2 + 2; + *idp++ = my_id; + for (p = strtok_r(start, " \n", &next); p && (idp < endp); + p = strtok_r(NULL, " \n", &next), idp++) { + *idp = (ID)strtol(p, (char **)NULL, 10); + } + if (idp < endp) { + *idp = 0; + } else if (idp == endp) { + conflict[i] = (ID *)slapi_ch_realloc((char *)conflict[i], + (len/2 + 3) * sizeof(ID)); + idp = conflict[i]; + *(idp + len/2 + 2) = 0; + } + conflict[i+1] = NULL; + return 0; +} + +static void +free_IDarray(ID ***dn_norm_sp_conflict) +{ + int i; + if ((NULL == dn_norm_sp_conflict) || (NULL == *dn_norm_sp_conflict)) { + return; + } + for (i = 0; (*dn_norm_sp_conflict)[i]; i++) { + slapi_ch_free((void **)&(*dn_norm_sp_conflict)[i]); + } + slapi_ch_free((void **)dn_norm_sp_conflict); +} + +/* + * dn_norm_sp_conflicts is a double array of IDs. + * Format: + * primary_ID0: conflict_ID ... + * primary_ID1: conflict_ID ... + * ... + * + * is_conflict_ID looks for ghe given id in hte conflict_ID lists + * If found, its primary_ID is returned. + * Otherwise, 0 is returned. + */ +static ID +is_conflict_ID(ID **dn_norm_sp_conflicts, int max, ID id) +{ + int i; + ID *idp; + for (i = 0; i < max; i++) { + for (idp = dn_norm_sp_conflicts[i]; idp && *idp; idp++) { + if (*idp == id) { + return *dn_norm_sp_conflicts[i]; + } + } + } + return 0; +} + +/* * Producer thread for upgrading dn format * FLAG_UPGRADEDNFORMAT | FLAG_DRYRUN -- check the necessity of dn upgrade * FLAG_UPGRADEDNFORMAT -- execute dn upgrade @@ -1263,6 +1391,41 @@ upgradedn_add_to_list(struct upgradedn_attr **ud_list, * from ABORTED (aborted or error) and FINISHED (scan all the entries * and found no candidate to upgrade) */ +/* + * FLAG_UPGRADEDNFORMAT: need to take care space reduction + * dn: cn=TEST USER0 --> dn: cn=TEST USER0 + * If multiple DNs are found having different count of spaces, + * remove the second and later with exporting them into ldif file. + * In the dry run, it retrieves all the entries having spaces in dn and + * store the DN and OID in a file. Mark, conflicts if any. + * In the real run, store only no conflict entries and first conflicted one. + * The rest are stored in an ldif file. + * + * Cases: + * FLAG_DRYRUN | FLAG_UPGRADEDNFORMAT | FLAG_UPGRADEDNFORMAT_V1 + * ==> 1 & 2-1 + * FLAG_DRYRUN | FLAG_UPGRADEDNFORMAT_V1 + * ==> 2-1 + * FLAG_UPGRADEDNFORMAT | FLAG_UPGRADEDNFORMAT_V1 + * ==> 1 & 2-2,3 + * FLAG_UPGRADEDNFORMAT_V1 + * ==> 2-2,3 + * + * 1) dn normalization + * 2) space handling + * 2-1) upgradedn dry-run: output into a file (in the instance db dir?) + * format: + * <original dn>:OID + * 2-2) upgrade script sorts the file; retrieve duplicated DNs + * <dn>:OID0 + * <dupped dn>:OID1 + * <dupped dn>:OID2 + * + * format: + * OID0:OID1 OID2 ... + * 2-3) upgradedn: compare the OID with the OID1,OID2, + * if match, rename the rdn value to "value <entryid>". + */ void upgradedn_producer(void *param) { @@ -1279,8 +1442,6 @@ upgradedn_producer(void *param) Slapi_Attr *a = NULL; Slapi_DN *sdn = NULL; char *workdn = NULL; - int doit = 0; - int skipit = 0; struct upgradedn_attr *ud_list = NULL; char **ud_vals = NULL; char **ud_valp = NULL; @@ -1288,6 +1449,18 @@ upgradedn_producer(void *param) Slapi_Attr *ud_attr = NULL; char *ecopy = NULL; const char *normdn = NULL; + char *rdn = NULL; /* original rdn */ + int is_dryrun = 0; /* FLAG_DRYRUN */ + int chk_dn_norm = 0; /* FLAG_UPGRADEDNFORMAT */ + int chk_dn_norm_sp = 0; /* FLAG_UPGRADEDNFORMAT_V1 */ + ID **dn_norm_sp_conflicts = NULL; + int do_dn_norm = 0; /* do dn_norm */ + int do_dn_norm_sp = 0; /* do dn_norm_sp */ + int rdn_has_spaces = 0; + int info_state = 0; /* state to add to info->state (for dryrun only) */ + int skipit = 0; + ID pid; + struct backdn *bdn = NULL; /* vars for Berkeley DB */ DB_ENV *env = NULL; @@ -1304,9 +1477,21 @@ upgradedn_producer(void *param) PR_ASSERT(info != NULL); PR_ASSERT(inst != NULL); PR_ASSERT(be != NULL); - - if ( job->flags & FLAG_ABORT ) + + if ( job->flags & FLAG_ABORT ) { goto error; + } + + is_dryrun = job->flags & FLAG_DRYRUN; + chk_dn_norm = job->flags & FLAG_UPGRADEDNFORMAT; + chk_dn_norm_sp = job->flags & FLAG_UPGRADEDNFORMAT_V1; + + if (!chk_dn_norm && !chk_dn_norm_sp) { + /* Nothing to do... */ + LDAPDebug0Args(LDAP_DEBUG_ANY, "UpgradeDnFormat is not required.\n"); + info->state = FINISHED; + goto done; + } sleeptime = PR_MillisecondsToInterval(import_sleep_time); @@ -1320,15 +1505,14 @@ upgradedn_producer(void *param) /* open id2entry with dedicated db env and db handler */ if ( dblayer_get_aux_id2entry( be, &db, &env, NULL ) != 0 || db == NULL || env == NULL) { - LDAPDebug( LDAP_DEBUG_ANY, "Could not open id2entry\n", 0, 0, 0 ); + LDAPDebug0Args(LDAP_DEBUG_ANY, "Could not open id2entry\n"); goto error; } /* get a cursor to we can walk over the table */ db_rval = db->cursor(db, NULL, &dbc, 0); if ( db_rval || !dbc ) { - LDAPDebug( LDAP_DEBUG_ANY, - "Failed to get cursor for reindexing\n", 0, 0, 0 ); + LDAPDebug0Args(LDAP_DEBUG_ANY, "Failed to get cursor for reindexing\n"); dblayer_release_id2entry(be, db); goto error; } @@ -1352,18 +1536,22 @@ upgradedn_producer(void *param) slapi_ch_free(&(data.data)); key.flags = DB_DBT_MALLOC; data.flags = DB_DBT_MALLOC; - if (isfirst) - { + if (isfirst) { db_rval = dbc->c_get(dbc, &key, &data, DB_FIRST); isfirst = 0; - } - else - { + } else { db_rval = dbc->c_get(dbc, &key, &data, DB_NEXT); } if (0 != db_rval) { - if (DB_NOTFOUND != db_rval) { + if (DB_NOTFOUND == db_rval) { + LDAPDebug1Arg(LDAP_DEBUG_ANY, "%s: Finished to read database\n", + inst->inst_name); + if (job->task) { + slapi_task_log_notice(job->task, + "%s: Finished to read database", inst->inst_name); + } + } else { LDAPDebug(LDAP_DEBUG_ANY, "%s: Failed to read database, " "errno=%d (%s)\n", inst->inst_name, db_rval, dblayer_strerror(db_rval)); @@ -1389,19 +1577,19 @@ upgradedn_producer(void *param) memcpy(ecopy, data.dptr, data.dsize); *(ecopy + data.dsize) = '\0'; normdn = NULL; - doit = 0; + do_dn_norm = 0; + do_dn_norm_sp = 0; + rdn_has_spaces = 0; if (entryrdn_get_switch()) { - char *rdn = NULL; - /* rdn is allocated in get_value_from_string */ + /* original rdn is allocated in get_value_from_string */ rc = get_value_from_string((const char *)data.dptr, "rdn", &rdn); if (rc) { /* data.dptr may not include rdn: ..., try "dn: ..." */ e = slapi_str2entry(data.dptr, SLAPI_STR2ENTRY_USE_OBSOLETE_DNFORMAT); } else { - struct backdn *bdn = - dncache_find_id(&inst->inst_dncache, temp_id); + bdn = dncache_find_id(&inst->inst_dncache, temp_id); if (bdn) { /* don't free dn */ normdn = (char *)slapi_sdn_get_dn(bdn->dn_sdn); @@ -1425,7 +1613,7 @@ upgradedn_producer(void *param) if (rc) { rc = 0; /* assume this is a suffix */ } else { - ID pid = (ID)strtol(pid_str, (char **)NULL, 10); + pid = (ID)strtol(pid_str, (char **)NULL, 10); slapi_ch_free_string(&pid_str); /* if pid is larger than the current pid temp_id, * the parent entry hasn't */ @@ -1433,7 +1621,7 @@ upgradedn_producer(void *param) pid, &id, &psrdn, &curr_entry); if (rc) { LDAPDebug2Args( LDAP_DEBUG_ANY, - "ldbm2index: Failed to compose dn for " + "uptradedn: Failed to compose dn for " "(rdn: %s, ID: %d)\n", rdn, temp_id); slapi_ch_free_string(&rdn); slapi_rdn_done(&psrdn); @@ -1444,7 +1632,7 @@ upgradedn_producer(void *param) slapi_rdn_done(&psrdn); if (rc) { LDAPDebug2Args( LDAP_DEBUG_ANY, - "ldbm2index: Failed to compose dn for " + "uptradedn: Failed to compose dn for " "(rdn: %s, ID: %d) from Slapi_RDN\n", rdn, temp_id); slapi_ch_free_string(&rdn); @@ -1455,23 +1643,34 @@ upgradedn_producer(void *param) rdn, pdn?",":"", pdn?pdn:""); slapi_ch_free_string(&pdn); } - /* dn is not dup'ed in slapi_sdn_new_dn_byref. - * It's set to bdn and put in the dn cache. */ - sdn = slapi_sdn_new_normdn_byref(normdn); - bdn = backdn_init(sdn, temp_id, 0); - CACHE_ADD( &inst->inst_dncache, bdn, NULL ); - CACHE_RETURN(&inst->inst_dncache, &bdn); - slapi_log_error(SLAPI_LOG_CACHE, "ldbm2index", - "entryrdn_lookup_dn returned: %s, " - "and set to dn cache\n", normdn); + if (is_dryrun) { + /* if not dryrun, we may change the DN, In such case, + * we need to put the new value to cache.*/ + /* dn is dup'ed in slapi_sdn_new_dn_byval. + * It's set to bdn and put in the dn cache. */ + sdn = slapi_sdn_new_normdn_byref(normdn); + bdn = backdn_init(sdn, temp_id, 0); + CACHE_ADD( &inst->inst_dncache, bdn, NULL ); + CACHE_RETURN(&inst->inst_dncache, &bdn); + normdn = slapi_sdn_get_dn(sdn); + slapi_log_error(SLAPI_LOG_CACHE, "uptradedn", + "entryrdn_lookup_dn returned: %s, " + "and set to dn cache\n", normdn); + } } e = slapi_str2entry_ext(normdn, NULL, data.dptr, SLAPI_STR2ENTRY_USE_OBSOLETE_DNFORMAT); - slapi_ch_free_string(&rdn); } } else { e = slapi_str2entry(data.data, SLAPI_STR2ENTRY_USE_OBSOLETE_DNFORMAT); + rdn = slapi_ch_strdup(slapi_entry_get_rdn_const(e)); + if (NULL == rdn) { + Slapi_RDN srdn; + slapi_rdn_init_dn(&srdn, slapi_entry_get_dn_const(e)); + rdn = (char *)slapi_rdn_get_rdn(&srdn); /* rdn is allocated in + * slapi_rdn_init_dn */ + } } if ( NULL == e ) { if (job->task) { @@ -1485,6 +1684,7 @@ upgradedn_producer(void *param) continue; } + /* From here, e != NULL */ /* * treat dn specially since the entry was generated with the flag * SLAPI_STR2ENTRY_USE_OBSOLETE_DNFORMAT @@ -1499,24 +1699,154 @@ upgradedn_producer(void *param) normdn = slapi_sdn_get_dn(&(e->e_sdn)); } - /* From here, e != NULL */ + /* + * If dryrun && check_dn_norm_sp, + * check if rdn's containing multi spaces exist or not. + * If any, output the DN:ID into a file INST_dn_norm_sp.txt in the + * ldifdir. We have to continue checking all the entries. + */ + if (chk_dn_norm_sp) { + char *dn_id; + char *path; /* <ldifdir>/<inst>_dn_norm_sp.txt is used for + the temp work file */ + /* open "path" once, and set FILE* to upgradefd */ + if (NULL == job->upgradefd) { + char *ldifdir = config_get_ldifdir(); + if (ldifdir) { + path = slapi_ch_smprintf("%s/%s_dn_norm_sp.txt", + ldifdir, inst->inst_name); + slapi_ch_free_string(&ldifdir); + } else { + path = slapi_ch_smprintf("/var/tmp/%s_dn_norm_sp.txt", + inst->inst_name); + } + if (is_dryrun) { + job->upgradefd = fopen(path, "w"); + if (NULL == job->upgradefd) { + if (job->task) { + slapi_task_log_notice(job->task, + "%s: No DNs to fix.\n", inst->inst_name); + } + LDAPDebug1Arg(LDAP_DEBUG_ANY, + "%s: No DNs to fix.\n", inst->inst_name); + slapi_ch_free_string(&path); + goto bail; + } + } else { + job->upgradefd = fopen(path, "r"); + if (NULL == job->upgradefd) { + if (job->task) { + slapi_task_log_notice(job->task, + "%s: Error: failed to open a file \"%s\"", + inst->inst_name, path); + } + LDAPDebug2Args(LDAP_DEBUG_ANY, + "%s: Error: failed to open a file \"%s\"\n", + inst->inst_name, path); + slapi_ch_free_string(&path); + goto error; + } + } + } + slapi_ch_free_string(&path); + if (is_dryrun) { + rdn_has_spaces = has_spaces(rdn); + if (rdn_has_spaces > 0) { + dn_id = slapi_ch_smprintf("%s:%u\n", + slapi_entry_get_dn_const(e), temp_id); + if (EOF == fputs(dn_id, job->upgradefd)) { + if (job->task) { + slapi_task_log_notice(job->task, + "%s: Error: failed to write a line \"%s\"", + inst->inst_name, dn_id); + } + LDAPDebug2Args(LDAP_DEBUG_ANY, + "%s: Error: failed to write a line \"%s\"", + inst->inst_name, dn_id); + slapi_ch_free_string(&dn_id); + goto error; + } + slapi_ch_free_string(&dn_id); + if (rdn_has_spaces > 1) { + /* If an rdn containing multi spaces exists, + * let's check the conflict. */ + do_dn_norm_sp = 1; + } + } + } else { /* !is_dryrun */ + /* check the oid and parentid. */ + /* Set conflict list once, and refer it laster. */ + static int my_idx = 0; + ID alt_id; + if (NULL == dn_norm_sp_conflicts) { + char buf[BUFSIZ]; + char *p; + int my_max = 8; + while (fgets(buf, BUFSIZ-1, job->upgradefd)) { + /* search "OID0: OID1 OID2 ... */ + if (!isdigit(*buf) || !(p = PL_strchr(buf, ':'))) { + continue; + } + if (add_IDs_to_IDarray(&dn_norm_sp_conflicts, &my_max, + my_idx, buf)) { + LDAPDebug1Arg(LDAP_DEBUG_ANY, + "Failed to set IDs %s to conflict list\n", + buf); + goto error; + } + my_idx++; + } + } + alt_id = is_conflict_ID(dn_norm_sp_conflicts, my_idx, temp_id); + if (alt_id) { + char *parentdn = NULL; + char *newrdn = NULL; + if (alt_id != temp_id) { + parentdn = slapi_dn_parent(normdn); + newrdn = slapi_create_dn_string("%s %u", rdn, temp_id); + /* This entry is a conflict of alt_id */ + LDAPDebug(LDAP_DEBUG_ANY, + "Entry %s (%lu) is a conflict of (%lu)\n", + normdn, temp_id, alt_id); + LDAPDebug2Args(LDAP_DEBUG_ANY, "Renaming \"%s\" to \"%s\"\n", + rdn, newrdn); + normdn = slapi_ch_smprintf("%s,%s", newrdn, parentdn); + /* Reset DN and RDN in the entry */ + slapi_sdn_done(&(e->e_sdn)); + slapi_sdn_init_normdn_passin(&(e->e_sdn), normdn); + /* normdn = slapi_sdn_get_dn(&(e->e_sdn)); */ + } + info_state |= DN_NORM_SP; + upgradedn_add_to_list(&ud_list, + slapi_ch_strdup(LDBM_ENTRYRDN_STR), + slapi_ch_strdup(rdn), 0); + rc = slapi_entry_add_rdn_values(e); + if (rc) { + LDAPDebug(LDAP_DEBUG_ANY, "%s: Failed to add rdn values" + " to an entry: %s (id %lu)\n", + inst->inst_name, normdn, (u_long)temp_id); + goto error; + } + } /* !alt_id */ + } /* !is_dryrun */ + } /* if (chk_dn_norm_sp) */ + + /* dn is dup'ed in slapi_sdn_new_dn_byval. + * It's set to bdn and put in the dn cache. */ + /* Waited to put normdn into dncache until it could be modified in + * chk_dn_norm_sp. */ + sdn = slapi_sdn_new_normdn_byref(normdn); + bdn = backdn_init(sdn, temp_id, 0); + CACHE_ADD(&inst->inst_dncache, bdn, NULL); + CACHE_RETURN(&inst->inst_dncache, &bdn); + slapi_log_error(SLAPI_LOG_CACHE, "uptradedn", + "set dn %s to dn cache\n", normdn); /* Check DN syntax attr values if it contains '\\' or not */ /* Start from the rdn */ - if (entryrdn_get_switch()) { /* subtree-rename: on */ - char *rdn = NULL; - size_t rdnlen = 0; + if (chk_dn_norm) { char *endrdn = NULL; char *rdnp = NULL; - rc = get_value_from_string((const char *)ecopy, "rdn", &rdn); - if (rc || (NULL == rdn)) { - LDAPDebug2Args(LDAP_DEBUG_ANY, - "%s: WARNING: skipping an entry with no RDN (id %lu)\n", - inst->inst_name, (u_long)temp_id); - slapi_entry_free(e); e = NULL; - continue; - } - rdnlen = strlen(rdn); - endrdn = rdn + rdnlen - 1; + endrdn = rdn + strlen(rdn) - 1; rdnp = PL_strchr(rdn, '='); if (NULL == rdnp) { @@ -1528,24 +1858,23 @@ upgradedn_producer(void *param) continue; } /* rdn contains '\\'. We have to update the value */ - if (PL_strchr(rdn, '\\')) { - doit = 1; + if (PL_strchr(rdnp, '\\')) { + do_dn_norm = 1; } else { while ((++rdnp <= endrdn) && (*rdnp == ' ') && (*rdnp == '\t')) ; /* DN contains an RDN <type>="<value>" ? */ if ((rdnp != endrdn) && ('"' == *rdnp) && ('"' == *endrdn)) { - doit = 1; + do_dn_norm = 1; } } - if (doit) { + if (do_dn_norm) { upgradedn_add_to_list(&ud_list, slapi_ch_strdup(LDBM_ENTRYRDN_STR), slapi_ch_strdup(rdn), 0); - LDAPDebug(LDAP_DEBUG_TRACE, - "%s: Found upgradedn candidate: %s (id %lu)\n", - inst->inst_name, *ud_valp, (u_long)temp_id); - /* entryrdn format */ + LDAPDebug2Args(LDAP_DEBUG_TRACE, + "%s: Found upgradedn candidate: (id %lu)\n", + inst->inst_name, (u_long)temp_id); /* * In case rdn is type="<RDN>" or type=<\D\N>, * add the rdn value if it's not there. @@ -1559,130 +1888,146 @@ upgradedn_producer(void *param) continue; } } - slapi_ch_free_string(&rdn); - } - for (a = e->e_attrs; a; a = a->a_next) { - if (!slapi_attr_is_dn_syntax_attr(a)) { - continue; /* not a syntax dn attr */ - } - - /* dn syntax attr */ - rc = get_values_from_string((const char *)ecopy, - a->a_type, &ud_vals); - if (rc || (NULL == ud_vals)) { - continue; /* empty; ignore it */ - } - - for (ud_valp = ud_vals; ud_valp && *ud_valp; ud_valp++) { - char **rdns = NULL; - char **rdnsp = NULL; - char *valueptr = NULL; - char *endvalue = NULL; - int isentrydn = 0; - - /* Also check RDN contains double quoted values */ - if (strcasecmp(a->a_type, "entrydn")) { - /* except entrydn */ - workdn = slapi_ch_strdup(*ud_valp); - isentrydn = 0; - } else { - /* entrydn: Get Slapi DN */ - sdn = slapi_entry_get_sdn(e); - workdn = slapi_ch_strdup(slapi_sdn_get_dn(sdn)); - isentrydn = 1; + /* checking the DN sintax values in the attribute list */ + for (a = e->e_attrs; a; a = a->a_next) { + if (!slapi_attr_is_dn_syntax_attr(a)) { + continue; /* not a syntax dn attr */ } - rdns = slapi_ldap_explode_dn(workdn, 0); - skipit = 0; - for (rdnsp = rdns; rdnsp && *rdnsp; rdnsp++) { - valueptr = PL_strchr(*rdnsp, '='); - if (NULL == valueptr) { - skipit = 1; - break; - } - endvalue = *rdnsp + strlen(*rdnsp) - 1; - while ((++valueptr <= endvalue) && - ((' ' == *valueptr) || ('\t' == *valueptr))) ; - if (0 == strlen(valueptr)) { - skipit = 1; - break; + + /* dn syntax attr */ + rc = get_values_from_string((const char *)ecopy, + a->a_type, &ud_vals); + if (rc || (NULL == ud_vals)) { + continue; /* empty; ignore it */ + } + + for (ud_valp = ud_vals; ud_valp && *ud_valp; ud_valp++) { + char **rdns = NULL; + char **rdnsp = NULL; + char *valueptr = NULL; + char *endvalue = NULL; + int isentrydn = 0; + + /* Also check RDN contains double quoted values */ + if (strcasecmp(a->a_type, "entrydn")) { + /* except entrydn */ + workdn = slapi_ch_strdup(*ud_valp); + isentrydn = 0; + } else { + /* entrydn: Get Slapi DN */ + sdn = slapi_entry_get_sdn(e); + workdn = slapi_ch_strdup(slapi_sdn_get_dn(sdn)); + isentrydn = 1; } - /* DN syntax value contains an RDN <type>="<value>" or - * '\\' in the value ? - * If yes, let's upgrade the dn format. */ - if ((('"' == *valueptr) && ('"' == *endvalue)) || - PL_strchr(valueptr, '\\')) { - doit = 1; - upgradedn_add_to_list(&ud_list, - slapi_ch_strdup(a->a_type), - slapi_ch_strdup(*ud_valp), - isentrydn?0:OLD_DN_NORMALIZE); - LDAPDebug(LDAP_DEBUG_TRACE, - "%s: Found upgradedn candidate: %s (id %lu)\n", - inst->inst_name, valueptr, (u_long)temp_id); - if (!entryrdn_get_switch() && isentrydn) { - /* entrydn format */ - /* - * In case entrydn is type="<DN>",<REST> or - * type=<\D\N>,<REST>, - * add the rdn value if it's not there. - */ - rc = slapi_entry_add_rdn_values(e); - if (rc) { - LDAPDebug(LDAP_DEBUG_ANY, - "%s: Failed to add rdn values " - "to an entry: %s (id %lu)\n", - inst->inst_name, normdn, (u_long)temp_id); - slapi_entry_free(e); e = NULL; - continue; + rdns = slapi_ldap_explode_dn(workdn, 0); + skipit = 0; + for (rdnsp = rdns; rdnsp && *rdnsp; rdnsp++) { + valueptr = PL_strchr(*rdnsp, '='); + if (NULL == valueptr) { + skipit = 1; + break; + } + endvalue = *rdnsp + strlen(*rdnsp) - 1; + while ((++valueptr <= endvalue) && + ((' ' == *valueptr) || ('\t' == *valueptr))) ; + if (0 == strlen(valueptr)) { + skipit = 1; + break; + } + /* DN syntax value contains an RDN <type>="<value>" or + * '\\' in the value ? + * If yes, let's upgrade the dn format. */ + if ((('"' == *valueptr) && ('"' == *endvalue)) || + PL_strchr(valueptr, '\\')) { + do_dn_norm = 1; + upgradedn_add_to_list(&ud_list, + slapi_ch_strdup(a->a_type), + slapi_ch_strdup(*ud_valp), + isentrydn?0:OLD_DN_NORMALIZE); + LDAPDebug(LDAP_DEBUG_TRACE, + "%s: Found upgradedn candidate: %s (id %lu)\n", + inst->inst_name, valueptr, (u_long)temp_id); + if (!entryrdn_get_switch() && isentrydn) { + /* entrydn format */ + /* + * In case entrydn is type="<DN>",<REST> or + * type=<\D\N>,<REST>, + * add the rdn value if it's not there. + */ + rc = slapi_entry_add_rdn_values(e); + if (rc) { + LDAPDebug(LDAP_DEBUG_ANY, + "%s: Failed to add rdn values " + "to an entry: %s (id %lu)\n", + inst->inst_name, normdn, (u_long)temp_id); + slapi_entry_free(e); e = NULL; + continue; + } } + break; } + /* + * else if (the rdn contains multiple spaces)? + * if yes, they are reduced to one. + * SET HAVE_MULTI_SPACES??? + */ + } /* for (rdnsp = rdns; rdnsp && *rdnsp; rdnsp++) */ + if (rdns) { + slapi_ldap_value_free(rdns); + } else { + skipit = 1; + } + if (skipit) { break; } - } /* for (rdnsp = rdns; rdnsp && *rdnsp; rdnsp++) */ - if (rdns) { - slapi_ldap_value_free(rdns); - } else { - skipit = 1; - } + slapi_ch_free_string(&workdn); + } /* for (ud_valp = ud_vals; ud_valp && *ud_valp; ud_valp++) */ + charray_free(ud_vals); + ud_vals = NULL; if (skipit) { + LDAPDebug(LDAP_DEBUG_ANY, "%s: WARNING: skipping an entry " + "with a corrupted dn (syntax value): %s " + "(id %lu)\n", + inst->inst_name, + workdn?workdn:"unknown", (u_long)temp_id); + slapi_ch_free_string(&workdn); + upgradedn_free_list(&ud_list); break; } - slapi_ch_free_string(&workdn); - } /* for (ud_valp = ud_vals; ud_valp && *ud_valp; ud_valp++) */ - charray_free(ud_vals); - ud_vals = NULL; + } /* for (a = e->e_attrs; a; a = a->a_next) */ if (skipit) { - LDAPDebug(LDAP_DEBUG_ANY, "%s: WARNING: skipping an entry " - "with a corrupted dn (syntax value): %s " - "(id %lu)\n", - inst->inst_name, - workdn?workdn:"unknown", (u_long)temp_id); - slapi_ch_free_string(&workdn); upgradedn_free_list(&ud_list); - break; + slapi_entry_free(e); e = NULL; + continue; } - } /* for (a = e->e_attrs; a; a = a->a_next) */ - if (skipit) { - upgradedn_free_list(&ud_list); - slapi_entry_free(e); e = NULL; - continue; - } + } /* end of if (chk_do_norm) */ + slapi_ch_free_string(&rdn); - if (!doit) { + if (is_dryrun) { + if (do_dn_norm) { + info_state |= DN_NORM; + /* + * If dryrun AND (found we need to do dn norm) AND + * (no need to check spaces), + * then we can quit here to return. + */ + if (!chk_dn_norm_sp) { + finished = 0; /* make it sure ... */ + upgradedn_free_list(&ud_list); + slapi_entry_free(e); e = NULL; + /* found upgrade dn candidates */ + goto bail; + } + } + if (do_dn_norm_sp) { + info_state |= DN_NORM_SP; + } /* We don't have to update dn syntax values. */ upgradedn_free_list(&ud_list); slapi_entry_free(e); e = NULL; continue; } - - /* doit */ - if (job->flags & FLAG_DRYRUN) { - /* We can return SUCCESS (== found upgrade dn candidates) */ - finished = 0; /* make it sure ... */ - upgradedn_free_list(&ud_list); - slapi_entry_free(e); e = NULL; - goto bail; - } + skipit = 0; for (ud_ptr = ud_list; ud_ptr; ud_ptr = ud_ptr->ud_next) { Slapi_Value *value = NULL; @@ -1810,23 +2155,14 @@ upgradedn_producer(void *param) id++; if (job->flags & FLAG_ABORT) goto error; - if (info->command == STOP) - { + if (info->command == STOP) { finished = 1; } } bail: dbc->c_close(dbc); dblayer_release_aux_id2entry( be, db, env ); - if (job->flags & FLAG_DRYRUN) { - if (finished) { /* Set if dn upgrade candidates are not found */ - info->state = FINISHED; - } else { /* At least one dn upgrade candidate is found */ - info->state = QUIT; - } - } else { - info->state = FINISHED; - } + info->state = FINISHED | info_state; goto done; error: @@ -1837,8 +2173,12 @@ error: info->state = ABORTED; done: + free_IDarray(&dn_norm_sp_conflicts); slapi_ch_free_string(&ecopy); slapi_ch_free(&(data.data)); + if (job->upgradefd) { + fclose(job->upgradefd); + } } static void @@ -1883,7 +2223,7 @@ foreman_do_parentid(ImportJob *job, FifoItem *fi, struct attrinfo *parentid_ai) int ret = 0; struct backentry *entry = fi->entry; - if (job->flags & FLAG_UPGRADEDNFORMAT) { + if (job->flags & (FLAG_UPGRADEDNFORMAT|FLAG_UPGRADEDNFORMAT_V1)) { /* Get the parentid attribute value from deleted attr list */ Slapi_Value *value = NULL; Slapi_Attr *pid_to_del = @@ -1950,7 +2290,7 @@ foreman_do_entrydn(ImportJob *job, FifoItem *fi) IDList *IDL; struct backentry *entry = fi->entry; - if (job->flags & FLAG_UPGRADEDNFORMAT) { + if (job->flags & (FLAG_UPGRADEDNFORMAT|FLAG_UPGRADEDNFORMAT_V1)) { /* Get the entrydn attribute value from deleted attr list */ Slapi_Value *value = NULL; Slapi_Attr *entrydn_to_del = @@ -1994,7 +2334,7 @@ foreman_do_entrydn(ImportJob *job, FifoItem *fi) /* So, we do an index read first */ err = 0; IDL = index_read(be, LDBM_ENTRYDN_STR, indextype_EQUALITY, &bv, NULL, &err); - if (job->flags & FLAG_UPGRADEDNFORMAT) { + if (job->flags & (FLAG_UPGRADEDNFORMAT|FLAG_UPGRADEDNFORMAT_V1)) { /* * In the UPGRADEDNFORMAT case, if entrydn value exists, * that means either 1) entrydn is not upgraded (ID == entry->ep_id) @@ -2058,7 +2398,7 @@ foreman_do_entryrdn(ImportJob *job, FifoItem *fi) int ret = 0; struct backentry *entry = fi->entry; - if (job->flags & FLAG_UPGRADEDNFORMAT) { + if (job->flags & (FLAG_UPGRADEDNFORMAT|FLAG_UPGRADEDNFORMAT_V1)) { /* Get the entrydn attribute value from deleted attr list */ Slapi_Value *value = NULL; Slapi_Attr *entryrdn_to_del = NULL; @@ -2084,6 +2424,10 @@ foreman_do_entryrdn(ImportJob *job, FifoItem *fi) } } slapi_attr_free(&entryrdn_to_del); + /* Waited to update e_srdn to delete the old entryrdn. + * Now updated it to adjust to the new s_dn. */ + slapi_rdn_set_all_dn(&(entry->ep_entry->e_srdn), + slapi_entry_get_dn_const(entry->ep_entry)); } } ret = entryrdn_index_entry(be, entry, BE_INDEX_ADD, NULL); @@ -2169,7 +2513,19 @@ import_foreman(void *param) import_log_notice(job, "WARNING: entry for id %d is missing", id); continue; } - + if (job->flags & FLAG_UPGRADEDNFORMAT_V1) { + if (entryrdn_get_switch()) { /* subtree-rename: on */ + /* insert into the entryrdn index */ + ret = foreman_do_entryrdn(job, fi); + } else { + /* insert into the entrydn index */ + ret = foreman_do_entrydn(job, fi); + if (ret == -1) { + goto cont; /* skip entry */ + } + } + goto next; + } /* first, fill in any operational attributes */ /* add_op_attrs wants a pblock for some reason. */ if (job->flags & FLAG_UPGRADEDNFORMAT) { @@ -2312,7 +2668,7 @@ import_foreman(void *param) if (job->flags & FLAG_ABORT) { goto error; } - +next: if (!(job->flags & FLAG_REINDEXING))/* reindex reads data from id2entry */ { /* insert into the id2entry index @@ -2348,14 +2704,14 @@ import_foreman(void *param) goto error; } - if (entryrdn_get_switch() /* subtree-rename: on */ || + if (entryrdn_get_switch() /* subtree-rename: on */ && !slapi_entry_flag_is_set(fi->entry->ep_entry, SLAPI_ENTRY_FLAG_TOMBSTONE)) { /* parentid index * (we have to do this here, because the parentID is dependent on * looking up by entrydn/entryrdn.) * Only add to the parent index if the entry is not a tombstone && - * subtree-rename is off. + * subtree-rename is on. */ ret = foreman_do_parentid(job, fi, parentid_ai); if (ret != 0) @@ -2536,7 +2892,7 @@ import_worker(void *param) slapi_pblock_destroy(pb); } else { /* No, process regular index */ - if (job->flags & FLAG_UPGRADEDNFORMAT) { + if (job->flags & (FLAG_UPGRADEDNFORMAT|FLAG_UPGRADEDNFORMAT_V1)) { /* Get the attribute value from deleted attr list */ Slapi_Value *value = NULL; const struct berval *bval = NULL; diff --git a/ldap/servers/slapd/back-ldbm/import.c b/ldap/servers/slapd/back-ldbm/import.c index e3966ff..aedff9f 100644 --- a/ldap/servers/slapd/back-ldbm/import.c +++ b/ldap/servers/slapd/back-ldbm/import.c @@ -51,7 +51,9 @@ #include "import.h" #define ERR_IMPORT_ABORTED -23 -#define DRYRUN_QUIT -24 +#define NEED_DN_NORM -24 +#define NEED_DN_NORM_SP -25 +#define NEED_DN_NORM_BT -26 /********** routines to manipulate the entry fifo **********/ @@ -111,7 +113,9 @@ FifoItem *import_fifo_fetch(ImportJob *job, ID id, int worker) if (fi->bad) { if (fi->bad == FIFOITEM_BAD) { fi->bad = FIFOITEM_BAD_PRINTED; - import_log_notice(job, "WARNING: bad entry: ID %d", id); + if (!(job->flags & FLAG_UPGRADEDNFORMAT_V1)) { + import_log_notice(job, "WARNING: bad entry: ID %d", id); + } } return NULL; } @@ -197,7 +201,7 @@ void import_log_notice(ImportJob *job, char *format, ...) slapi_task_log_notice(job->task, "%s", buffer); } /* also save it in the logs for posterity */ - if (job->flags & FLAG_UPGRADEDNFORMAT) { + if (job->flags & (FLAG_UPGRADEDNFORMAT|FLAG_UPGRADEDNFORMAT_V1)) { LDAPDebug(LDAP_DEBUG_ANY, "upgradedn %s: %s\n", job->inst->inst_name, buffer, 0); } else if (job->flags & FLAG_REINDEXING) { @@ -260,7 +264,7 @@ static int import_attr_callback(void *node, void *param) if (job->flags & FLAG_DRYRUN) { /* dryrun; we don't need the workers */ return 0; } - if (job->flags & FLAG_UPGRADEDNFORMAT) { + if (job->flags & (FLAG_UPGRADEDNFORMAT|FLAG_UPGRADEDNFORMAT_V1)) { /* Bring up import workers just for indexes having DN syntax * attribute type. (except entrydn -- taken care below) */ int rc = 0; @@ -707,6 +711,7 @@ static int import_monitor_threads(ImportJob *job, int *status) time_t last_time = 0; time_t time_interval = 0; int rc = 0; + int corestate = 0; for (current_worker = job->worker_list; current_worker != NULL; current_worker = current_worker->next) { @@ -774,14 +779,31 @@ static int import_monitor_threads(ImportJob *job, int *status) import_calc_rate(current_worker, time_interval); import_print_worker_status(current_worker); } - if (current_worker->state == QUIT) { - rc = DRYRUN_QUIT; /* Set the RC; Don't abort now; - We have to stop other threads */ - } else if (current_worker->state != FINISHED) { - finished = 0; - } + corestate = current_worker->state & CORESTATE; if (current_worker->state == ABORTED) { +LDAPDebug0Args(LDAP_DEBUG_ANY, "import_monitor_threads: current_worker->state is ABORTED\n"); goto error_abort; + } else if ((corestate == QUIT) || (corestate == FINISHED)) { +LDAPDebug1Arg(LDAP_DEBUG_ANY, "import_monitor_threads: current_worker->state is %s\n", (corestate==QUIT)?"QUIT":"FINISHED"); + if (DN_NORM_BT == (DN_NORM_BT & current_worker->state)) { + /* upgrading dn norm (both) is needed */ + rc = NEED_DN_NORM_BT; /* Set the RC; Don't abort now; + * We have to stop other + * threads */ + } else if (DN_NORM == (DN_NORM_BT & current_worker->state)) { + /* upgrading dn norm is needed */ + rc = NEED_DN_NORM; /* Set the RC; Don't abort now; + * We have to stop other threads + */ + } else if (DN_NORM_SP == (DN_NORM_BT & current_worker->state)) { + /* upgrading spaces in dn norm is needed */ + rc = NEED_DN_NORM_SP; /* Set the RC; Don't abort now; + * We have to stop other + * threads */ + } + current_worker->state = corestate; + } else if (current_worker->state != FINISHED) { + finished = 0; } } @@ -906,7 +928,9 @@ static int import_run_pass(ImportJob *job, int *status) /* Monitor the threads until we're done or fail */ ret = import_monitor_threads(job, status); - if ((ret == ERR_IMPORT_ABORTED) || (ret == DRYRUN_QUIT)) { + if ((ret == ERR_IMPORT_ABORTED) || (ret == NEED_DN_NORM) || + (ret == NEED_DN_NORM_SP) || (ret == NEED_DN_NORM_BT)) { +import_log_notice(job, "Thread monitoring returned: %d\n", ret); goto error; } else if (ret != 0) { import_log_notice(job, "Thread monitoring aborted: %d\n", ret); @@ -1140,11 +1164,16 @@ int import_main_offline(void *arg) if (job->task) slapi_task_inc_refcount(job->task); - if (job->flags & FLAG_UPGRADEDNFORMAT) { + if (job->flags & (FLAG_UPGRADEDNFORMAT|FLAG_UPGRADEDNFORMAT_V1)) { if (job->flags & FLAG_DRYRUN) { opstr = "Upgrade Dn Dryrun"; + } else if ((job->flags & (FLAG_UPGRADEDNFORMAT|FLAG_UPGRADEDNFORMAT_V1)) + == (FLAG_UPGRADEDNFORMAT|FLAG_UPGRADEDNFORMAT_V1)) { + opstr = "Upgrade Dn (Full)"; + } else if (job->flags & FLAG_UPGRADEDNFORMAT_V1) { + opstr = "Upgrade Dn (Spaces)"; } else { - opstr = "Upgrade Dn"; + opstr = "Upgrade Dn (RFC 4514)"; } } else if (job->flags & FLAG_REINDEXING) { opstr = "Reindexing"; @@ -1187,7 +1216,7 @@ int import_main_offline(void *arg) /* start the producer */ import_init_worker_info(producer, job); producer->work_type = PRODUCER; - if (job->flags & FLAG_UPGRADEDNFORMAT) + if (job->flags & (FLAG_UPGRADEDNFORMAT|FLAG_UPGRADEDNFORMAT_V1)) { if (! CREATE_THREAD(PR_USER_THREAD, (VFP)upgradedn_producer, producer, PR_PRIORITY_NORMAL, PR_GLOBAL_BOUND_THREAD, @@ -1278,11 +1307,10 @@ int import_main_offline(void *arg) aborted = 1; goto error; } - if (ret == DRYRUN_QUIT) { - goto error; /* Found the candidate; close the db files and quit */ - } - - if (0 != ret) { + if ((ret == NEED_DN_NORM) || (ret == NEED_DN_NORM_SP) || + (ret == NEED_DN_NORM_BT)) { + goto error; + } else if (0 != ret) { /* Some horrible fate has befallen the import */ import_log_notice(job, "Fatal pass error %d", ret); goto error; @@ -1414,7 +1442,8 @@ error: } } if (0 != ret) { - if (!(job->flags & FLAG_DRYRUN)) { /* If not dryrun */ + if (!(job->flags & (FLAG_DRYRUN|FLAG_UPGRADEDNFORMAT_V1))) { + /* If not dryrun NOR upgradedn space */ /* if running in the dry run mode, don't touch the db */ dblayer_delete_instance_dir(be); } @@ -1475,23 +1504,40 @@ error: } } - if (job->flags & FLAG_DRYRUN) { + if (job->flags & (FLAG_DRYRUN|FLAG_UPGRADEDNFORMAT_V1)) { if (0 == ret) { import_log_notice(job, "%s complete. %s is up-to-date.", opstr, job->inst->inst_name); + ret = 0; + if (job->task) { + slapi_task_dec_refcount(job->task); + } + import_all_done(job, ret); + } else if (NEED_DN_NORM_BT == ret) { + import_log_notice(job, "%s complete. %s needs upgradednformat all.", + opstr, job->inst->inst_name); + if (job->task) { + slapi_task_dec_refcount(job->task); + } + import_all_done(job, ret); ret = 1; + } else if (NEED_DN_NORM == ret) { + import_log_notice(job, "%s complete. %s needs upgradednformat.", + opstr, job->inst->inst_name); if (job->task) { slapi_task_dec_refcount(job->task); } import_all_done(job, ret); - } else if (DRYRUN_QUIT == ret) { - import_log_notice(job, "%s complete. %s needs upgradednformat.", + ret = 2; + } else if (NEED_DN_NORM_SP == ret) { + import_log_notice(job, + "%s complete. %s needs upgradednformat spaces.", opstr, job->inst->inst_name); if (job->task) { slapi_task_dec_refcount(job->task); } import_all_done(job, ret); - ret = 0; + ret = 3; } else { ret = -1; if (job->task != NULL) { @@ -1581,8 +1627,13 @@ int ldbm_back_ldif2ldbm_deluxe(Slapi_PBlock *pb) job->flags = FLAG_USE_FILES; if (NULL == name_array) { /* no ldif file is given -> reindexing or upgradedn */ - if (up_flags & SLAPI_UPGRADEDNFORMAT) { - job->flags |= FLAG_UPGRADEDNFORMAT; + if (up_flags & (SLAPI_UPGRADEDNFORMAT|SLAPI_UPGRADEDNFORMAT_V1)) { + if (up_flags & SLAPI_UPGRADEDNFORMAT) { + job->flags |= FLAG_UPGRADEDNFORMAT; + } + if (up_flags & SLAPI_UPGRADEDNFORMAT_V1) { + job->flags |= FLAG_UPGRADEDNFORMAT_V1; + } if (up_flags & SLAPI_DRYRUN) { job->flags |= FLAG_DRYRUN; } diff --git a/ldap/servers/slapd/back-ldbm/import.h b/ldap/servers/slapd/back-ldbm/import.h index c9a3f8a..7870403 100644 --- a/ldap/servers/slapd/back-ldbm/import.h +++ b/ldap/servers/slapd/back-ldbm/import.h @@ -159,8 +159,9 @@ typedef struct { PRLock *wire_lock; /* lock for serializing wire imports */ PRCondVar *wire_cv; /* ... and ordering the startup */ PRThread *main_thread; /* for FRI: import_main() thread id */ - int encrypt; - Slapi_Value *usn_value; /* entryusn for import */ + int encrypt; + Slapi_Value *usn_value; /* entryusn for import */ + FILE *upgradefd; /* used for the upgrade */ } ImportJob; #define FLAG_INDEX_ATTRS 0x01 /* should we index the attributes? */ @@ -173,6 +174,7 @@ typedef struct { #define FLAG_DN2RDN 0x40 /* modify backend to the rdn format */ #define FLAG_UPGRADEDNFORMAT 0x80 /* read from id2entry and do upgrade dn */ #define FLAG_DRYRUN 0x100 /* dryrun for upgrade dn */ +#define FLAG_UPGRADEDNFORMAT_V1 0x200 /* taking care multiple spaces in dn */ /* Structure holding stuff about a worker thread and what it's up to */ @@ -202,12 +204,17 @@ struct _import_worker_info { #define ABORT 3 #define STOP 4 -/* Values for state */ -#define WAITING 1 -#define RUNNING 2 -#define FINISHED 3 -#define ABORTED 4 -#define QUIT 5 /* quit intentionally. to distinguish from ABORTED & FINISHED */ +/* Values for job state */ +#define WAITING 0x1 +#define RUNNING 0x2 +#define FINISHED 0x4 +#define ABORTED 0x8 +#define QUIT 0x10 /* quit intentionally. + * introduced to distinguish from ABORTED, FINISHED */ +#define CORESTATE 0xff +#define DN_NORM 0x100 /* do dn normalization in upgrade */ +#define DN_NORM_SP 0x200 /* do dn normalization for multi spaces in upgrade */ +#define DN_NORM_BT (DN_NORM | DN_NORM_SP) /* this is just a convenience, because the slapi_ch_* calls are annoying */ #define CALLOC(name) (name *)slapi_ch_calloc(1, sizeof(name)) diff --git a/ldap/servers/slapd/back-ldbm/ldif2ldbm.c b/ldap/servers/slapd/back-ldbm/ldif2ldbm.c index 8818ba7..0793041 100644 --- a/ldap/servers/slapd/back-ldbm/ldif2ldbm.c +++ b/ldap/servers/slapd/back-ldbm/ldif2ldbm.c @@ -204,8 +204,9 @@ int ldbm_back_ok_to_dump(const char *dn, char **include, char **exclude) * allowed to specify yourself on entries. * Currenty the list of these is: numSubordinates, hasSubordinates */ -int add_op_attrs(Slapi_PBlock *pb, struct ldbminfo *li, struct backentry *ep, - int *status) +int +add_op_attrs(Slapi_PBlock *pb, struct ldbminfo *li, struct backentry *ep, + int *status) { backend *be; char *pdn; @@ -3549,7 +3550,7 @@ int ldbm_back_upgradednformat(Slapi_PBlock *pb) slapi_pblock_get(pb, SLAPI_BACKEND_TASK, &task); slapi_pblock_get(pb, SLAPI_DB2LDIF_SERVER_RUNNING, &server_running); slapi_pblock_get(pb, SLAPI_BACKEND_INSTANCE_NAME, &instance_name); - slapi_pblock_get( pb, SLAPI_SEQ_TYPE, &ud_flags ); + slapi_pblock_get(pb, SLAPI_SEQ_TYPE, &ud_flags); run_from_cmdline = (task_flags & SLAPI_TASK_RUNNING_FROM_COMMANDLINE); slapi_pblock_get(pb, SLAPI_PLUGIN_PRIVATE, &li); @@ -3624,11 +3625,39 @@ int ldbm_back_upgradednformat(Slapi_PBlock *pb) workdbdir = rel2abspath(rawworkdbdir); dbversion_read(li, workdbdir, &ldbmversion, &dataversion); - if (ldbmversion && PL_strstr(ldbmversion, BDB_DNFORMAT)) { + if (ldbmversion) { + char *ptr = PL_strstr(ldbmversion, BDB_DNFORMAT); + if (ptr) { + size_t dnformat_len = strlen(ptr); + /* DN format is RFC 4514 compliant */ + if (strlen(ptr) == strlen(BDB_DNFORMAT)) { /* no version */ + /* + * DN format is RFC 4514 compliant. + * But it hasn't taken care of the multiple spaces yet. + */ + ud_flags &= ~SLAPI_UPGRADEDNFORMAT; + ud_flags |= SLAPI_UPGRADEDNFORMAT_V1; + slapi_pblock_set(pb, SLAPI_SEQ_TYPE, &ud_flags); + rc = 3; /* 0: need upgrade (dn norm sp, only) */ + } else { + /* DN format already takes care of the multiple spaces */ + slapi_log_error(SLAPI_LOG_FATAL, "Upgrade DN Format", + "Instance %s in %s is up-to-date\n", + instance_name, workdbdir); + rc = 0; /* 0: up-to-date */ + goto bail; + } + } else { + /* DN format is not RFC 4514 compliant */ + ud_flags |= SLAPI_UPGRADEDNFORMAT | SLAPI_UPGRADEDNFORMAT_V1; + slapi_pblock_set(pb, SLAPI_SEQ_TYPE, &ud_flags); + rc = 1; /* 0: need upgrade (both) */ + } + } else { slapi_log_error(SLAPI_LOG_FATAL, "Upgrade DN Format", - "Instance %s in %s is up-to-date\n", + "Failed to get DBVERSION (Instance name: %s, dir %s)\n", instance_name, workdbdir); - rc = 1; /* 1: up-to-date; 0: need upgrade; otherwise: error */ + rc = -1; /* error */ goto bail; } @@ -3681,7 +3710,7 @@ int ldbm_back_upgradednformat(Slapi_PBlock *pb) } *sep = '/'; if (((0 == rc) && !(ud_flags & SLAPI_DRYRUN)) || - ((rc > 0) && (ud_flags & SLAPI_DRYRUN))) { + ((rc == 0) && (ud_flags & SLAPI_DRYRUN))) { /* modify the DBVERSION files if the DN upgrade was successful OR * if DRYRUN, the backend instance is up-to-date. */ dbversion_write(li, workdbdir, NULL, DBVERSION_ALL); /* inst db dir */ diff --git a/ldap/servers/slapd/slapi-private.h b/ldap/servers/slapd/slapi-private.h index 937c2bb..375c33f 100644 --- a/ldap/servers/slapd/slapi-private.h +++ b/ldap/servers/slapd/slapi-private.h @@ -1223,6 +1223,7 @@ void DS_Sleep(PRIntervalTime ticks); generate entryrdn index */ #define SLAPI_UPGRADEDNFORMAT 0x8 /* specify this op is upgradednformat */ #define SLAPI_DRYRUN 0x10 /* dryrun mode for upgradednformat */ +#define SLAPI_UPGRADEDNFORMAT_V1 0x20 /* taking care multipe spaces */ /*

1 0

ldap/admin ldap/servers
by Noriko Hosoi 09 Apr '13

09 Apr '13

ldap/admin/src/scripts/80upgradednformat.pl | 206 ------- ldap/admin/src/scripts/80upgradednformat.pl.in | 307 ++++++++++ ldap/servers/slapd/back-ldbm/back-ldbm.h | 1 ldap/servers/slapd/back-ldbm/dbversion.c | 3 ldap/servers/slapd/back-ldbm/import-threads.c | 714 ++++++++++++++++++------- ldap/servers/slapd/back-ldbm/import.c | 103 ++- ldap/servers/slapd/back-ldbm/import.h | 23 ldap/servers/slapd/back-ldbm/ldif2ldbm.c | 43 + ldap/servers/slapd/slapi-private.h | 1 9 files changed, 974 insertions(+), 427 deletions(-) New commits: commit 69ff83598d517bed84922b1c7dd67cab023b4d99 Author: Noriko Hosoi <nhosoi(a)redhat.com> Date: Tue Apr 9 10:27:39 2013 -0700 Ticket #529 - dn normalization must handle multiple space characters in attributes Bug description: This is the second half of the fix for #529. The first half fixed the DN normalization which used to allow DNs where only the number of spaces are different. Now it is rejucted as expected. But it breaks the backward compatibility. Fix description: The upgrade script 80upgradednformat.pl called from "setup-ds.pl -u" checks the duplicated DNs and rename them if necessary. For instance, if there are 2 DNs: cn=test user0,dc=example,dc=com (entryid: N) cn=test user0,dc=example,dc=com (entryid: M) then the upgrade script/tool modifies the second one as follows: cn=test user0 M,dc=example,dc=com (entryid: M) and the original "cn: test user0" is kept in the attribute. The modified result is reported in "setup-ds.pl -u" as follows: !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Duplicated DN(s) were found and renamed. !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Renamed entry IDs are listed in /var/lib/dirsrv/slapd-ID/ldif/userRoot_conflict.txt. Contents of the conflict.txt: prinary entry ID: duplicated entry IDs 13:16 18 14:17 https://fedorahosted.org/389/ticket/529 Reviewed and commentted by Rich and Mark (Thanks!!) diff --git a/ldap/admin/src/scripts/80upgradednformat.pl b/ldap/admin/src/scripts/80upgradednformat.pl deleted file mode 100644 index 092b7cd..0000000 --- a/ldap/admin/src/scripts/80upgradednformat.pl +++ /dev/null @@ -1,206 +0,0 @@ -use Mozilla::LDAP::Conn; -use Mozilla::LDAP::Utils qw(normalizeDN); -use Mozilla::LDAP::API qw(:constant ldap_url_parse ldap_explode_dn); -use File::Basename; -use File::Copy; -use DSUpdate qw(isOffline); - -# Upgrade DN format if needed. -# For each backend instance, -# run upgradednformat with -N (dryrun mode), -# if it returns 0 (Upgrade candidates are found), -# recursively copy the instance dir to the work dir (dnupgrade) -# run upgradednformat w/o -N against the DB in the work dir -# if it went ok, replace the original instance dir with the work dir. -# Note: This script does nothing if the server is up. -sub runinst { - my ($inf, $inst, $dseldif, $conn) = @_; - - my @errs; - - my $config = "cn=config"; - my $config_entry = $conn->search($config, "base", "(cn=*)"); - if (!$config_entry) { - return ("error_no_configuration_entry", $!); - } - - # Check if the server is up or not - my $rc; - ($rc, @errs) = isOffline($inf, $inst, $conn); - if (!$rc) { - return @errs; - } - my $mappingtree = "cn=mapping tree,cn=config"; - my $ldbmbase = "cn=ldbm database,cn=plugins,cn=config"; - - my $backend_entry; - my $mtentry = $conn->search($mappingtree, "onelevel", "(cn=*)", 0, @attr); - if (!$mtentry) { - return ("error_no_mapping_tree_entries", $!); - } - - my $db_config_entry = - $conn->search("cn=config,cn=ldbm database,cn=plugins,cn=config", - "base", "(objectclass=*)"); - if (!$db_config_entry) { - return ('error_finding_config_entry', - 'cn=config,cn=ldbm database,cn=plugins,cn=config', - $conn->getErrorString()); - } - # If subtree rename swich is not found in the config file, - # set off to the switch and upgrade dn format using entrydn. - my $switch = $db_config_entry->getValues('nsslapd-subtree-rename-switch'); - if ("" eq $switch) { - $db_config_entry->addValue('nsslapd-subtree-rename-switch', "off"); - $conn->update($db_config_entry); - } - - # If a suffix in the mapping tree is doube-quoted and - # the cn value has only the double-quoted value, e.g. - # dn: cn="dc=example,dc=com",cn=mapping tree,cn=config - # cn: "dc=example,dc=com" - # the following code adds non-quoted value: - # cn: dc=example,dc=com - while ($mtentry) { - my $numvals = $mtentry->size("cn"); - my $i; - my $withquotes = -1; - my $noquotes = -1; - for ($i = 0; $i < $numvals; $i++) { - if ($mtentry->{"cn"}[$i] =~ /^".*"$/) { - $withquotes = $i; - } else { - $noquotes = $i; - } - } - if ($withquotes >= 0 && $noquotes == -1) { - # Has only cn: "<suffix>" - # Adding cn: <suffix> - my $stripped = $mtentry->{"cn"}[$withquotes]; - $stripped =~ s/^"(.*)"$/$1/; - $mtentry->addValue("cn", $stripped); - $conn->update($mtentry); - } - $mtentry = $conn->nextEntry(); - } - - my $instancedir = $config_entry->{"nsslapd-instancedir"}[0]; - my $upgradednformat = $instancedir . "/upgradednformat"; - - # Scan through all of the backends to see if any of them - # contain escape characters in the DNs. If we find any - # escapes, we need to run the conversion tool on that - # backend. - $backend_entry = $conn->search($ldbmbase, "onelevel", "(objectClass=nsBackendInstance)", 0, @attr); - if (!$backend_entry) { - return ("error_no_backend_entries", $!); - } - - while ($backend_entry) { - my $backend = $backend_entry->{"cn"}[0]; - my $dbinstdir = $backend_entry->{"nsslapd-directory"}[0]; - my $workdir = $dbinstdir . "/dnupgrade"; - my $dbdir = dirname($dbinstdir); - my $pdbdir = dirname($dbdir); - my $instname = basename($dbinstdir); - - if ("$dbdir" eq "" || "$instname" eq "") { - push @errs, ["error_invalid_dbinst_dir", $dbinstdir]; - return @errs; - } - - # clean up db region files, which might contain the old pages - if ( -d $dbdir && -f $dbdir."/__db.001") { - unlink <$dbdir/__db.*>; - } - - if (-e "$dbinstdir/id2entry.db4") { - # Check if any DNs contain escape characters with dbscan. - # dryrun mode - # return values: 0 -- need to upgrade dn format - # 1 -- no need to upgrade dn format - # -1 -- error - my $escapes = system("$upgradednformat -n $backend -a $dbinstdir -N"); - if (0 == $escapes) { - # need to upgrade dn format - my $rc = 0; - - if (system("cd $pdbdir; tar cf - db/DBVERSION | (cd $dbinstdir; tar xf -)") || - system("cd $pdbdir; tar cf - db/$instname/DBVERSION | (cd $dbinstdir; tar xf -)") || - system("cd $pdbdir; tar cf - db/$instname/*.db4 | (cd $dbinstdir; tar xf -)")) { - push @errs, ["error_cant_backup_db", $backend, $!]; - return @errs; - } - my @stat = stat("$dbdir"); - my $mode = $stat[2]; - my $uid = $stat[4]; - my $gid = $stat[5]; - - move("$dbinstdir/db", "$workdir"); - chmod($mode, $workdir); - chown($uid, $gid, $workdir); - - @stat = stat("$dbinstdir"); - $mode = $stat[2]; - $uid = $stat[4]; - $gid = $stat[5]; - - chmod($mode, "$workdir/$instname"); - chown($uid, $gid, "$workdir/$instname"); - - # call conversion tool here and get return status. - $rc = system("$upgradednformat -n $backend -a $workdir/$instname"); - if ($rc == 0) { # success - move("$dbinstdir", "$dbinstdir.orig"); - move("$dbinstdir.orig/dnupgrade/$instname", "$dbinstdir"); - copy("$dbinstdir.orig/dnupgrade/DBVERSION", "$dbdir"); - } else { - # Conversion failed. Cleanup and bail. - unlink <$dbinstdir/dnupgrade/$backend/*>; - rmdir("$dbinstdir/dnupgrade/$backend"); - unlink <$dbinstdir/dnupgrade/*>; - rmdir("$dbinstdir/dnupgrade"); - return ("error_cant_convert_db", $backend, $rc); - } - } else { - # already upgraded or an error occurred. - # check ancestorid to see if it has not-sorted ID list or not. - my $ancestorid = $dbinstdir . "/ancestorid.db4"; - if (-e "$ancestorid") { - my $disorder = 0; - open(ANCESTOR, "/usr/bin/dbscan -f $ancestorid -r |"); - while (<ANCESTOR>) { - if (!/^=[0-9]*/) { - chomp($_); - my @IDs = split(/ | /, $_); - # print "ID count: $#IDs\n"; - my $lasti = $#IDs; - for (my $i = 1; $i < $lasti; $i++) { - if ($IDs[$i] >= $IDs[$i + 1]) { - $disorder = 1; - last; - } - } - # print "Result: $disorder \n"; - if ($disorder) { - last; - } - } - } - close(ANCESTOR); - - # ancestorid index is in disorder; need to reindex it. - if ($disorder) { - print "The ancestorid index in $backend is in disorder; Reindexing $ancestorid.\n"; - my $reindex = $instancedir . "/db2index"; - my $rc = system("$reindex -n $backend -t ancestorid"); - } - } - } - } - - $backend_entry = $conn->nextEntry(); - } - - return (); -} diff --git a/ldap/admin/src/scripts/80upgradednformat.pl.in b/ldap/admin/src/scripts/80upgradednformat.pl.in new file mode 100644 index 0000000..6d14099 --- /dev/null +++ b/ldap/admin/src/scripts/80upgradednformat.pl.in @@ -0,0 +1,307 @@ +use Mozilla::LDAP::Conn; +use Mozilla::LDAP::Utils qw(normalizeDN); +use Mozilla::LDAP::API qw(:constant ldap_url_parse ldap_explode_dn); +use File::Basename; +use File::Copy; +use DSUpdate qw(isOffline); + +# Upgrade DN format if needed. +# For each backend instance, +# run upgradednformat with -N (dryrun mode), +# if it returns 0 (Upgrade candidates are found), +# recursively copy the instance dir to the work dir (dnupgrade) +# run upgradednformat w/o -N against the DB in the work dir +# if it went ok, replace the original instance dir with the work dir. +# Note: This script does nothing if the server is up. +sub runinst { + my ($inf, $inst, $dseldif, $conn) = @_; + + my @errs; + + my $config = "cn=config"; + my $config_entry = $conn->search($config, "base", "(cn=*)"); + if (!$config_entry) { + return ("error_no_configuration_entry", $!); + } + + # Check if the server is up or not + my $rc; + ($rc, @errs) = isOffline($inf, $inst, $conn); + if (!$rc) { + return @errs; + } + my $mappingtree = "cn=mapping tree,cn=config"; + my $ldbmbase = "cn=ldbm database,cn=plugins,cn=config"; + + my $backend_entry; + my $mtentry = $conn->search($mappingtree, "onelevel", "(cn=*)", 0, @attr); + if (!$mtentry) { + return ("error_no_mapping_tree_entries", $!); + } + + my $db_config_entry = + $conn->search("cn=config,cn=ldbm database,cn=plugins,cn=config", + "base", "(objectclass=*)"); + if (!$db_config_entry) { + return ('error_finding_config_entry', + 'cn=config,cn=ldbm database,cn=plugins,cn=config', + $conn->getErrorString()); + } + # If subtree rename swich is not found in the config file, + # set off to the switch and upgrade dn format using entrydn. + my $switch = $db_config_entry->getValues('nsslapd-subtree-rename-switch'); + if ("" eq $switch) { + $db_config_entry->addValue('nsslapd-subtree-rename-switch', "off"); + $conn->update($db_config_entry); + } + + # If a suffix in the mapping tree is doube-quoted and + # the cn value has only the double-quoted value, e.g. + # dn: cn="dc=example,dc=com",cn=mapping tree,cn=config + # cn: "dc=example,dc=com" + # the following code adds non-quoted value: + # cn: dc=example,dc=com + while ($mtentry) { + my $numvals = $mtentry->size("cn"); + my $i; + my $withquotes = -1; + my $noquotes = -1; + for ($i = 0; $i < $numvals; $i++) { + if ($mtentry->{"cn"}[$i] =~ /^".*"$/) { + $withquotes = $i; + } else { + $noquotes = $i; + } + } + if ($withquotes >= 0 && $noquotes == -1) { + # Has only cn: "<suffix>" + # Adding cn: <suffix> + my $stripped = $mtentry->{"cn"}[$withquotes]; + $stripped =~ s/^"(.*)"$/$1/; + $mtentry->addValue("cn", $stripped); + $conn->update($mtentry); + } + $mtentry = $conn->nextEntry(); + } + + my $ldifdir = $config_entry->{"nsslapd-ldifdir"}[0]; + my $instancedir = $config_entry->{"nsslapd-instancedir"}[0]; + my ($slapd, $serverID) = split(/-/, $instancedir); + my $upgradednformat = "@sbindir@/upgradednformat"; + my $reindex = "@sbindir@/db2index"; + + # Scan through all of the backends to see if any of them + # contain escape characters in the DNs. If we find any + # escapes, we need to run the conversion tool on that + # backend. + $backend_entry = $conn->search($ldbmbase, "onelevel", "(objectClass=nsBackendInstance)", 0, @attr); + if (!$backend_entry) { + return ("error_no_backend_entries", $!); + } + + while ($backend_entry) { + my $backend = $backend_entry->{"cn"}[0]; + my $dbinstdir = $backend_entry->{"nsslapd-directory"}[0]; + my $workdir = $dbinstdir . "/dnupgrade"; + my $dbdir = dirname($dbinstdir); + my $pdbdir = dirname($dbdir); + my $instname = basename($dbinstdir); + my $dn_norm_sp_txt = $ldifdir . "/" . $instname . "_dn_norm_sp.txt"; + + if ("$dbdir" eq "" || "$instname" eq "") { + push @errs, ["error_invalid_dbinst_dir", $dbinstdir]; + return @errs; + } + + # clean up db region files, which might contain the old pages + if ( -d $dbdir && -f $dbdir."/__db.001") { + unlink <$dbdir/__db.*>; + } + + my $escapes = 0; + my $rc = 0; + my $cmd = 0; + if ((-e "$dbinstdir/id2entry.db") || (-e "$dbinstdir/id2entry.db4")) { + # Check if any DNs contain escape characters with dbscan. + # dryrun mode + # return values: 0 -- need to upgrade dn format + # 1 -- no need to upgrade dn format + # -1 -- error + $cmd = "$upgradednformat -n $backend -a $dbinstdir -N"; + $rc = system("$cmd"); + if ($rc & 127) { + push @errs, [ 'error_running_command', $cmd, $rc, $! ]; + return @errs; + } + $escapes = $rc >> 8; +# $escapes == 0 ==> no need to do dn upgrade +# $escapes == 1 ==> need to do dn upgrade (both dn upgrade and spaces) +# $escapes == 2 ==> need to do dn upgrade (dn upgrade only) +# $escapes == 3 ==> need to do dn upgrade (dn upgrade spaces only) + if ($escapes <= 0) { + # already upgraded or an error occurred. + # check ancestorid to see if it has not-sorted ID list or not. + my $ancestorid = $dbinstdir . "/ancestorid.db4"; + if (!(-e "$ancestorid")) { + $ancestorid = $dbinstdir . "/ancestorid.db"; + } + if (-e "$ancestorid") { + my $disorder = 0; + open(ANCESTOR, "/usr/bin/dbscan -f $ancestorid -r |"); + while (<ANCESTOR>) { + if (!/^=[0-9]*/) { + chomp($_); + my @IDs = split(/ | /, $_); + # print "ID count: $#IDs\n"; + my $lasti = $#IDs; + for (my $i = 1; $i < $lasti; $i++) { + if ($IDs[$i] >= $IDs[$i + 1]) { + $disorder = 1; + last; + } + } + # print "Result: $disorder \n"; + if ($disorder) { + last; + } + } + } + close(ANCESTOR); + + # ancestorid index is in disorder; need to reindex it. + if ($disorder) { + print "The ancestorid index in $backend is in disorder; Reindexing $ancestorid.\n"; + $cmd = "$reindex -Z $serverID -n $backend -t ancestorid"; + $rc = system("$cmd"); + if ($rc & 127) { + push @errs, [ 'error_running_command', $cmd, $rc, $! ]; + return @errs; + } + } + } + } else { + # need to upgrade dn format + $rc = 0; + + if (system("cd $pdbdir; tar cf - db/DBVERSION | (cd $dbinstdir; tar xf -)") || + system("cd $pdbdir; tar cf - db/$instname/DBVERSION | (cd $dbinstdir; tar xf -)") || + system("cd $pdbdir; tar cf - db/$instname/*.db* | (cd $dbinstdir; tar xf -)")) { + push @errs, [ "error_cant_backup_db", $backend, $! ]; + return @errs; + } + my @stat = stat("$dbdir"); + my $mode = $stat[2]; + my $uid = $stat[4]; + my $gid = $stat[5]; + + move("$dbinstdir/db", "$workdir"); + chmod($mode, $workdir); + chown($uid, $gid, $workdir); + + @stat = stat("$dbinstdir"); + $mode = $stat[2]; + $uid = $stat[4]; + $gid = $stat[5]; + + chmod($mode, "$workdir/$instname"); + chown($uid, $gid, "$workdir/$instname"); + + my $do_dn_norm_sp = 0; + if ((1 == $escapes) || (3 == $escapes)) { + # We are taking care of spaces in DN. + my $sorted = $dn_norm_sp_txt . ".sorted"; + $cmd = "sort $dn_norm_sp_txt"; + $rc = system("$cmd > $sorted"); + if ($rc) { + debug(1, "Error: $cmd failed - output $sorted: $!\n"); + push @errs, [ 'error_running_command', $cmd, $rc, $! ]; + return @errs; + } + # Create a work file from $sorted. + my $workfile = $sorted . ".work"; + # print "$sorted -> $workfile\n"; + + open(SORTEDFILE, "$sorted"); + open(WORKFILE, "> $workfile"); + my $prev_dn = ""; + my $prev_id = 0; + my $new = 1; + for (my $line = <SORTEDFILE>; $line; $line = <SORTEDFILE>) { + chomp($line); + my ($dn, $id) = split(":", $line, 2); + if ($dn eq $prev_dn) { + if ($new == 1) { + print WORKFILE "$prev_id:$id"; + $new = 0; + $do_dn_norm_sp = 1; # go ahead and fix it. + } else { + print WORKFILE " $id"; + } + } else { + if (0 == $new) { + print WORKFILE "\n"; + $new = 1; + } + } + $prev_dn = $dn; + $prev_id = $id; + } + close(WORKFILE); + close(SORTEDFILE); + unlink <$sorted>; + move("$dn_norm_sp_txt", "$dn_norm_sp_txt.orig"); + if ($do_dn_norm_sp) { + move("$workfile", "$dn_norm_sp_txt"); + # otherwise, we don't need $dn_norm_sp_txt + } + } else { + unlink <$dn_norm_sp_txt>; + } + + if ((1 == $escapes) || (2 == $escapes) || (3 == $escapes)) { + # call conversion tool here and get return status. + $cmd = "$upgradednformat -Z $serverID -n $backend -a $workdir/$instname"; + $rc = system("$cmd"); + if ($rc & 127) { + push @errs, [ 'error_running_command', $cmd, $rc, $! ]; + return @errs; + } + $escapes = $rc >> 8; + } + if ((0 == $rc) || (1 == $escapes) || (3 == $escapes)) { + # success + move("$dbinstdir", "$dbinstdir.orig"); + move("$dbinstdir.orig/dnupgrade/$instname", "$dbinstdir"); + copy("$dbinstdir.orig/dnupgrade/DBVERSION", "$dbdir"); + if ((1 == $escapes) || (3 == $escapes)) { + $cmd = "$reindex -Z $serverID -n $backend -t entryrdn"; + $rc = system("$cmd"); + if ($rc & 127) { + push @errs, [ 'error_running_command', $cmd, $rc, $! ]; + return @errs; + } + } + my $dn_norm_sp_txt = $ldifdir . "/" . $instname . "_dn_norm_sp.txt"; + my $conflict = $ldifdir . "/" . $instname . "_conflict.txt"; + system("echo prinary entry ID: duplicated entry IDs > $conflict"); + system("cat $dn_norm_sp_txt >> $conflict"); + print "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"; + print "Duplicated DN(s) were found and renamed.\n"; + print "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"; + print "Renamed entry IDs are listed in $conflict.\n"; + } else { + # Conversion failed. Cleanup and bail. + unlink <$dbinstdir/dnupgrade/$backend/*>; + rmdir("$dbinstdir/dnupgrade/$backend"); + unlink <$dbinstdir/dnupgrade/*>; + rmdir("$dbinstdir/dnupgrade"); + return ("error_cant_convert_db", $backend, $rc); + } + } + } + + $backend_entry = $conn->nextEntry(); + } + + return (); +} diff --git a/ldap/servers/slapd/back-ldbm/back-ldbm.h b/ldap/servers/slapd/back-ldbm/back-ldbm.h index 726ac40..871d489 100644 --- a/ldap/servers/slapd/back-ldbm/back-ldbm.h +++ b/ldap/servers/slapd/back-ldbm/back-ldbm.h @@ -177,6 +177,7 @@ typedef unsigned short u_int16_t; #define BDB_RDNFORMAT "rdn-format" /* Subtree rename enabled */ #define BDB_RDNFORMAT_VERSION "2" /* rdn-format version (by default, 0) */ #define BDB_DNFORMAT "dn-4514" /* DN format RFC 4514 compliant */ +#define BDB_DNFORMAT_VERSION "1" /* DN format version */ #define DBVERSION_NEWIDL 0x1 #define DBVERSION_RDNFORMAT 0x2 diff --git a/ldap/servers/slapd/back-ldbm/dbversion.c b/ldap/servers/slapd/back-ldbm/dbversion.c index 32a60e5..e4eee32 100644 --- a/ldap/servers/slapd/back-ldbm/dbversion.c +++ b/ldap/servers/slapd/back-ldbm/dbversion.c @@ -119,7 +119,8 @@ dbversion_write(struct ldbminfo *li, const char *directory, ptr = buf + len; } if (flags & DBVERSION_DNFORMAT) { - PR_snprintf(ptr, sizeof(buf) - len, "/%s", BDB_DNFORMAT); + PR_snprintf(ptr, sizeof(buf) - len, "/%s-%s", + BDB_DNFORMAT, BDB_DNFORMAT_VERSION); len = strlen(buf); ptr = buf + len; } diff --git a/ldap/servers/slapd/back-ldbm/import-threads.c b/ldap/servers/slapd/back-ldbm/import-threads.c index 54f8159..1c526ef 100644 --- a/ldap/servers/slapd/back-ldbm/import-threads.c +++ b/ldap/servers/slapd/back-ldbm/import-threads.c @@ -1241,6 +1241,134 @@ upgradedn_add_to_list(struct upgradedn_attr **ud_list, } /* + * Return value: count of max consecutive spaces + */ +static int +has_spaces(const char *str) +{ + char *p = (char *)str; + char *np; + char *endp = p + strlen(str); + int wcnt; + int maxwcnt = 0; +next: + if ((np = strchr(p, ' ')) || (np = strchr(p, '\t'))) { + wcnt = 0; + while ((np < endp) && isspace(*np)) { + wcnt++; + np++; + } + if (maxwcnt < wcnt) { + maxwcnt = wcnt; + } + p = np; + goto next; + } else { + goto bail; + } +bail: + return maxwcnt; +} + +static int +add_IDs_to_IDarray(ID ***dn_norm_sp_conflict, int *max, int i, char *strids) +{ + char *p, *next, *start; + ID my_id; + ID **conflict; + ID *idp; + ID *endp; + size_t len; + + if ((NULL == dn_norm_sp_conflict) || (NULL == max) || (0 == *max)) { + return 1; + } + p = PL_strchr(strids, ':'); + if (NULL == p) { + LDAPDebug1Arg(LDAP_DEBUG_ANY, + "Format error: no ':' in %s\n", strids); + return 1; + } + *p = '\0'; + my_id = (ID)strtol(strids, (char **)NULL, 10); + if (!my_id) { + LDAPDebug1Arg(LDAP_DEBUG_ANY, "Invalid ID in %s\n", strids); + return 1; + } + + if (NULL == *dn_norm_sp_conflict) { + *dn_norm_sp_conflict = (ID **)slapi_ch_malloc(sizeof(ID *) * *max); + } else if (*max == i + 1) { + *max *= 2; + *dn_norm_sp_conflict = (ID **)slapi_ch_realloc((char *)*dn_norm_sp_conflict, + sizeof(ID *) * *max); + } + conflict = *dn_norm_sp_conflict; + + len = strlen(strids); + while (isspace(*(++p))); + start = p; + /* since the format is "ID ID ID ...", max count is len/2 + 1 and +1 for my_id */ + conflict[i] = (ID *)slapi_ch_calloc((len/2 + 2), sizeof(ID)); + idp = conflict[i]; + endp = idp + len/2 + 2; + *idp++ = my_id; + for (p = strtok_r(start, " \n", &next); p && (idp < endp); + p = strtok_r(NULL, " \n", &next), idp++) { + *idp = (ID)strtol(p, (char **)NULL, 10); + } + if (idp < endp) { + *idp = 0; + } else if (idp == endp) { + conflict[i] = (ID *)slapi_ch_realloc((char *)conflict[i], + (len/2 + 3) * sizeof(ID)); + idp = conflict[i]; + *(idp + len/2 + 2) = 0; + } + conflict[i+1] = NULL; + return 0; +} + +static void +free_IDarray(ID ***dn_norm_sp_conflict) +{ + int i; + if ((NULL == dn_norm_sp_conflict) || (NULL == *dn_norm_sp_conflict)) { + return; + } + for (i = 0; (*dn_norm_sp_conflict)[i]; i++) { + slapi_ch_free((void **)&(*dn_norm_sp_conflict)[i]); + } + slapi_ch_free((void **)dn_norm_sp_conflict); +} + +/* + * dn_norm_sp_conflicts is a double array of IDs. + * Format: + * primary_ID0: conflict_ID ... + * primary_ID1: conflict_ID ... + * ... + * + * is_conflict_ID looks for ghe given id in hte conflict_ID lists + * If found, its primary_ID is returned. + * Otherwise, 0 is returned. + */ +static ID +is_conflict_ID(ID **dn_norm_sp_conflicts, int max, ID id) +{ + int i; + ID *idp; + for (i = 0; i < max; i++) { + for (idp = dn_norm_sp_conflicts[i]; idp && *idp; idp++) { + if (*idp == id) { + return *dn_norm_sp_conflicts[i]; + } + } + } + return 0; +} + +/* * Producer thread for upgrading dn format * FLAG_UPGRADEDNFORMAT | FLAG_DRYRUN -- check the necessity of dn upgrade * FLAG_UPGRADEDNFORMAT -- execute dn upgrade @@ -1263,6 +1391,41 @@ upgradedn_add_to_list(struct upgradedn_attr **ud_list, * from ABORTED (aborted or error) and FINISHED (scan all the entries * and found no candidate to upgrade) */ +/* + * FLAG_UPGRADEDNFORMAT: need to take care space reduction + * dn: cn=TEST USER0 --> dn: cn=TEST USER0 + * If multiple DNs are found having different count of spaces, + * remove the second and later with exporting them into ldif file. + * In the dry run, it retrieves all the entries having spaces in dn and + * store the DN and OID in a file. Mark, conflicts if any. + * In the real run, store only no conflict entries and first conflicted one. + * The rest are stored in an ldif file. + * + * Cases: + * FLAG_DRYRUN | FLAG_UPGRADEDNFORMAT | FLAG_UPGRADEDNFORMAT_V1 + * ==> 1 & 2-1 + * FLAG_DRYRUN | FLAG_UPGRADEDNFORMAT_V1 + * ==> 2-1 + * FLAG_UPGRADEDNFORMAT | FLAG_UPGRADEDNFORMAT_V1 + * ==> 1 & 2-2,3 + * FLAG_UPGRADEDNFORMAT_V1 + * ==> 2-2,3 + * + * 1) dn normalization + * 2) space handling + * 2-1) upgradedn dry-run: output into a file (in the instance db dir?) + * format: + * <original dn>:OID + * 2-2) upgrade script sorts the file; retrieve duplicated DNs + * <dn>:OID0 + * <dupped dn>:OID1 + * <dupped dn>:OID2 + * + * format: + * OID0:OID1 OID2 ... + * 2-3) upgradedn: compare the OID with the OID1,OID2, + * if match, rename the rdn value to "value <entryid>". + */ void upgradedn_producer(void *param) { @@ -1279,8 +1442,6 @@ upgradedn_producer(void *param) Slapi_Attr *a = NULL; Slapi_DN *sdn = NULL; char *workdn = NULL; - int doit = 0; - int skipit = 0; struct upgradedn_attr *ud_list = NULL; char **ud_vals = NULL; char **ud_valp = NULL; @@ -1288,6 +1449,18 @@ upgradedn_producer(void *param) Slapi_Attr *ud_attr = NULL; char *ecopy = NULL; const char *normdn = NULL; + char *rdn = NULL; /* original rdn */ + int is_dryrun = 0; /* FLAG_DRYRUN */ + int chk_dn_norm = 0; /* FLAG_UPGRADEDNFORMAT */ + int chk_dn_norm_sp = 0; /* FLAG_UPGRADEDNFORMAT_V1 */ + ID **dn_norm_sp_conflicts = NULL; + int do_dn_norm = 0; /* do dn_norm */ + int do_dn_norm_sp = 0; /* do dn_norm_sp */ + int rdn_has_spaces = 0; + int info_state = 0; /* state to add to info->state (for dryrun only) */ + int skipit = 0; + ID pid; + struct backdn *bdn = NULL; /* vars for Berkeley DB */ DB_ENV *env = NULL; @@ -1304,9 +1477,21 @@ upgradedn_producer(void *param) PR_ASSERT(info != NULL); PR_ASSERT(inst != NULL); PR_ASSERT(be != NULL); - - if ( job->flags & FLAG_ABORT ) + + if ( job->flags & FLAG_ABORT ) { goto error; + } + + is_dryrun = job->flags & FLAG_DRYRUN; + chk_dn_norm = job->flags & FLAG_UPGRADEDNFORMAT; + chk_dn_norm_sp = job->flags & FLAG_UPGRADEDNFORMAT_V1; + + if (!chk_dn_norm && !chk_dn_norm_sp) { + /* Nothing to do... */ + LDAPDebug0Args(LDAP_DEBUG_ANY, "UpgradeDnFormat is not required.\n"); + info->state = FINISHED; + goto done; + } sleeptime = PR_MillisecondsToInterval(import_sleep_time); @@ -1320,15 +1505,14 @@ upgradedn_producer(void *param) /* open id2entry with dedicated db env and db handler */ if ( dblayer_get_aux_id2entry( be, &db, &env, NULL ) != 0 || db == NULL || env == NULL) { - LDAPDebug( LDAP_DEBUG_ANY, "Could not open id2entry\n", 0, 0, 0 ); + LDAPDebug0Args(LDAP_DEBUG_ANY, "Could not open id2entry\n"); goto error; } /* get a cursor to we can walk over the table */ db_rval = db->cursor(db, NULL, &dbc, 0); if ( db_rval || !dbc ) { - LDAPDebug( LDAP_DEBUG_ANY, - "Failed to get cursor for reindexing\n", 0, 0, 0 ); + LDAPDebug0Args(LDAP_DEBUG_ANY, "Failed to get cursor for reindexing\n"); dblayer_release_id2entry(be, db); goto error; } @@ -1352,18 +1536,22 @@ upgradedn_producer(void *param) slapi_ch_free(&(data.data)); key.flags = DB_DBT_MALLOC; data.flags = DB_DBT_MALLOC; - if (isfirst) - { + if (isfirst) { db_rval = dbc->c_get(dbc, &key, &data, DB_FIRST); isfirst = 0; - } - else - { + } else { db_rval = dbc->c_get(dbc, &key, &data, DB_NEXT); } if (0 != db_rval) { - if (DB_NOTFOUND != db_rval) { + if (DB_NOTFOUND == db_rval) { + LDAPDebug1Arg(LDAP_DEBUG_ANY, "%s: Finished to read database\n", + inst->inst_name); + if (job->task) { + slapi_task_log_notice(job->task, + "%s: Finished to read database", inst->inst_name); + } + } else { LDAPDebug(LDAP_DEBUG_ANY, "%s: Failed to read database, " "errno=%d (%s)\n", inst->inst_name, db_rval, dblayer_strerror(db_rval)); @@ -1389,19 +1577,19 @@ upgradedn_producer(void *param) memcpy(ecopy, data.dptr, data.dsize); *(ecopy + data.dsize) = '\0'; normdn = NULL; - doit = 0; + do_dn_norm = 0; + do_dn_norm_sp = 0; + rdn_has_spaces = 0; if (entryrdn_get_switch()) { - char *rdn = NULL; - /* rdn is allocated in get_value_from_string */ + /* original rdn is allocated in get_value_from_string */ rc = get_value_from_string((const char *)data.dptr, "rdn", &rdn); if (rc) { /* data.dptr may not include rdn: ..., try "dn: ..." */ e = slapi_str2entry(data.dptr, SLAPI_STR2ENTRY_USE_OBSOLETE_DNFORMAT); } else { - struct backdn *bdn = - dncache_find_id(&inst->inst_dncache, temp_id); + bdn = dncache_find_id(&inst->inst_dncache, temp_id); if (bdn) { /* don't free dn */ normdn = (char *)slapi_sdn_get_dn(bdn->dn_sdn); @@ -1425,7 +1613,7 @@ upgradedn_producer(void *param) if (rc) { rc = 0; /* assume this is a suffix */ } else { - ID pid = (ID)strtol(pid_str, (char **)NULL, 10); + pid = (ID)strtol(pid_str, (char **)NULL, 10); slapi_ch_free_string(&pid_str); /* if pid is larger than the current pid temp_id, * the parent entry hasn't */ @@ -1433,7 +1621,7 @@ upgradedn_producer(void *param) pid, &id, &psrdn, &curr_entry); if (rc) { LDAPDebug2Args( LDAP_DEBUG_ANY, - "ldbm2index: Failed to compose dn for " + "uptradedn: Failed to compose dn for " "(rdn: %s, ID: %d)\n", rdn, temp_id); slapi_ch_free_string(&rdn); slapi_rdn_done(&psrdn); @@ -1444,7 +1632,7 @@ upgradedn_producer(void *param) slapi_rdn_done(&psrdn); if (rc) { LDAPDebug2Args( LDAP_DEBUG_ANY, - "ldbm2index: Failed to compose dn for " + "uptradedn: Failed to compose dn for " "(rdn: %s, ID: %d) from Slapi_RDN\n", rdn, temp_id); slapi_ch_free_string(&rdn); @@ -1455,23 +1643,34 @@ upgradedn_producer(void *param) rdn, pdn?",":"", pdn?pdn:""); slapi_ch_free_string(&pdn); } - /* dn is not dup'ed in slapi_sdn_new_dn_byref. - * It's set to bdn and put in the dn cache. */ - sdn = slapi_sdn_new_normdn_byref(normdn); - bdn = backdn_init(sdn, temp_id, 0); - CACHE_ADD( &inst->inst_dncache, bdn, NULL ); - CACHE_RETURN(&inst->inst_dncache, &bdn); - slapi_log_error(SLAPI_LOG_CACHE, "ldbm2index", - "entryrdn_lookup_dn returned: %s, " - "and set to dn cache\n", normdn); + if (is_dryrun) { + /* if not dryrun, we may change the DN, In such case, + * we need to put the new value to cache.*/ + /* dn is dup'ed in slapi_sdn_new_dn_byval. + * It's set to bdn and put in the dn cache. */ + sdn = slapi_sdn_new_normdn_byref(normdn); + bdn = backdn_init(sdn, temp_id, 0); + CACHE_ADD( &inst->inst_dncache, bdn, NULL ); + CACHE_RETURN(&inst->inst_dncache, &bdn); + normdn = slapi_sdn_get_dn(sdn); + slapi_log_error(SLAPI_LOG_CACHE, "uptradedn", + "entryrdn_lookup_dn returned: %s, " + "and set to dn cache\n", normdn); + } } e = slapi_str2entry_ext(normdn, NULL, data.dptr, SLAPI_STR2ENTRY_USE_OBSOLETE_DNFORMAT); - slapi_ch_free_string(&rdn); } } else { e = slapi_str2entry(data.data, SLAPI_STR2ENTRY_USE_OBSOLETE_DNFORMAT); + rdn = slapi_ch_strdup(slapi_entry_get_rdn_const(e)); + if (NULL == rdn) { + Slapi_RDN srdn; + slapi_rdn_init_dn(&srdn, slapi_entry_get_dn_const(e)); + rdn = (char *)slapi_rdn_get_rdn(&srdn); /* rdn is allocated in + * slapi_rdn_init_dn */ + } } if ( NULL == e ) { if (job->task) { @@ -1485,6 +1684,7 @@ upgradedn_producer(void *param) continue; } + /* From here, e != NULL */ /* * treat dn specially since the entry was generated with the flag * SLAPI_STR2ENTRY_USE_OBSOLETE_DNFORMAT @@ -1499,24 +1699,154 @@ upgradedn_producer(void *param) normdn = slapi_sdn_get_dn(&(e->e_sdn)); } - /* From here, e != NULL */ + /* + * If dryrun && check_dn_norm_sp, + * check if rdn's containing multi spaces exist or not. + * If any, output the DN:ID into a file INST_dn_norm_sp.txt in the + * ldifdir. We have to continue checking all the entries. + */ + if (chk_dn_norm_sp) { + char *dn_id; + char *path; /* <ldifdir>/<inst>_dn_norm_sp.txt is used for + the temp work file */ + /* open "path" once, and set FILE* to upgradefd */ + if (NULL == job->upgradefd) { + char *ldifdir = config_get_ldifdir(); + if (ldifdir) { + path = slapi_ch_smprintf("%s/%s_dn_norm_sp.txt", + ldifdir, inst->inst_name); + slapi_ch_free_string(&ldifdir); + } else { + path = slapi_ch_smprintf("/var/tmp/%s_dn_norm_sp.txt", + inst->inst_name); + } + if (is_dryrun) { + job->upgradefd = fopen(path, "w"); + if (NULL == job->upgradefd) { + if (job->task) { + slapi_task_log_notice(job->task, + "%s: No DNs to fix.\n", inst->inst_name); + } + LDAPDebug1Arg(LDAP_DEBUG_ANY, + "%s: No DNs to fix.\n", inst->inst_name); + slapi_ch_free_string(&path); + goto bail; + } + } else { + job->upgradefd = fopen(path, "r"); + if (NULL == job->upgradefd) { + if (job->task) { + slapi_task_log_notice(job->task, + "%s: Error: failed to open a file \"%s\"", + inst->inst_name, path); + } + LDAPDebug2Args(LDAP_DEBUG_ANY, + "%s: Error: failed to open a file \"%s\"\n", + inst->inst_name, path); + slapi_ch_free_string(&path); + goto error; + } + } + } + slapi_ch_free_string(&path); + if (is_dryrun) { + rdn_has_spaces = has_spaces(rdn); + if (rdn_has_spaces > 0) { + dn_id = slapi_ch_smprintf("%s:%u\n", + slapi_entry_get_dn_const(e), temp_id); + if (EOF == fputs(dn_id, job->upgradefd)) { + if (job->task) { + slapi_task_log_notice(job->task, + "%s: Error: failed to write a line \"%s\"", + inst->inst_name, dn_id); + } + LDAPDebug2Args(LDAP_DEBUG_ANY, + "%s: Error: failed to write a line \"%s\"", + inst->inst_name, dn_id); + slapi_ch_free_string(&dn_id); + goto error; + } + slapi_ch_free_string(&dn_id); + if (rdn_has_spaces > 1) { + /* If an rdn containing multi spaces exists, + * let's check the conflict. */ + do_dn_norm_sp = 1; + } + } + } else { /* !is_dryrun */ + /* check the oid and parentid. */ + /* Set conflict list once, and refer it laster. */ + static int my_idx = 0; + ID alt_id; + if (NULL == dn_norm_sp_conflicts) { + char buf[BUFSIZ]; + char *p; + int my_max = 8; + while (fgets(buf, BUFSIZ-1, job->upgradefd)) { + /* search "OID0: OID1 OID2 ... */ + if (!isdigit(*buf) || !(p = PL_strchr(buf, ':'))) { + continue; + } + if (add_IDs_to_IDarray(&dn_norm_sp_conflicts, &my_max, + my_idx, buf)) { + LDAPDebug1Arg(LDAP_DEBUG_ANY, + "Failed to set IDs %s to conflict list\n", + buf); + goto error; + } + my_idx++; + } + } + alt_id = is_conflict_ID(dn_norm_sp_conflicts, my_idx, temp_id); + if (alt_id) { + char *parentdn = NULL; + char *newrdn = NULL; + if (alt_id != temp_id) { + parentdn = slapi_dn_parent(normdn); + newrdn = slapi_create_dn_string("%s %u", rdn, temp_id); + /* This entry is a conflict of alt_id */ + LDAPDebug(LDAP_DEBUG_ANY, + "Entry %s (%lu) is a conflict of (%lu)\n", + normdn, temp_id, alt_id); + LDAPDebug2Args(LDAP_DEBUG_ANY, "Renaming \"%s\" to \"%s\"\n", + rdn, newrdn); + normdn = slapi_ch_smprintf("%s,%s", newrdn, parentdn); + /* Reset DN and RDN in the entry */ + slapi_sdn_done(&(e->e_sdn)); + slapi_sdn_init_normdn_passin(&(e->e_sdn), normdn); + /* normdn = slapi_sdn_get_dn(&(e->e_sdn)); */ + } + info_state |= DN_NORM_SP; + upgradedn_add_to_list(&ud_list, + slapi_ch_strdup(LDBM_ENTRYRDN_STR), + slapi_ch_strdup(rdn), 0); + rc = slapi_entry_add_rdn_values(e); + if (rc) { + LDAPDebug(LDAP_DEBUG_ANY, "%s: Failed to add rdn values" + " to an entry: %s (id %lu)\n", + inst->inst_name, normdn, (u_long)temp_id); + goto error; + } + } /* !alt_id */ + } /* !is_dryrun */ + } /* if (chk_dn_norm_sp) */ + + /* dn is dup'ed in slapi_sdn_new_dn_byval. + * It's set to bdn and put in the dn cache. */ + /* Waited to put normdn into dncache until it could be modified in + * chk_dn_norm_sp. */ + sdn = slapi_sdn_new_normdn_byref(normdn); + bdn = backdn_init(sdn, temp_id, 0); + CACHE_ADD(&inst->inst_dncache, bdn, NULL); + CACHE_RETURN(&inst->inst_dncache, &bdn); + slapi_log_error(SLAPI_LOG_CACHE, "uptradedn", + "set dn %s to dn cache\n", normdn); /* Check DN syntax attr values if it contains '\\' or not */ /* Start from the rdn */ - if (entryrdn_get_switch()) { /* subtree-rename: on */ - char *rdn = NULL; - size_t rdnlen = 0; + if (chk_dn_norm) { char *endrdn = NULL; char *rdnp = NULL; - rc = get_value_from_string((const char *)ecopy, "rdn", &rdn); - if (rc || (NULL == rdn)) { - LDAPDebug2Args(LDAP_DEBUG_ANY, - "%s: WARNING: skipping an entry with no RDN (id %lu)\n", - inst->inst_name, (u_long)temp_id); - slapi_entry_free(e); e = NULL; - continue; - } - rdnlen = strlen(rdn); - endrdn = rdn + rdnlen - 1; + endrdn = rdn + strlen(rdn) - 1; rdnp = PL_strchr(rdn, '='); if (NULL == rdnp) { @@ -1528,24 +1858,23 @@ upgradedn_producer(void *param) continue; } /* rdn contains '\\'. We have to update the value */ - if (PL_strchr(rdn, '\\')) { - doit = 1; + if (PL_strchr(rdnp, '\\')) { + do_dn_norm = 1; } else { while ((++rdnp <= endrdn) && (*rdnp == ' ') && (*rdnp == '\t')) ; /* DN contains an RDN <type>="<value>" ? */ if ((rdnp != endrdn) && ('"' == *rdnp) && ('"' == *endrdn)) { - doit = 1; + do_dn_norm = 1; } } - if (doit) { + if (do_dn_norm) { upgradedn_add_to_list(&ud_list, slapi_ch_strdup(LDBM_ENTRYRDN_STR), slapi_ch_strdup(rdn), 0); - LDAPDebug(LDAP_DEBUG_TRACE, - "%s: Found upgradedn candidate: %s (id %lu)\n", - inst->inst_name, *ud_valp, (u_long)temp_id); - /* entryrdn format */ + LDAPDebug2Args(LDAP_DEBUG_TRACE, + "%s: Found upgradedn candidate: (id %lu)\n", + inst->inst_name, (u_long)temp_id); /* * In case rdn is type="<RDN>" or type=<\D\N>, * add the rdn value if it's not there. @@ -1559,130 +1888,146 @@ upgradedn_producer(void *param) continue; } } - slapi_ch_free_string(&rdn); - } - for (a = e->e_attrs; a; a = a->a_next) { - if (!slapi_attr_is_dn_syntax_attr(a)) { - continue; /* not a syntax dn attr */ - } - - /* dn syntax attr */ - rc = get_values_from_string((const char *)ecopy, - a->a_type, &ud_vals); - if (rc || (NULL == ud_vals)) { - continue; /* empty; ignore it */ - } - - for (ud_valp = ud_vals; ud_valp && *ud_valp; ud_valp++) { - char **rdns = NULL; - char **rdnsp = NULL; - char *valueptr = NULL; - char *endvalue = NULL; - int isentrydn = 0; - - /* Also check RDN contains double quoted values */ - if (strcasecmp(a->a_type, "entrydn")) { - /* except entrydn */ - workdn = slapi_ch_strdup(*ud_valp); - isentrydn = 0; - } else { - /* entrydn: Get Slapi DN */ - sdn = slapi_entry_get_sdn(e); - workdn = slapi_ch_strdup(slapi_sdn_get_dn(sdn)); - isentrydn = 1; + /* checking the DN sintax values in the attribute list */ + for (a = e->e_attrs; a; a = a->a_next) { + if (!slapi_attr_is_dn_syntax_attr(a)) { + continue; /* not a syntax dn attr */ } - rdns = slapi_ldap_explode_dn(workdn, 0); - skipit = 0; - for (rdnsp = rdns; rdnsp && *rdnsp; rdnsp++) { - valueptr = PL_strchr(*rdnsp, '='); - if (NULL == valueptr) { - skipit = 1; - break; - } - endvalue = *rdnsp + strlen(*rdnsp) - 1; - while ((++valueptr <= endvalue) && - ((' ' == *valueptr) || ('\t' == *valueptr))) ; - if (0 == strlen(valueptr)) { - skipit = 1; - break; + + /* dn syntax attr */ + rc = get_values_from_string((const char *)ecopy, + a->a_type, &ud_vals); + if (rc || (NULL == ud_vals)) { + continue; /* empty; ignore it */ + } + + for (ud_valp = ud_vals; ud_valp && *ud_valp; ud_valp++) { + char **rdns = NULL; + char **rdnsp = NULL; + char *valueptr = NULL; + char *endvalue = NULL; + int isentrydn = 0; + + /* Also check RDN contains double quoted values */ + if (strcasecmp(a->a_type, "entrydn")) { + /* except entrydn */ + workdn = slapi_ch_strdup(*ud_valp); + isentrydn = 0; + } else { + /* entrydn: Get Slapi DN */ + sdn = slapi_entry_get_sdn(e); + workdn = slapi_ch_strdup(slapi_sdn_get_dn(sdn)); + isentrydn = 1; } - /* DN syntax value contains an RDN <type>="<value>" or - * '\\' in the value ? - * If yes, let's upgrade the dn format. */ - if ((('"' == *valueptr) && ('"' == *endvalue)) || - PL_strchr(valueptr, '\\')) { - doit = 1; - upgradedn_add_to_list(&ud_list, - slapi_ch_strdup(a->a_type), - slapi_ch_strdup(*ud_valp), - isentrydn?0:OLD_DN_NORMALIZE); - LDAPDebug(LDAP_DEBUG_TRACE, - "%s: Found upgradedn candidate: %s (id %lu)\n", - inst->inst_name, valueptr, (u_long)temp_id); - if (!entryrdn_get_switch() && isentrydn) { - /* entrydn format */ - /* - * In case entrydn is type="<DN>",<REST> or - * type=<\D\N>,<REST>, - * add the rdn value if it's not there. - */ - rc = slapi_entry_add_rdn_values(e); - if (rc) { - LDAPDebug(LDAP_DEBUG_ANY, - "%s: Failed to add rdn values " - "to an entry: %s (id %lu)\n", - inst->inst_name, normdn, (u_long)temp_id); - slapi_entry_free(e); e = NULL; - continue; + rdns = slapi_ldap_explode_dn(workdn, 0); + skipit = 0; + for (rdnsp = rdns; rdnsp && *rdnsp; rdnsp++) { + valueptr = PL_strchr(*rdnsp, '='); + if (NULL == valueptr) { + skipit = 1; + break; + } + endvalue = *rdnsp + strlen(*rdnsp) - 1; + while ((++valueptr <= endvalue) && + ((' ' == *valueptr) || ('\t' == *valueptr))) ; + if (0 == strlen(valueptr)) { + skipit = 1; + break; + } + /* DN syntax value contains an RDN <type>="<value>" or + * '\\' in the value ? + * If yes, let's upgrade the dn format. */ + if ((('"' == *valueptr) && ('"' == *endvalue)) || + PL_strchr(valueptr, '\\')) { + do_dn_norm = 1; + upgradedn_add_to_list(&ud_list, + slapi_ch_strdup(a->a_type), + slapi_ch_strdup(*ud_valp), + isentrydn?0:OLD_DN_NORMALIZE); + LDAPDebug(LDAP_DEBUG_TRACE, + "%s: Found upgradedn candidate: %s (id %lu)\n", + inst->inst_name, valueptr, (u_long)temp_id); + if (!entryrdn_get_switch() && isentrydn) { + /* entrydn format */ + /* + * In case entrydn is type="<DN>",<REST> or + * type=<\D\N>,<REST>, + * add the rdn value if it's not there. + */ + rc = slapi_entry_add_rdn_values(e); + if (rc) { + LDAPDebug(LDAP_DEBUG_ANY, + "%s: Failed to add rdn values " + "to an entry: %s (id %lu)\n", + inst->inst_name, normdn, (u_long)temp_id); + slapi_entry_free(e); e = NULL; + continue; + } } + break; } + /* + * else if (the rdn contains multiple spaces)? + * if yes, they are reduced to one. + * SET HAVE_MULTI_SPACES??? + */ + } /* for (rdnsp = rdns; rdnsp && *rdnsp; rdnsp++) */ + if (rdns) { + slapi_ldap_value_free(rdns); + } else { + skipit = 1; + } + if (skipit) { break; } - } /* for (rdnsp = rdns; rdnsp && *rdnsp; rdnsp++) */ - if (rdns) { - slapi_ldap_value_free(rdns); - } else { - skipit = 1; - } + slapi_ch_free_string(&workdn); + } /* for (ud_valp = ud_vals; ud_valp && *ud_valp; ud_valp++) */ + charray_free(ud_vals); + ud_vals = NULL; if (skipit) { + LDAPDebug(LDAP_DEBUG_ANY, "%s: WARNING: skipping an entry " + "with a corrupted dn (syntax value): %s " + "(id %lu)\n", + inst->inst_name, + workdn?workdn:"unknown", (u_long)temp_id); + slapi_ch_free_string(&workdn); + upgradedn_free_list(&ud_list); break; } - slapi_ch_free_string(&workdn); - } /* for (ud_valp = ud_vals; ud_valp && *ud_valp; ud_valp++) */ - charray_free(ud_vals); - ud_vals = NULL; + } /* for (a = e->e_attrs; a; a = a->a_next) */ if (skipit) { - LDAPDebug(LDAP_DEBUG_ANY, "%s: WARNING: skipping an entry " - "with a corrupted dn (syntax value): %s " - "(id %lu)\n", - inst->inst_name, - workdn?workdn:"unknown", (u_long)temp_id); - slapi_ch_free_string(&workdn); upgradedn_free_list(&ud_list); - break; + slapi_entry_free(e); e = NULL; + continue; } - } /* for (a = e->e_attrs; a; a = a->a_next) */ - if (skipit) { - upgradedn_free_list(&ud_list); - slapi_entry_free(e); e = NULL; - continue; - } + } /* end of if (chk_do_norm) */ + slapi_ch_free_string(&rdn); - if (!doit) { + if (is_dryrun) { + if (do_dn_norm) { + info_state |= DN_NORM; + /* + * If dryrun AND (found we need to do dn norm) AND + * (no need to check spaces), + * then we can quit here to return. + */ + if (!chk_dn_norm_sp) { + finished = 0; /* make it sure ... */ + upgradedn_free_list(&ud_list); + slapi_entry_free(e); e = NULL; + /* found upgrade dn candidates */ + goto bail; + } + } + if (do_dn_norm_sp) { + info_state |= DN_NORM_SP; + } /* We don't have to update dn syntax values. */ upgradedn_free_list(&ud_list); slapi_entry_free(e); e = NULL; continue; } - - /* doit */ - if (job->flags & FLAG_DRYRUN) { - /* We can return SUCCESS (== found upgrade dn candidates) */ - finished = 0; /* make it sure ... */ - upgradedn_free_list(&ud_list); - slapi_entry_free(e); e = NULL; - goto bail; - } + skipit = 0; for (ud_ptr = ud_list; ud_ptr; ud_ptr = ud_ptr->ud_next) { Slapi_Value *value = NULL; @@ -1810,23 +2155,14 @@ upgradedn_producer(void *param) id++; if (job->flags & FLAG_ABORT) goto error; - if (info->command == STOP) - { + if (info->command == STOP) { finished = 1; } } bail: dbc->c_close(dbc); dblayer_release_aux_id2entry( be, db, env ); - if (job->flags & FLAG_DRYRUN) { - if (finished) { /* Set if dn upgrade candidates are not found */ - info->state = FINISHED; - } else { /* At least one dn upgrade candidate is found */ - info->state = QUIT; - } - } else { - info->state = FINISHED; - } + info->state = FINISHED | info_state; goto done; error: @@ -1837,8 +2173,12 @@ error: info->state = ABORTED; done: + free_IDarray(&dn_norm_sp_conflicts); slapi_ch_free_string(&ecopy); slapi_ch_free(&(data.data)); + if (job->upgradefd) { + fclose(job->upgradefd); + } } static void @@ -1883,7 +2223,7 @@ foreman_do_parentid(ImportJob *job, FifoItem *fi, struct attrinfo *parentid_ai) int ret = 0; struct backentry *entry = fi->entry; - if (job->flags & FLAG_UPGRADEDNFORMAT) { + if (job->flags & (FLAG_UPGRADEDNFORMAT|FLAG_UPGRADEDNFORMAT_V1)) { /* Get the parentid attribute value from deleted attr list */ Slapi_Value *value = NULL; Slapi_Attr *pid_to_del = @@ -1950,7 +2290,7 @@ foreman_do_entrydn(ImportJob *job, FifoItem *fi) IDList *IDL; struct backentry *entry = fi->entry; - if (job->flags & FLAG_UPGRADEDNFORMAT) { + if (job->flags & (FLAG_UPGRADEDNFORMAT|FLAG_UPGRADEDNFORMAT_V1)) { /* Get the entrydn attribute value from deleted attr list */ Slapi_Value *value = NULL; Slapi_Attr *entrydn_to_del = @@ -1994,7 +2334,7 @@ foreman_do_entrydn(ImportJob *job, FifoItem *fi) /* So, we do an index read first */ err = 0; IDL = index_read(be, LDBM_ENTRYDN_STR, indextype_EQUALITY, &bv, NULL, &err); - if (job->flags & FLAG_UPGRADEDNFORMAT) { + if (job->flags & (FLAG_UPGRADEDNFORMAT|FLAG_UPGRADEDNFORMAT_V1)) { /* * In the UPGRADEDNFORMAT case, if entrydn value exists, * that means either 1) entrydn is not upgraded (ID == entry->ep_id) @@ -2058,7 +2398,7 @@ foreman_do_entryrdn(ImportJob *job, FifoItem *fi) int ret = 0; struct backentry *entry = fi->entry; - if (job->flags & FLAG_UPGRADEDNFORMAT) { + if (job->flags & (FLAG_UPGRADEDNFORMAT|FLAG_UPGRADEDNFORMAT_V1)) { /* Get the entrydn attribute value from deleted attr list */ Slapi_Value *value = NULL; Slapi_Attr *entryrdn_to_del = NULL; @@ -2084,6 +2424,10 @@ foreman_do_entryrdn(ImportJob *job, FifoItem *fi) } } slapi_attr_free(&entryrdn_to_del); + /* Waited to update e_srdn to delete the old entryrdn. + * Now updated it to adjust to the new s_dn. */ + slapi_rdn_set_all_dn(&(entry->ep_entry->e_srdn), + slapi_entry_get_dn_const(entry->ep_entry)); } } ret = entryrdn_index_entry(be, entry, BE_INDEX_ADD, NULL); @@ -2169,7 +2513,19 @@ import_foreman(void *param) import_log_notice(job, "WARNING: entry for id %d is missing", id); continue; } - + if (job->flags & FLAG_UPGRADEDNFORMAT_V1) { + if (entryrdn_get_switch()) { /* subtree-rename: on */ + /* insert into the entryrdn index */ + ret = foreman_do_entryrdn(job, fi); + } else { + /* insert into the entrydn index */ + ret = foreman_do_entrydn(job, fi); + if (ret == -1) { + goto cont; /* skip entry */ + } + } + goto next; + } /* first, fill in any operational attributes */ /* add_op_attrs wants a pblock for some reason. */ if (job->flags & FLAG_UPGRADEDNFORMAT) { @@ -2312,7 +2668,7 @@ import_foreman(void *param) if (job->flags & FLAG_ABORT) { goto error; } - +next: if (!(job->flags & FLAG_REINDEXING))/* reindex reads data from id2entry */ { /* insert into the id2entry index @@ -2348,14 +2704,14 @@ import_foreman(void *param) goto error; } - if (entryrdn_get_switch() /* subtree-rename: on */ || + if (entryrdn_get_switch() /* subtree-rename: on */ && !slapi_entry_flag_is_set(fi->entry->ep_entry, SLAPI_ENTRY_FLAG_TOMBSTONE)) { /* parentid index * (we have to do this here, because the parentID is dependent on * looking up by entrydn/entryrdn.) * Only add to the parent index if the entry is not a tombstone && - * subtree-rename is off. + * subtree-rename is on. */ ret = foreman_do_parentid(job, fi, parentid_ai); if (ret != 0) @@ -2536,7 +2892,7 @@ import_worker(void *param) slapi_pblock_destroy(pb); } else { /* No, process regular index */ - if (job->flags & FLAG_UPGRADEDNFORMAT) { + if (job->flags & (FLAG_UPGRADEDNFORMAT|FLAG_UPGRADEDNFORMAT_V1)) { /* Get the attribute value from deleted attr list */ Slapi_Value *value = NULL; const struct berval *bval = NULL; diff --git a/ldap/servers/slapd/back-ldbm/import.c b/ldap/servers/slapd/back-ldbm/import.c index e3966ff..aedff9f 100644 --- a/ldap/servers/slapd/back-ldbm/import.c +++ b/ldap/servers/slapd/back-ldbm/import.c @@ -51,7 +51,9 @@ #include "import.h" #define ERR_IMPORT_ABORTED -23 -#define DRYRUN_QUIT -24 +#define NEED_DN_NORM -24 +#define NEED_DN_NORM_SP -25 +#define NEED_DN_NORM_BT -26 /********** routines to manipulate the entry fifo **********/ @@ -111,7 +113,9 @@ FifoItem *import_fifo_fetch(ImportJob *job, ID id, int worker) if (fi->bad) { if (fi->bad == FIFOITEM_BAD) { fi->bad = FIFOITEM_BAD_PRINTED; - import_log_notice(job, "WARNING: bad entry: ID %d", id); + if (!(job->flags & FLAG_UPGRADEDNFORMAT_V1)) { + import_log_notice(job, "WARNING: bad entry: ID %d", id); + } } return NULL; } @@ -197,7 +201,7 @@ void import_log_notice(ImportJob *job, char *format, ...) slapi_task_log_notice(job->task, "%s", buffer); } /* also save it in the logs for posterity */ - if (job->flags & FLAG_UPGRADEDNFORMAT) { + if (job->flags & (FLAG_UPGRADEDNFORMAT|FLAG_UPGRADEDNFORMAT_V1)) { LDAPDebug(LDAP_DEBUG_ANY, "upgradedn %s: %s\n", job->inst->inst_name, buffer, 0); } else if (job->flags & FLAG_REINDEXING) { @@ -260,7 +264,7 @@ static int import_attr_callback(void *node, void *param) if (job->flags & FLAG_DRYRUN) { /* dryrun; we don't need the workers */ return 0; } - if (job->flags & FLAG_UPGRADEDNFORMAT) { + if (job->flags & (FLAG_UPGRADEDNFORMAT|FLAG_UPGRADEDNFORMAT_V1)) { /* Bring up import workers just for indexes having DN syntax * attribute type. (except entrydn -- taken care below) */ int rc = 0; @@ -707,6 +711,7 @@ static int import_monitor_threads(ImportJob *job, int *status) time_t last_time = 0; time_t time_interval = 0; int rc = 0; + int corestate = 0; for (current_worker = job->worker_list; current_worker != NULL; current_worker = current_worker->next) { @@ -774,14 +779,31 @@ static int import_monitor_threads(ImportJob *job, int *status) import_calc_rate(current_worker, time_interval); import_print_worker_status(current_worker); } - if (current_worker->state == QUIT) { - rc = DRYRUN_QUIT; /* Set the RC; Don't abort now; - We have to stop other threads */ - } else if (current_worker->state != FINISHED) { - finished = 0; - } + corestate = current_worker->state & CORESTATE; if (current_worker->state == ABORTED) { +LDAPDebug0Args(LDAP_DEBUG_ANY, "import_monitor_threads: current_worker->state is ABORTED\n"); goto error_abort; + } else if ((corestate == QUIT) || (corestate == FINISHED)) { +LDAPDebug1Arg(LDAP_DEBUG_ANY, "import_monitor_threads: current_worker->state is %s\n", (corestate==QUIT)?"QUIT":"FINISHED"); + if (DN_NORM_BT == (DN_NORM_BT & current_worker->state)) { + /* upgrading dn norm (both) is needed */ + rc = NEED_DN_NORM_BT; /* Set the RC; Don't abort now; + * We have to stop other + * threads */ + } else if (DN_NORM == (DN_NORM_BT & current_worker->state)) { + /* upgrading dn norm is needed */ + rc = NEED_DN_NORM; /* Set the RC; Don't abort now; + * We have to stop other threads + */ + } else if (DN_NORM_SP == (DN_NORM_BT & current_worker->state)) { + /* upgrading spaces in dn norm is needed */ + rc = NEED_DN_NORM_SP; /* Set the RC; Don't abort now; + * We have to stop other + * threads */ + } + current_worker->state = corestate; + } else if (current_worker->state != FINISHED) { + finished = 0; } } @@ -906,7 +928,9 @@ static int import_run_pass(ImportJob *job, int *status) /* Monitor the threads until we're done or fail */ ret = import_monitor_threads(job, status); - if ((ret == ERR_IMPORT_ABORTED) || (ret == DRYRUN_QUIT)) { + if ((ret == ERR_IMPORT_ABORTED) || (ret == NEED_DN_NORM) || + (ret == NEED_DN_NORM_SP) || (ret == NEED_DN_NORM_BT)) { +import_log_notice(job, "Thread monitoring returned: %d\n", ret); goto error; } else if (ret != 0) { import_log_notice(job, "Thread monitoring aborted: %d\n", ret); @@ -1140,11 +1164,16 @@ int import_main_offline(void *arg) if (job->task) slapi_task_inc_refcount(job->task); - if (job->flags & FLAG_UPGRADEDNFORMAT) { + if (job->flags & (FLAG_UPGRADEDNFORMAT|FLAG_UPGRADEDNFORMAT_V1)) { if (job->flags & FLAG_DRYRUN) { opstr = "Upgrade Dn Dryrun"; + } else if ((job->flags & (FLAG_UPGRADEDNFORMAT|FLAG_UPGRADEDNFORMAT_V1)) + == (FLAG_UPGRADEDNFORMAT|FLAG_UPGRADEDNFORMAT_V1)) { + opstr = "Upgrade Dn (Full)"; + } else if (job->flags & FLAG_UPGRADEDNFORMAT_V1) { + opstr = "Upgrade Dn (Spaces)"; } else { - opstr = "Upgrade Dn"; + opstr = "Upgrade Dn (RFC 4514)"; } } else if (job->flags & FLAG_REINDEXING) { opstr = "Reindexing"; @@ -1187,7 +1216,7 @@ int import_main_offline(void *arg) /* start the producer */ import_init_worker_info(producer, job); producer->work_type = PRODUCER; - if (job->flags & FLAG_UPGRADEDNFORMAT) + if (job->flags & (FLAG_UPGRADEDNFORMAT|FLAG_UPGRADEDNFORMAT_V1)) { if (! CREATE_THREAD(PR_USER_THREAD, (VFP)upgradedn_producer, producer, PR_PRIORITY_NORMAL, PR_GLOBAL_BOUND_THREAD, @@ -1278,11 +1307,10 @@ int import_main_offline(void *arg) aborted = 1; goto error; } - if (ret == DRYRUN_QUIT) { - goto error; /* Found the candidate; close the db files and quit */ - } - - if (0 != ret) { + if ((ret == NEED_DN_NORM) || (ret == NEED_DN_NORM_SP) || + (ret == NEED_DN_NORM_BT)) { + goto error; + } else if (0 != ret) { /* Some horrible fate has befallen the import */ import_log_notice(job, "Fatal pass error %d", ret); goto error; @@ -1414,7 +1442,8 @@ error: } } if (0 != ret) { - if (!(job->flags & FLAG_DRYRUN)) { /* If not dryrun */ + if (!(job->flags & (FLAG_DRYRUN|FLAG_UPGRADEDNFORMAT_V1))) { + /* If not dryrun NOR upgradedn space */ /* if running in the dry run mode, don't touch the db */ dblayer_delete_instance_dir(be); } @@ -1475,23 +1504,40 @@ error: } } - if (job->flags & FLAG_DRYRUN) { + if (job->flags & (FLAG_DRYRUN|FLAG_UPGRADEDNFORMAT_V1)) { if (0 == ret) { import_log_notice(job, "%s complete. %s is up-to-date.", opstr, job->inst->inst_name); + ret = 0; + if (job->task) { + slapi_task_dec_refcount(job->task); + } + import_all_done(job, ret); + } else if (NEED_DN_NORM_BT == ret) { + import_log_notice(job, "%s complete. %s needs upgradednformat all.", + opstr, job->inst->inst_name); + if (job->task) { + slapi_task_dec_refcount(job->task); + } + import_all_done(job, ret); ret = 1; + } else if (NEED_DN_NORM == ret) { + import_log_notice(job, "%s complete. %s needs upgradednformat.", + opstr, job->inst->inst_name); if (job->task) { slapi_task_dec_refcount(job->task); } import_all_done(job, ret); - } else if (DRYRUN_QUIT == ret) { - import_log_notice(job, "%s complete. %s needs upgradednformat.", + ret = 2; + } else if (NEED_DN_NORM_SP == ret) { + import_log_notice(job, + "%s complete. %s needs upgradednformat spaces.", opstr, job->inst->inst_name); if (job->task) { slapi_task_dec_refcount(job->task); } import_all_done(job, ret); - ret = 0; + ret = 3; } else { ret = -1; if (job->task != NULL) { @@ -1581,8 +1627,13 @@ int ldbm_back_ldif2ldbm_deluxe(Slapi_PBlock *pb) job->flags = FLAG_USE_FILES; if (NULL == name_array) { /* no ldif file is given -> reindexing or upgradedn */ - if (up_flags & SLAPI_UPGRADEDNFORMAT) { - job->flags |= FLAG_UPGRADEDNFORMAT; + if (up_flags & (SLAPI_UPGRADEDNFORMAT|SLAPI_UPGRADEDNFORMAT_V1)) { + if (up_flags & SLAPI_UPGRADEDNFORMAT) { + job->flags |= FLAG_UPGRADEDNFORMAT; + } + if (up_flags & SLAPI_UPGRADEDNFORMAT_V1) { + job->flags |= FLAG_UPGRADEDNFORMAT_V1; + } if (up_flags & SLAPI_DRYRUN) { job->flags |= FLAG_DRYRUN; } diff --git a/ldap/servers/slapd/back-ldbm/import.h b/ldap/servers/slapd/back-ldbm/import.h index c9a3f8a..7870403 100644 --- a/ldap/servers/slapd/back-ldbm/import.h +++ b/ldap/servers/slapd/back-ldbm/import.h @@ -159,8 +159,9 @@ typedef struct { PRLock *wire_lock; /* lock for serializing wire imports */ PRCondVar *wire_cv; /* ... and ordering the startup */ PRThread *main_thread; /* for FRI: import_main() thread id */ - int encrypt; - Slapi_Value *usn_value; /* entryusn for import */ + int encrypt; + Slapi_Value *usn_value; /* entryusn for import */ + FILE *upgradefd; /* used for the upgrade */ } ImportJob; #define FLAG_INDEX_ATTRS 0x01 /* should we index the attributes? */ @@ -173,6 +174,7 @@ typedef struct { #define FLAG_DN2RDN 0x40 /* modify backend to the rdn format */ #define FLAG_UPGRADEDNFORMAT 0x80 /* read from id2entry and do upgrade dn */ #define FLAG_DRYRUN 0x100 /* dryrun for upgrade dn */ +#define FLAG_UPGRADEDNFORMAT_V1 0x200 /* taking care multiple spaces in dn */ /* Structure holding stuff about a worker thread and what it's up to */ @@ -202,12 +204,17 @@ struct _import_worker_info { #define ABORT 3 #define STOP 4 -/* Values for state */ -#define WAITING 1 -#define RUNNING 2 -#define FINISHED 3 -#define ABORTED 4 -#define QUIT 5 /* quit intentionally. to distinguish from ABORTED & FINISHED */ +/* Values for job state */ +#define WAITING 0x1 +#define RUNNING 0x2 +#define FINISHED 0x4 +#define ABORTED 0x8 +#define QUIT 0x10 /* quit intentionally. + * introduced to distinguish from ABORTED, FINISHED */ +#define CORESTATE 0xff +#define DN_NORM 0x100 /* do dn normalization in upgrade */ +#define DN_NORM_SP 0x200 /* do dn normalization for multi spaces in upgrade */ +#define DN_NORM_BT (DN_NORM | DN_NORM_SP) /* this is just a convenience, because the slapi_ch_* calls are annoying */ #define CALLOC(name) (name *)slapi_ch_calloc(1, sizeof(name)) diff --git a/ldap/servers/slapd/back-ldbm/ldif2ldbm.c b/ldap/servers/slapd/back-ldbm/ldif2ldbm.c index 8818ba7..0793041 100644 --- a/ldap/servers/slapd/back-ldbm/ldif2ldbm.c +++ b/ldap/servers/slapd/back-ldbm/ldif2ldbm.c @@ -204,8 +204,9 @@ int ldbm_back_ok_to_dump(const char *dn, char **include, char **exclude) * allowed to specify yourself on entries. * Currenty the list of these is: numSubordinates, hasSubordinates */ -int add_op_attrs(Slapi_PBlock *pb, struct ldbminfo *li, struct backentry *ep, - int *status) +int +add_op_attrs(Slapi_PBlock *pb, struct ldbminfo *li, struct backentry *ep, + int *status) { backend *be; char *pdn; @@ -3549,7 +3550,7 @@ int ldbm_back_upgradednformat(Slapi_PBlock *pb) slapi_pblock_get(pb, SLAPI_BACKEND_TASK, &task); slapi_pblock_get(pb, SLAPI_DB2LDIF_SERVER_RUNNING, &server_running); slapi_pblock_get(pb, SLAPI_BACKEND_INSTANCE_NAME, &instance_name); - slapi_pblock_get( pb, SLAPI_SEQ_TYPE, &ud_flags ); + slapi_pblock_get(pb, SLAPI_SEQ_TYPE, &ud_flags); run_from_cmdline = (task_flags & SLAPI_TASK_RUNNING_FROM_COMMANDLINE); slapi_pblock_get(pb, SLAPI_PLUGIN_PRIVATE, &li); @@ -3624,11 +3625,39 @@ int ldbm_back_upgradednformat(Slapi_PBlock *pb) workdbdir = rel2abspath(rawworkdbdir); dbversion_read(li, workdbdir, &ldbmversion, &dataversion); - if (ldbmversion && PL_strstr(ldbmversion, BDB_DNFORMAT)) { + if (ldbmversion) { + char *ptr = PL_strstr(ldbmversion, BDB_DNFORMAT); + if (ptr) { + size_t dnformat_len = strlen(ptr); + /* DN format is RFC 4514 compliant */ + if (strlen(ptr) == strlen(BDB_DNFORMAT)) { /* no version */ + /* + * DN format is RFC 4514 compliant. + * But it hasn't taken care of the multiple spaces yet. + */ + ud_flags &= ~SLAPI_UPGRADEDNFORMAT; + ud_flags |= SLAPI_UPGRADEDNFORMAT_V1; + slapi_pblock_set(pb, SLAPI_SEQ_TYPE, &ud_flags); + rc = 3; /* 0: need upgrade (dn norm sp, only) */ + } else { + /* DN format already takes care of the multiple spaces */ + slapi_log_error(SLAPI_LOG_FATAL, "Upgrade DN Format", + "Instance %s in %s is up-to-date\n", + instance_name, workdbdir); + rc = 0; /* 0: up-to-date */ + goto bail; + } + } else { + /* DN format is not RFC 4514 compliant */ + ud_flags |= SLAPI_UPGRADEDNFORMAT | SLAPI_UPGRADEDNFORMAT_V1; + slapi_pblock_set(pb, SLAPI_SEQ_TYPE, &ud_flags); + rc = 1; /* 0: need upgrade (both) */ + } + } else { slapi_log_error(SLAPI_LOG_FATAL, "Upgrade DN Format", - "Instance %s in %s is up-to-date\n", + "Failed to get DBVERSION (Instance name: %s, dir %s)\n", instance_name, workdbdir); - rc = 1; /* 1: up-to-date; 0: need upgrade; otherwise: error */ + rc = -1; /* error */ goto bail; } @@ -3681,7 +3710,7 @@ int ldbm_back_upgradednformat(Slapi_PBlock *pb) } *sep = '/'; if (((0 == rc) && !(ud_flags & SLAPI_DRYRUN)) || - ((rc > 0) && (ud_flags & SLAPI_DRYRUN))) { + ((rc == 0) && (ud_flags & SLAPI_DRYRUN))) { /* modify the DBVERSION files if the DN upgrade was successful OR * if DRYRUN, the backend instance is up-to-date. */ dbversion_write(li, workdbdir, NULL, DBVERSION_ALL); /* inst db dir */ diff --git a/ldap/servers/slapd/slapi-private.h b/ldap/servers/slapd/slapi-private.h index 937c2bb..375c33f 100644 --- a/ldap/servers/slapd/slapi-private.h +++ b/ldap/servers/slapd/slapi-private.h @@ -1223,6 +1223,7 @@ void DS_Sleep(PRIntervalTime ticks); generate entryrdn index */ #define SLAPI_UPGRADEDNFORMAT 0x8 /* specify this op is upgradednformat */ #define SLAPI_DRYRUN 0x10 /* dryrun mode for upgradednformat */ +#define SLAPI_UPGRADEDNFORMAT_V1 0x20 /* taking care multipe spaces */ /*

1 0

ldap/servers
by Mark Reynolds 09 Apr '13

09 Apr '13

ldap/servers/plugins/acl/acl.h | 2 ++ ldap/servers/plugins/acl/acl_ext.c | 18 ++++++++++++++++-- ldap/servers/plugins/acl/aclinit.c | 2 +- ldap/servers/plugins/acl/acllas.c | 19 +++++++++++++++---- ldap/servers/slapd/libslapd.def | 1 + ldap/servers/slapd/pblock.c | 9 +++++++++ ldap/servers/slapd/slapi-plugin.h | 9 +++++++++ 7 files changed, 53 insertions(+), 7 deletions(-) New commits: commit 579fcf46d3205406618692bd87359a2d3c79b8d4 Author: Mark Reynolds <mreynolds(a)redhat.com> Date: Mon Apr 8 23:04:30 2013 -0400 Ticket 77 - [RFE] Add ACI support for ldapi RFE Description: Add support for LDAPI connection types in access control. Fix Description: Extended the authMethod keyword to include LDAPI. (targetattr = "*") (version 3.0;acl "ldapi";allow (all)(userdn = "ldap:///anyone" and authmethod = "ldapi" or ip = "127.0.0.1");) https://fedorahosted.org/389/ticket/77 Reviewed by: richm(Thanks!) diff --git a/ldap/servers/plugins/acl/acl.h b/ldap/servers/plugins/acl/acl.h index 5e840a9..0eb8570 100644 --- a/ldap/servers/plugins/acl/acl.h +++ b/ldap/servers/plugins/acl/acl.h @@ -209,6 +209,7 @@ typedef enum #define DS_ATTR_AUTHTYPE "authtype" #define DS_ATTR_CERT "clientcert" #define DS_ATTR_SSF "ssf" +#define DS_ATTR_LDAPI "ldapi" #define ACL_ANOM_MAX_ACL 40 struct scoped_entry_anominfo { @@ -664,6 +665,7 @@ typedef struct { Acl_PBlock *aclpb; Slapi_Entry *resourceEntry; int ssf; + char *ldapi; }lasInfo; diff --git a/ldap/servers/plugins/acl/acl_ext.c b/ldap/servers/plugins/acl/acl_ext.c index f8699b4..b3f9072 100644 --- a/ldap/servers/plugins/acl/acl_ext.c +++ b/ldap/servers/plugins/acl/acl_ext.c @@ -446,7 +446,6 @@ acl__handle_plugin_config_entry (Slapi_Entry *e, void *callback_data ) int acl_create_aclpb_pool () { - Acl_PBlock *aclpb; Acl_PBlock *prev_aclpb; Acl_PBlock *first_aclpb; @@ -648,6 +647,11 @@ acl__malloc_aclpb ( ) "Unable to set the AUTH TYPE in the Plist\n"); goto error; } + if (PListInitProp(aclpb->aclpb_proplist, 0, DS_ATTR_LDAPI, aclpb, 0) < 0) { + slapi_log_error(SLAPI_LOG_FATAL, plugin_name, + "Unable to set the AUTH TYPE in the Plist\n"); + goto error; + } if (PListInitProp(aclpb->aclpb_proplist, 0, DS_ATTR_ENTRY, aclpb, 0) < 0) { slapi_log_error(SLAPI_LOG_FATAL, plugin_name, "Unable to set the ENTRY TYPE in the Plist\n"); @@ -771,7 +775,7 @@ acl__free_aclpb ( Acl_PBlock **aclpb_ptr) /* Initializes the aclpb */ void -acl_init_aclpb ( Slapi_PBlock *pb , Acl_PBlock *aclpb, const char *ndn, int copy_from_aclcb) +acl_init_aclpb ( Slapi_PBlock *pb, Acl_PBlock *aclpb, const char *ndn, int copy_from_aclcb) { struct acl_cblock *aclcb = NULL; char *authType; @@ -818,6 +822,13 @@ acl_init_aclpb ( Slapi_PBlock *pb , Acl_PBlock *aclpb, const char *ndn, int copy "Unable to set the AUTH TYPE in the Plist\n"); return; } + if(slapi_is_ldapi_conn(pb)){ + if(PListAssignValue(aclpb->aclpb_proplist, DS_ATTR_LDAPI, "yes", 0) < 0){ + slapi_log_error(SLAPI_LOG_FATAL, plugin_name, + "Unable to set the AUTH TYPE in the Plist\n"); + return; + } + } slapi_pblock_get ( pb, SLAPI_OPERATION_SSF, &ssf); if (PListAssignValue(aclpb->aclpb_proplist, DS_ATTR_SSF, (const void *)ssf, 0) < 0) { slapi_log_error(SLAPI_LOG_FATAL, plugin_name, @@ -1036,6 +1047,9 @@ acl__done_aclpb ( struct acl_pblock *aclpb ) PListDeleteProp(aclpb->aclpb_proplist, rc, NULL); } + /* reset the LDAPI property */ + PListAssignValue(aclpb->aclpb_proplist, DS_ATTR_LDAPI, NULL, 0); + /* * Remove the DS_ATTR_IP property from the property list. * The value of this property is just the property pointer diff --git a/ldap/servers/plugins/acl/aclinit.c b/ldap/servers/plugins/acl/aclinit.c index 9628075..12efa74 100644 --- a/ldap/servers/plugins/acl/aclinit.c +++ b/ldap/servers/plugins/acl/aclinit.c @@ -547,7 +547,7 @@ __aclinit__RegisterLases(void) return ACL_ERR; } if (ACL_LasRegister(NULL, DS_LAS_AUTHMETHOD, - (LASEvalFunc_t)DS_LASAuthMethodEval, + (LASEvalFunc_t)DS_LASAuthMethodEval, (LASFlushFunc_t)NULL) < 0) { slapi_log_error (SLAPI_LOG_FATAL, plugin_name, "Unable to register CLIENTAUTHTYPE Las\n"); diff --git a/ldap/servers/plugins/acl/acllas.c b/ldap/servers/plugins/acl/acllas.c index 5bcb482..eb440e7 100644 --- a/ldap/servers/plugins/acl/acllas.c +++ b/ldap/servers/plugins/acl/acllas.c @@ -1684,7 +1684,9 @@ DS_LASAuthMethodEval(NSErr_t *errp, char *attr_name, CmpOp_t comparator, /* None method means, we don't care -- otherwise we care */ if ((strcasecmp(attr, "none") == 0) || - (strcasecmp(attr, lasinfo.authType) == 0)) { + (strcasecmp(attr, lasinfo.authType) == 0) || + (lasinfo.ldapi && strcasecmp(attr, DS_ATTR_LDAPI) == 0)) + { matched = ACL_TRUE; } @@ -3678,6 +3680,7 @@ acllas__handle_client_search ( Slapi_Entry *e, void *callback_data ) aclpb->aclpb_client_entry = slapi_entry_dup ( e ); return 0; } + /* * * Do all the necessary setup for all the @@ -3694,7 +3697,6 @@ acllas__handle_client_search ( Slapi_Entry *e, void *callback_data ) * #define LAS_EVAL_FAIL -4 * #define LAS_EVAL_INVALID -5 */ - static int __acllas_setup ( NSErr_t *errp, char *attr_name, CmpOp_t comparator, int allow_range, char *attr_pattern, int *cachable, void **LAS_cookie, @@ -3750,8 +3752,8 @@ __acllas_setup ( NSErr_t *errp, char *attr_name, CmpOp_t comparator, return LAS_EVAL_FAIL; } - if ((rc = PListFindValue(subject, DS_ATTR_ENTRY, - (void **)&linfo->resourceEntry, NULL)) < 0) { + if ((rc = PListFindValue(subject, DS_ATTR_ENTRY, + (void **)&linfo->resourceEntry, NULL)) < 0){ acl_print_acllib_err(errp, NULL); slapi_log_error( SLAPI_LOG_ACL, plugin_name, "%s:Unable to get the Slapi_Entry attr(%d)\n",lasName, rc); @@ -3767,6 +3769,15 @@ __acllas_setup ( NSErr_t *errp, char *attr_name, CmpOp_t comparator, "%s:Unable to get the ACLPB(%d)\n", lasName, rc); return LAS_EVAL_FAIL; } + + /* LDAPI? */ + if ((rc = PListFindValue(subject, DS_ATTR_LDAPI, (void **)&linfo->ldapi, NULL)) < 0){ + slapi_log_error( SLAPI_LOG_ACL, plugin_name, + "%s:Unable to get LDAPI value(%d)\n", lasName, rc); + + return LAS_EVAL_FAIL; + } + if (NULL == attr_pattern ) { slapi_log_error( SLAPI_LOG_ACL, plugin_name, "%s:No rule value in the ACL\n", lasName); diff --git a/ldap/servers/slapd/libslapd.def b/ldap/servers/slapd/libslapd.def index 70a2993..4b7cd7b 100644 --- a/ldap/servers/slapd/libslapd.def +++ b/ldap/servers/slapd/libslapd.def @@ -1199,3 +1199,4 @@ EXPORTS config_get_pw_mincategories @1206 config_get_pw_mintokenlength @1207 slapi_check_account_lock @1208 + slapi_is_ldapi_conn @1209 diff --git a/ldap/servers/slapd/pblock.c b/ldap/servers/slapd/pblock.c index 8d8c66f..d88d224 100644 --- a/ldap/servers/slapd/pblock.c +++ b/ldap/servers/slapd/pblock.c @@ -3535,6 +3535,15 @@ slapi_pblock_set( Slapi_PBlock *pblock, int arg, void *value ) return( 0 ); } +int +slapi_is_ldapi_conn(Slapi_PBlock *pb) +{ + if(pb && pb->pb_conn){ + return pb->pb_conn->c_unix_local; + } else { + return 0; + } +} /* * Clears (and free's as appropriate) the bind DN and related credentials diff --git a/ldap/servers/slapd/slapi-plugin.h b/ldap/servers/slapd/slapi-plugin.h index f2dc6af..25ccb00 100644 --- a/ldap/servers/slapd/slapi-plugin.h +++ b/ldap/servers/slapd/slapi-plugin.h @@ -6067,6 +6067,15 @@ int slapi_back_transaction_commit(Slapi_PBlock *pb); */ int slapi_back_transaction_abort(Slapi_PBlock *pb); +/** + * Checks if the connection type is LDAPI + * + * \param pb Parameter block. + * + * \return \c 1 if connection type is LDAPI + * \return \c 0 if connection is not LDAPI + */ +int slapi_is_ldapi_conn(Slapi_PBlock *pb); int slapi_be_is_flag_set(Slapi_Backend * be, int flag); void slapi_be_set_flag(Slapi_Backend * be, int flag);

1 0

Changes to '389-ds-base-1.3.1'
by Richard Allen Megginson 09 Apr '13

09 Apr '13

New branch '389-ds-base-1.3.1' available with the following commits:

1 0

ldap/admin ldap/servers man/man8
by Mark Reynolds 09 Apr '13

09 Apr '13

ldap/admin/src/scripts/fixup-memberof.pl.in | 4 ++-- ldap/servers/plugins/memberof/memberof.c | 2 +- man/man8/fixup-memberof.pl.8 | 3 ++- 3 files changed, 5 insertions(+), 4 deletions(-) New commits: commit de61f34aea3ffe197a173bfa6ef7a34137d6e36c Author: Mark Reynolds <mreynolds(a)redhat.com> Date: Tue Apr 9 09:59:20 2013 -0400 Ticket 47315 - filter option in fixup-memberof requires more clarification Bug Description: The usgae/documentaton states that if you don't supply a filter, then "all" the entries will be checked. Actually only users that have "objectclass: inetuser" are checked. Fix Description: Added another objectclass to the default query, inetadmin. inetuser & inetadmin are the only standard objectclasses that have memberOf in thier definitions. Also updated the man page, and the script usage to state what the internal query/filter is. https://fedorahosted.org/389/ticket/47315 Reviewed by: richm(Thanks!) diff --git a/ldap/admin/src/scripts/fixup-memberof.pl.in b/ldap/admin/src/scripts/fixup-memberof.pl.in index 43c24d2..a495249 100644 --- a/ldap/admin/src/scripts/fixup-memberof.pl.in +++ b/ldap/admin/src/scripts/fixup-memberof.pl.in @@ -61,8 +61,8 @@ sub usage { print(STDERR " -j filename - Read Directory Manager's password from file\n"); print(STDERR " -b baseDN - Base DN that contains entries to fix up.\n"); print(STDERR " -f filter - Filter for entries to fix up\n"); - print(STDERR " If omitted, all entries under the specified\n"); - print(STDERR " base will have their memberOf attribute regenerated.\n"); + print(STDERR " If omitted, all entries with objectclass inetuser/inetadmin under the\n"); + print(STDERR " specified base will have their memberOf attribute regenerated.\n"); print(STDERR " -P protocol - STARTTLS, LDAPS, LDAPI, LDAP (default: uses most secure protocol available)\n"); print(STDERR " -v - Verbose output\n"); print(STDERR " -h - Display usage\n"); diff --git a/ldap/servers/plugins/memberof/memberof.c b/ldap/servers/plugins/memberof/memberof.c index 8fcc88c..1c50b67 100644 --- a/ldap/servers/plugins/memberof/memberof.c +++ b/ldap/servers/plugins/memberof/memberof.c @@ -2393,7 +2393,7 @@ int memberof_task_add(Slapi_PBlock *pb, Slapi_Entry *e, goto out; } - if ((filter = fetch_attr(e, "filter", "(objectclass=inetuser)")) == NULL) + if ((filter = fetch_attr(e, "filter", "(|(objectclass=inetuser)(objectclass=inetadmin))")) == NULL) { *returncode = LDAP_OBJECT_CLASS_VIOLATION; rv = SLAPI_DSE_CALLBACK_ERROR; diff --git a/man/man8/fixup-memberof.pl.8 b/man/man8/fixup-memberof.pl.8 index 250997d..7716a58 100644 --- a/man/man8/fixup-memberof.pl.8 +++ b/man/man8/fixup-memberof.pl.8 @@ -46,7 +46,8 @@ The name of the file that contains the root DN password. The DN of the subtree containing the entries to update. .TP .B \fB\-f\fR \fIfilter\fR -An LDAP query filter to use to select the entries within the subtree to update. If there is no filter set, then the memberOf attribute is regenerated for every entry in the subtree. +An LDAP query filter to use to select the entries within the subtree to update. If there is no filter set, then +the memberOf attribute is regenerated for every entry in the subtree that has the objectclass inetuser/inetadmin. .TP .B \fB\-P\fR \fIprotocol\fR The connection protocol to connect to the Directory Server. Protocols are STARTTLS, LDAPS, LDAPI, and LDAP.

1 0

Branch '389-ds-base-1.3.0' - ldap/schema
by Mark Reynolds 08 Apr '13

08 Apr '13

ldap/schema/01core389.ldif | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) New commits: commit 15d5fb94a272eb7314c58895e75696928873533d Author: Mark Reynolds <mreynolds(a)redhat.com> Date: Mon Apr 8 20:13:01 2013 -0400 Ticket 47318 - server fails to start after upgrade(schema error) Bug Description: After upgrading the server fails to start because the Root DN access control plugin's entry changed to include nsslapd-Plugin objectclass. Fix Description: Update the schema to allow nsslapd-plugin-depends-on-type in the nsslapdPlugin objectclass. https://fedorahosted.org/389/ticket/47318 Reviewed by: nhosoi(Thanks!) (cherry picked from commit b139ff8e404bccacf88c34c03637da0e9c4c2a0d) diff --git a/ldap/schema/01core389.ldif b/ldap/schema/01core389.ldif index ad5b555..2f848e7 100644 --- a/ldap/schema/01core389.ldif +++ b/ldap/schema/01core389.ldif @@ -156,7 +156,7 @@ attributeTypes: ( 2.16.840.1.113730.3.1.2156 NAME 'nsslapd-sasl-max-buffer-size' # objectclasses # objectClasses: ( 2.16.840.1.113730.3.2.40 NAME 'directoryServerFeature' DESC 'Netscape defined objectclass' SUP top MAY ( oid $ cn $ multiLineDescription ) X-ORIGIN 'Netscape Directory Server' ) -objectClasses: ( 2.16.840.1.113730.3.2.41 NAME 'nsslapdPlugin' DESC 'Netscape defined objectclass' SUP top MUST ( cn $ nsslapd-pluginPath $ nsslapd-pluginInitFunc $ nsslapd-pluginType $ nsslapd-pluginId $ nsslapd-pluginVersion $ nsslapd-pluginVendor $ nsslapd-pluginDescription $ nsslapd-pluginEnabled ) MAY ( nsslapd-pluginConfigArea ) X-ORIGIN 'Netscape Directory Server' ) +objectClasses: ( 2.16.840.1.113730.3.2.41 NAME 'nsslapdPlugin' DESC 'Netscape defined objectclass' SUP top MUST ( cn $ nsslapd-pluginPath $ nsslapd-pluginInitFunc $ nsslapd-pluginType $ nsslapd-pluginId $ nsslapd-pluginVersion $ nsslapd-pluginVendor $ nsslapd-pluginDescription $ nsslapd-pluginEnabled ) MAY ( nsslapd-pluginConfigArea $ nsslapd-plugin-depends-on-type ) X-ORIGIN 'Netscape Directory Server' ) objectClasses: ( 2.16.840.1.113730.3.2.44 NAME 'nsIndex' DESC 'Netscape defined objectclass' SUP top MUST ( cn $ nsSystemIndex ) MAY ( description $ nsIndexType $ nsMatchingRule ) X-ORIGIN 'Netscape Directory Server' ) objectClasses: ( 2.16.840.1.113730.3.2.109 NAME 'nsBackendInstance' DESC 'Netscape defined objectclass' SUP top MUST ( CN ) X-ORIGIN 'Netscape Directory Server' ) objectClasses: ( 2.16.840.1.113730.3.2.110 NAME 'nsMappingTree' DESC 'Netscape defined objectclass' SUP top MUST ( CN ) X-ORIGIN 'Netscape Directory Server' )

1 0

Branch '389-ds-base-1.2.11' - ldap/schema
by Mark Reynolds 08 Apr '13

08 Apr '13

ldap/schema/01core389.ldif | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) New commits: commit b139ff8e404bccacf88c34c03637da0e9c4c2a0d Author: Mark Reynolds <mreynolds(a)redhat.com> Date: Mon Apr 8 20:13:01 2013 -0400 Ticket 47318 - server fails to start after upgrade(schema error) Bug Description: After upgrading the server fails to start because the Root DN access control plugin's entry changed to include nsslapd-Plugin objectclass. Fix Description: Update the schema to allow nsslapd-plugin-depends-on-type in the nsslapdPlugin objectclass. https://fedorahosted.org/389/ticket/47318 Reviewed by: nhosoi(Thanks!) diff --git a/ldap/schema/01core389.ldif b/ldap/schema/01core389.ldif index ad5b555..2f848e7 100644 --- a/ldap/schema/01core389.ldif +++ b/ldap/schema/01core389.ldif @@ -156,7 +156,7 @@ attributeTypes: ( 2.16.840.1.113730.3.1.2156 NAME 'nsslapd-sasl-max-buffer-size' # objectclasses # objectClasses: ( 2.16.840.1.113730.3.2.40 NAME 'directoryServerFeature' DESC 'Netscape defined objectclass' SUP top MAY ( oid $ cn $ multiLineDescription ) X-ORIGIN 'Netscape Directory Server' ) -objectClasses: ( 2.16.840.1.113730.3.2.41 NAME 'nsslapdPlugin' DESC 'Netscape defined objectclass' SUP top MUST ( cn $ nsslapd-pluginPath $ nsslapd-pluginInitFunc $ nsslapd-pluginType $ nsslapd-pluginId $ nsslapd-pluginVersion $ nsslapd-pluginVendor $ nsslapd-pluginDescription $ nsslapd-pluginEnabled ) MAY ( nsslapd-pluginConfigArea ) X-ORIGIN 'Netscape Directory Server' ) +objectClasses: ( 2.16.840.1.113730.3.2.41 NAME 'nsslapdPlugin' DESC 'Netscape defined objectclass' SUP top MUST ( cn $ nsslapd-pluginPath $ nsslapd-pluginInitFunc $ nsslapd-pluginType $ nsslapd-pluginId $ nsslapd-pluginVersion $ nsslapd-pluginVendor $ nsslapd-pluginDescription $ nsslapd-pluginEnabled ) MAY ( nsslapd-pluginConfigArea $ nsslapd-plugin-depends-on-type ) X-ORIGIN 'Netscape Directory Server' ) objectClasses: ( 2.16.840.1.113730.3.2.44 NAME 'nsIndex' DESC 'Netscape defined objectclass' SUP top MUST ( cn $ nsSystemIndex ) MAY ( description $ nsIndexType $ nsMatchingRule ) X-ORIGIN 'Netscape Directory Server' ) objectClasses: ( 2.16.840.1.113730.3.2.109 NAME 'nsBackendInstance' DESC 'Netscape defined objectclass' SUP top MUST ( CN ) X-ORIGIN 'Netscape Directory Server' ) objectClasses: ( 2.16.840.1.113730.3.2.110 NAME 'nsMappingTree' DESC 'Netscape defined objectclass' SUP top MUST ( CN ) X-ORIGIN 'Netscape Directory Server' )

1 0

ldap/servers
by Mark Reynolds 05 Apr '13

05 Apr '13

ldap/servers/plugins/replication/repl5_agmt.c | 28 ++++++++++++++-------- ldap/servers/plugins/replication/repl5_agmtlist.c | 23 +++++++++++------- 2 files changed, 32 insertions(+), 19 deletions(-) New commits: commit db6bcd7801a30abd40c09ea90e8ea7938983762f Author: Mark Reynolds <mreynolds(a)redhat.com> Date: Thu Apr 4 15:18:11 2013 -0400 Ticket 47304 - reinitialization of a master with a disabled agreement hangs Bug Description: When IPA tries to initialize a disabled agreement, the script hangs as the attribute nsds5beginReplicaRefresh is not removed, and always stays in "start" mode. Fix Description: Return an error 53, and error text, when trying to initialize a disabled agreement. Also improved the logging when an agmt or suffix is disabled. https://fedorahosted.org/389/ticket/47304 Reviewed by: richm(Thanks Rich!) diff --git a/ldap/servers/plugins/replication/repl5_agmt.c b/ldap/servers/plugins/replication/repl5_agmt.c index f25b7fb..b7d107e 100644 --- a/ldap/servers/plugins/replication/repl5_agmt.c +++ b/ldap/servers/plugins/replication/repl5_agmt.c @@ -2220,16 +2220,24 @@ agmt_set_last_init_status (Repl_Agmt *ra, int ldaprc, int replrc, const char *me } else if (replrc == NSDS50_REPL_DISABLED) { - PR_snprintf(ra->last_init_status, STATUS_LEN, "%d Total update aborted: " - "Replication agreement for %s\n can not be updated while the replica is disabled.\n" - "(If the suffix is disabled you must enable it then restart the server for replication to take place).", - replrc, ra->long_name ? ra->long_name : "a replica"); - /* Log into the errors log, as "ra->long_name" is not accessible from the caller */ - slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name, - "Total update aborted: Replication agreement for \"%s\" " - "can not be updated while the replica is disabled\n", ra->long_name ? ra->long_name : "a replica"); - slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name, - "(If the suffix is disabled you must enable it then restart the server for replication to take place).\n"); + if(agmt_is_enabled(ra)){ + slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name, "Total update aborted: " + "Replication agreement for \"%s\" can not be updated while the suffix is disabled.\n" + "You must enable it then restart the server for replication to take place).\n", + ra->long_name ? ra->long_name : "a replica"); + PR_snprintf(ra->last_init_status, STATUS_LEN, "%d Total update aborted: " + "Replication agreement for \"%s\" can not be updated while the suffix is disabled.\n" + "You must enable it then restart the server for replication to take place).", + replrc, ra->long_name ? ra->long_name : "a replica"); + } else { + /* You do not need to restart the server after enabling the agreement */ + slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name, "Total update aborted: " + "Replication agreement for \"%s\" can not be updated while the agreement is disabled\n", + ra->long_name ? ra->long_name : "a replica"); + PR_snprintf(ra->last_init_status, STATUS_LEN, "%d Total update aborted: " + "Replication agreement for \"%s\" can not be updated while the agreement is disabled.", + replrc, ra->long_name ? ra->long_name : "a replica"); + } } else { diff --git a/ldap/servers/plugins/replication/repl5_agmtlist.c b/ldap/servers/plugins/replication/repl5_agmtlist.c index 70f71a8..1167b0c 100644 --- a/ldap/servers/plugins/replication/repl5_agmtlist.c +++ b/ldap/servers/plugins/replication/repl5_agmtlist.c @@ -302,10 +302,8 @@ agmtlist_modify_callback(Slapi_PBlock *pb, Slapi_Entry *entryBefore, Slapi_Entry { PR_snprintf (errortext, SLAPI_DSE_RETURNTEXT_SIZE, "Invalid value (%s) value supplied for attr (%s)", val, mods[i]->mod_type); - slapi_log_error(SLAPI_LOG_REPL, repl_plugin_name, "agmtlist_modify_callback: %s\n", - errortext); + slapi_log_error(SLAPI_LOG_REPL, repl_plugin_name, "agmtlist_modify_callback: %s\n", errortext); } - slapi_ch_free ((void**)&val); } } @@ -529,15 +527,22 @@ agmtlist_modify_callback(Slapi_PBlock *pb, Slapi_Entry *entryBefore, Slapi_Entry } else if (start_initialize) { - if (agmt_initialize_replica(agmt) != 0) { - /* The suffix is disabled */ - agmt_set_last_init_status(agmt, 0, NSDS50_REPL_DISABLED, NULL); - } + if (agmt_initialize_replica(agmt) != 0) { + /* The suffix/repl agmt is disabled */ + agmt_set_last_init_status(agmt, 0, NSDS50_REPL_DISABLED, NULL); + if(agmt_is_enabled(agmt)){ + PR_snprintf(returntext, SLAPI_DSE_RETURNTEXT_SIZE, "Suffix is disabled"); + } else { + PR_snprintf(returntext, SLAPI_DSE_RETURNTEXT_SIZE, "Replication agreement is disabled"); + } + *returncode = LDAP_UNWILLING_TO_PERFORM; + rc = SLAPI_DSE_CALLBACK_ERROR; + } } else if (cancel_initialize) { - agmt_replica_init_done(agmt); - } + agmt_replica_init_done(agmt); + } if (update_the_schedule) {

1 0

ldap/servers
by Richard Allen Megginson 03 Apr '13

03 Apr '13

ldap/servers/slapd/back-ldbm/dblayer.c | 2 - ldap/servers/slapd/back-ldbm/proto-back-ldbm.h | 2 - ldap/servers/slapd/back-ldbm/start.c | 28 ++++++++++++------------- 3 files changed, 16 insertions(+), 16 deletions(-) New commits: commit 9d5deddef69f599103a94cac90c4e49016f68ccf Author: Rich Megginson <rmeggins(a)redhat.com> Date: Wed Apr 3 16:19:30 2013 -0600 Ticket #47312 - replace PR_GetFileInfo with PR_GetFileInfo64 https://fedorahosted.org/389/ticket/47312 Reviewed by: nhosoi (Thanks!) Branch: master Fix Description: Since we are returning a PRUint64 for the db file size, we have to use PRUint64 everywhere - on a 32-bit system, we can have 64-bit sized files. We also have to use the NSPR printf format macro to print the 64-bit values. Platforms tested: RHEL6 x86_64 Flag Day: no Doc impact: no diff --git a/ldap/servers/slapd/back-ldbm/dblayer.c b/ldap/servers/slapd/back-ldbm/dblayer.c index 0edd861..e272cfa 100644 --- a/ldap/servers/slapd/back-ldbm/dblayer.c +++ b/ldap/servers/slapd/back-ldbm/dblayer.c @@ -1919,7 +1919,7 @@ check_and_set_import_cache(struct ldbminfo *li) return 0; } -size_t +PRUint64 dblayer_get_id2entry_size(ldbm_instance *inst) { struct ldbminfo *li = NULL; diff --git a/ldap/servers/slapd/back-ldbm/proto-back-ldbm.h b/ldap/servers/slapd/back-ldbm/proto-back-ldbm.h index a0c3a42..e87c900 100644 --- a/ldap/servers/slapd/back-ldbm/proto-back-ldbm.h +++ b/ldap/servers/slapd/back-ldbm/proto-back-ldbm.h @@ -180,7 +180,7 @@ int dblayer_db_uses_mpool(DB_ENV *db_env); int dblayer_db_uses_logging(DB_ENV *db_env); int dblayer_bt_compare(DB *db, const DBT *dbt1, const DBT *dbt2); int dblayer_remove_env(struct ldbminfo *li); -size_t dblayer_get_id2entry_size(ldbm_instance *inst); +PRUint64 dblayer_get_id2entry_size(ldbm_instance *inst); int ldbm_back_get_info(Slapi_Backend *be, int cmd, void **info); int ldbm_back_set_info(Slapi_Backend *be, int cmd, void *info); diff --git a/ldap/servers/slapd/back-ldbm/start.c b/ldap/servers/slapd/back-ldbm/start.c index 4276c3b..4500ad7 100644 --- a/ldap/servers/slapd/back-ldbm/start.c +++ b/ldap/servers/slapd/back-ldbm/start.c @@ -154,38 +154,38 @@ ldbm_back_start( Slapi_PBlock *pb ) int zone_pages, db_pages, entry_pages, import_pages; Object *inst_obj; ldbm_instance *inst; - size_t cache_size; - size_t db_size; - size_t total_cache_size = 0; - size_t memsize = pages * pagesize; - size_t extra = 0; /* e.g., dncache size */ + PRUint64 cache_size; + PRUint64 db_size; + PRUint64 total_cache_size = 0; + PRUint64 memsize = pages * pagesize; + PRUint64 extra = 0; /* e.g., dncache size */ for (inst_obj = objset_first_obj(li->li_instance_set); inst_obj; inst_obj = objset_next_obj(li->li_instance_set, inst_obj)) { inst = (ldbm_instance *)object_get_data(inst_obj); - cache_size = cache_get_max_size(&(inst->inst_cache)); + cache_size = (PRUint64)cache_get_max_size(&(inst->inst_cache)); db_size = dblayer_get_id2entry_size(inst); if (cache_size < db_size) { LDAPDebug(LDAP_DEBUG_ANY, - "WARNING: %s: entry cache size %luB is " - "less than db size %luB; " + "WARNING: %s: entry cache size %" NSPRIu64 "B is " + "less than db size %" NSPRIu64 "B; " "We recommend to increase the entry cache size " "nsslapd-cachememsize.\n", inst->inst_name, cache_size, db_size); } else { LDAPDebug(LDAP_DEBUG_BACKLDBM, - "%s: entry cache size: %luB; db size: %luB\n", + "%s: entry cache size: %" NSPRIu64 "B; db size: %" NSPRIu64 "B\n", inst->inst_name, cache_size, db_size); } total_cache_size += cache_size; /* estimated overhead: dncache size * 2 */ - extra += cache_get_max_size(&(inst->inst_dncache)) * 2; + extra += (PRUint64)cache_get_max_size(&(inst->inst_dncache)) * 2; } LDAPDebug(LDAP_DEBUG_BACKLDBM, - "Total entry cache size: %luB; " - "dbcache size: %luB; " - "available memory size: %luB\n", - total_cache_size, li->li_dbcachesize, memsize - extra); + "Total entry cache size: %" NSPRIu64 "B; " + "dbcache size: %" NSPRIu64 "B; " + "available memory size: %" NSPRIu64 "B\n", + total_cache_size, (PRUint32)li->li_dbcachesize, memsize - extra); /* autosizing dbCache and entryCache */ if (li->li_cache_autosize > 0) { zone_pages = (li->li_cache_autosize * pages) / 100;

1 0

2026

2025

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

2009

2008

2007

2006

2005

389-commits April 2013