[389-commits] Branch '389-ds-base-1.3.2' - ldap/servers
Noriko Hosoi
nhosoi at fedoraproject.org
Mon Dec 16 21:22:11 UTC 2013
ldap/servers/plugins/replication/repl5_tot_protocol.c | 3 +
ldap/servers/slapd/back-ldbm/import-threads.c | 8 ++--
ldap/servers/slapd/connection.c | 36 +++++++++++++++---
ldap/servers/slapd/openldapber.h | 25 ++++++++++++
4 files changed, 62 insertions(+), 10 deletions(-)
New commits:
commit fde9ed5bf74b4ea1fff875bcb421137c78af1227
Author: Noriko Hosoi <nhosoi at redhat.com>
Date: Mon Dec 16 13:03:19 2013 -0800
Ticket #47606 - replica init/bulk import errors should be more verbose
Description:
1. maxbersize: If the size of an entry is larger than the consumer's
maxbersize, the following error used to be logged:
Incoming BER Element was too long, max allowable is ### bytes.
Change the nsslapd-maxbersize attribute in cn=config to increase.
This message does not indicate how large the maxbersize needs to be.
This patch adds the code to retrieve the failed ber size.
Revised message:
Incoming BER Element was @@@ bytes, max allowable is ### bytes.
Change the nsslapd-maxbersize attribute in cn=config to increase.
Note: There is no lber API that returns the ber size if it fails to
handle the ber. This patch borrows the internal structure of ber
and get the size. This could be risky since the size or structure
of the ber could be updated in the openldap/mozldap lber.
2. cache size: The bulk import depends upon the nsslapd-cachememsize
value in the backend instance entry (e.g., cn=userRoot,cn=ldbm
database,cn=plugins,cn=config). If an entry size is larger than
the cachememsize, the bulk import used to fail with this message:
import userRoot: REASON: entry too large (@@@ bytes) for the
import buffer size (### bytes). Try increasing nsslapd-
cachememsize.
Also, the message follows the skipping entry message:
import userRoot: WARNING: skipping entry "<DN>"
but actually, it did NOT "skip" the entry and continue the bulk
import, but it failed there and completely wiped out the backend
database.
This patch modifies the message as follows:
import userRoot: REASON: entry too large (@@@ bytes) for the
effective import buffer size (### bytes). Try increasing nsslapd-
cachememsize for the backend instance "userRoot".
and as the message mentions, it just skips the failed entry and
continues the bulk import.
3. In repl5_tot_result_threadmain, when conn_read_result_ex returns
non zero (non SUCCESS), it sets abort, but does not set any error
code to rc (return code), which is not considered as "finished" in
repl5_tot_waitfor_async_results and it contines waiting until the
code reaches the max loop count (about 5 minutes). This patch sets
LDAP_CONNECT_ERROR to the return code along with setting abort, if
conn_read_result_ex returns CONN_NOT_CONNECTED. This makes the bulk
import finishes quickly when it fails.
https://fedorahosted.org/389/ticket/47606
Reviewed by rmeggins at redhat.com (Thank you, Rich!!)
(cherry picked from commit 1119083d3d99993421609783efcb8962d78724fc)
diff --git a/ldap/servers/plugins/replication/repl5_tot_protocol.c b/ldap/servers/plugins/replication/repl5_tot_protocol.c
index a241128..3895ace 100644
--- a/ldap/servers/plugins/replication/repl5_tot_protocol.c
+++ b/ldap/servers/plugins/replication/repl5_tot_protocol.c
@@ -203,6 +203,9 @@ static void repl5_tot_result_threadmain(void *param)
/* If so then we need to take steps to abort the update process */
PR_Lock(cb->lock);
cb->abort = 1;
+ if (conres == CONN_NOT_CONNECTED) {
+ cb->rc = LDAP_CONNECT_ERROR;
+ }
PR_Unlock(cb->lock);
}
/* Should we stop ? */
diff --git a/ldap/servers/slapd/back-ldbm/import-threads.c b/ldap/servers/slapd/back-ldbm/import-threads.c
index 3a3aab8..9705d9e 100644
--- a/ldap/servers/slapd/back-ldbm/import-threads.c
+++ b/ldap/servers/slapd/back-ldbm/import-threads.c
@@ -3330,11 +3330,11 @@ static int bulk_import_queue(ImportJob *job, Slapi_Entry *entry)
newesize = (slapi_entry_size(ep->ep_entry) + sizeof(struct backentry));
if (newesize > job->fifo.bsize) { /* entry too big */
- import_log_notice(job, "WARNING: skipping entry \"%s\"",
- slapi_entry_get_dn(ep->ep_entry));
import_log_notice(job, "REASON: entry too large (%lu bytes) for "
- "the import buffer size (%lu bytes). Try increasing nsslapd-cachememsize.",
- (long unsigned int)newesize, (long unsigned int)job->fifo.bsize);
+ "the effective import buffer size (%lu bytes). "
+ "Try increasing nsslapd-cachememsize for the backend instance \"%s\".",
+ (long unsigned int)newesize, (long unsigned int)job->fifo.bsize,
+ job->inst->inst_name);
backentry_clear_entry(ep); /* entry is released in the frontend on failure*/
backentry_free( &ep ); /* release the backend wrapper, here */
PR_Unlock(job->wire_lock);
diff --git a/ldap/servers/slapd/connection.c b/ldap/servers/slapd/connection.c
index 4397c2a..2d8c91a 100644
--- a/ldap/servers/slapd/connection.c
+++ b/ldap/servers/slapd/connection.c
@@ -1815,6 +1815,32 @@ int connection_wait_for_new_work(Slapi_PBlock *pb, PRIntervalTime interval)
return ret;
}
+#ifdef USE_OPENLDAP
+#include "openldapber.h"
+#else
+#include "mozldap.h"
+#endif
+
+static ber_tag_t
+_ber_get_len(BerElement *ber, ber_len_t *lenp)
+{
+#ifdef USE_OPENLDAP
+ OLBerElement *lber = (OLBerElement *)ber;
+#else
+ MozElement *lber = (MozElement *)ber;
+#endif
+
+ if (NULL == lenp) {
+ return LBER_DEFAULT;
+ }
+ *lenp = 0;
+ if (NULL == lber) {
+ return LBER_DEFAULT;
+ }
+ *lenp = lber->ber_len;
+ return lber->ber_tag;
+}
+
/*
* Utility function called by connection_read_operation(). This is a
* small wrapper on top of libldap's ber_get_next_buffer_ext().
@@ -1855,18 +1881,16 @@ get_next_from_buffer( void *buffer, size_t buffer_size, ber_len_t *lenp,
if ((LBER_OVERFLOW == *tagp || LBER_DEFAULT == *tagp) && 0 == bytes_scanned &&
!SLAPD_SYSTEM_WOULD_BLOCK_ERROR(errno))
{
- if (LBER_OVERFLOW == *tagp)
- {
- err = SLAPD_DISCONNECT_BER_TOO_BIG;
- }
- else if (errno == ERANGE)
+ if ((LBER_OVERFLOW == *tagp) || (errno == ERANGE))
{
ber_len_t maxbersize = config_get_maxbersize();
+ ber_len_t tmplen = 0;
+ (void)_ber_get_len(ber, &tmplen);
/* openldap does not differentiate between length == 0
and length > max - all we know is that there was a
problem with the length - assume too big */
err = SLAPD_DISCONNECT_BER_TOO_BIG;
- log_ber_too_big_error(conn, 0, maxbersize);
+ log_ber_too_big_error(conn, tmplen, maxbersize);
}
else
{
diff --git a/ldap/servers/slapd/openldapber.h b/ldap/servers/slapd/openldapber.h
new file mode 100644
index 0000000..52644a5
--- /dev/null
+++ b/ldap/servers/slapd/openldapber.h
@@ -0,0 +1,25 @@
+/*
+ * openldap lber library does not provide an API which returns the ber size
+ * (ber->ber_len) when the ber tag is LBER_DEFAULT or LBER_OVERFLOW.
+ * The ber size is useful when issuing an error message to indicate how
+ * large the maxbersize needs to be set.
+ * Borrowed from liblber/lber-int.h
+ */
+struct lber_options {
+ short lbo_valid;
+ unsigned short lbo_options;
+ int lbo_debug;
+};
+struct berelement {
+ struct lber_options ber_opts;
+ ber_tag_t ber_tag;
+ ber_len_t ber_len;
+ ber_tag_t ber_usertag;
+ char *ber_buf;
+ char *ber_ptr;
+ char *ber_end;
+ char *ber_sos_ptr;
+ char *ber_rwptr;
+ void *ber_memctx;
+};
+typedef struct berelement OLBerElement;
More information about the 389-commits
mailing list