10:22 a.m.
389-Directory/1.2.11.15 B2013.238.2155
Nothing in errors, nothing in access log files
uname -a
Linux XXXX 2.6.32-358.18.1.el6.x86_64 #1 SMP Fri Aug 2 17:04:38 EDT 2013 x86_64 x86_64
x86_64 GNU/Linux
yum list | grep 389
389-admin.x86_64 1.1.29-1.el6 @epel-x86_64-server-6
389-admin-console.noarch 1.1.8-1.el6 @epel-x86_64-server-6
389-admin-console-doc.noarch 1.1.8-1.el6 @epel-x86_64-server-6
389-adminutil.x86_64 1.1.15-1.el6 installed
389-console.noarch 1.1.7-3.el5 installed
389-ds.noarch 1.2.2-1.el6 @epel-x86_64-server-6
389-ds-base.x86_64 1.2.11.15-22.el6_4 @rhel-x86_64-server-6
389-ds-base-debuginfo.x86_64 1.2.11.15-22.el6_4 @rhel-x86_64-server-6-debuginfo
389-ds-base-libs.x86_64 1.2.11.15-22.el6_4 @rhel-x86_64-server-6
389-ds-console.noarch 1.2.6-1.el6 @epel-x86_64-server-6
389-ds-console-doc.noarch 1.2.6-1.el6 @epel-x86_64-server-6
389-dsgw.x86_64 1.1.10-1.el6 @epel-x86_64-server-6
389-admin.i686 1.1.29-1.el6 epel-x86_64-server-6
389-adminutil.i686 1.1.15-1.el6 epel-x86_64-server-6
389-adminutil-devel.i686 1.1.15-1.el6 epel-x86_64-server-6
389-adminutil-devel.x86_64 1.1.15-1.el6 epel-x86_64-server-6
389-ds-base-debuginfo.i686 1.2.11.15-22.el6_4 rhel-x86_64-server-6-debuginfo
389-ds-base-devel.i686 1.2.11.15-22.el6_4 rhel-x86_64-server-optional-6
389-ds-base-devel.x86_64 1.2.11.15-22.el6_4 rhel-x86_64-server-optional-6
389-ds-base-libs.i686 1.2.11.15-22.el6_4 rhel-x86_64-server-6
I have a gcore of ns-slapd
#0 __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:136
#1 0x00007f6f9c6af3be in _L_lock_995 () from /lib64/libpthread.so.0
#2 0x00007f6f9c6af326 in __pthread_mutex_lock (mutex=0x18c8850) at
pthread_mutex_lock.c:101
#3 0x00007f6f9cd050b9 in PR_Lock (lock=0x18c8850) at
../../../mozilla/nsprpub/pr/src/pthreads/ptsynch.c:174
#4 0x00007f6f9d390bf0 in nssCertificate_Destroy (c=0x1a0f3f0) at certificate.c:112
#5 0x00007f6f9d684e97 in ssl_ResetSecurityInfo (sec=0x2774888, doMemset=0) at
sslsecur.c:947
#6 0x00007f6f9d684f1b in ssl_DestroySecurityInfo (sec=0x2774888) at sslsecur.c:978
#7 0x00007f6f9d6891e5 in ssl_DestroySocketContents (ss=0x2774800) at sslsock.c:404
#8 0x00007f6f9d68a752 in ssl_FreeSocket (ss=0x2774800) at sslsock.c:465
#9 0x00007f6f9d6808b8 in ssl_DefClose (ss=0x2774800) at ssldef.c:206
#10 0x0000000000414301 in connection_cleanup (conn=0x7f6f80753670) at
ldap/servers/slapd/connection.c:167
#11 0x0000000000415371 in connection_table_move_connection_out_of_active_list
(ct=0x1ca7bc0, c=0x7f6f80753670) at ldap/servers/slapd/conntable.c:322
#12 0x0000000000417d66 in setup_pr_read_pds (ports=0x7fff04c632e0) at
ldap/servers/slapd/daemon.c:1702
#13 slapd_daemon (ports=0x7fff04c632e0) at ldap/servers/slapd/daemon.c:1137
#14 0x000000000041f0df in main (argc=7, argv=0x7fff04c63678) at
ldap/servers/slapd/main.c:
did a kill -9 of the server (normal kill had no effect). Server is now back up and
running.
/mrg
10:52 a.m.
On 10/09/2013 09:22 AM, Michael R. Gettes wrote:
389-Directory/1.2.11.15 B2013.238.2155
Nothing in errors, nothing in access log files
uname -a
Linux XXXX 2.6.32-358.18.1.el6.x86_64 #1 SMP Fri Aug 2 17:04:38 EDT 2013 x86_64 x86_64
x86_64 GNU/Linux
yum list | grep 389
389-admin.x86_64 1.1.29-1.el6 @epel-x86_64-server-6
389-admin-console.noarch 1.1.8-1.el6 @epel-x86_64-server-6
389-admin-console-doc.noarch 1.1.8-1.el6 @epel-x86_64-server-6
389-adminutil.x86_64 1.1.15-1.el6 installed
389-console.noarch 1.1.7-3.el5 installed
389-ds.noarch 1.2.2-1.el6 @epel-x86_64-server-6
389-ds-base.x86_64 1.2.11.15-22.el6_4 @rhel-x86_64-server-6
389-ds-base-debuginfo.x86_64 1.2.11.15-22.el6_4 @rhel-x86_64-server-6-debuginfo
389-ds-base-libs.x86_64 1.2.11.15-22.el6_4 @rhel-x86_64-server-6
389-ds-console.noarch 1.2.6-1.el6 @epel-x86_64-server-6
389-ds-console-doc.noarch 1.2.6-1.el6 @epel-x86_64-server-6
389-dsgw.x86_64 1.1.10-1.el6 @epel-x86_64-server-6
389-admin.i686 1.1.29-1.el6 epel-x86_64-server-6
389-adminutil.i686 1.1.15-1.el6 epel-x86_64-server-6
389-adminutil-devel.i686 1.1.15-1.el6 epel-x86_64-server-6
389-adminutil-devel.x86_64 1.1.15-1.el6 epel-x86_64-server-6
389-ds-base-debuginfo.i686 1.2.11.15-22.el6_4 rhel-x86_64-server-6-debuginfo
389-ds-base-devel.i686 1.2.11.15-22.el6_4 rhel-x86_64-server-optional-6
389-ds-base-devel.x86_64 1.2.11.15-22.el6_4 rhel-x86_64-server-optional-6
389-ds-base-libs.i686 1.2.11.15-22.el6_4 rhel-x86_64-server-6
I have a gcore of ns-slapd
#0 __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:136
#1 0x00007f6f9c6af3be in _L_lock_995 () from /lib64/libpthread.so.0
#2 0x00007f6f9c6af326 in __pthread_mutex_lock (mutex=0x18c8850) at
pthread_mutex_lock.c:101
#3 0x00007f6f9cd050b9 in PR_Lock (lock=0x18c8850) at
../../../mozilla/nsprpub/pr/src/pthreads/ptsynch.c:174
#4 0x00007f6f9d390bf0 in nssCertificate_Destroy (c=0x1a0f3f0) at certificate.c:112
#5 0x00007f6f9d684e97 in ssl_ResetSecurityInfo (sec=0x2774888, doMemset=0) at
sslsecur.c:947
#6 0x00007f6f9d684f1b in ssl_DestroySecurityInfo (sec=0x2774888) at sslsecur.c:978
#7 0x00007f6f9d6891e5 in ssl_DestroySocketContents (ss=0x2774800) at sslsock.c:404
#8 0x00007f6f9d68a752 in ssl_FreeSocket (ss=0x2774800) at sslsock.c:465
#9 0x00007f6f9d6808b8 in ssl_DefClose (ss=0x2774800) at ssldef.c:206
#10 0x0000000000414301 in connection_cleanup (conn=0x7f6f80753670) at
ldap/servers/slapd/connection.c:167
#11 0x0000000000415371 in connection_table_move_connection_out_of_active_list
(ct=0x1ca7bc0, c=0x7f6f80753670) at ldap/servers/slapd/conntable.c:322
#12 0x0000000000417d66 in setup_pr_read_pds (ports=0x7fff04c632e0) at
ldap/servers/slapd/daemon.c:1702
#13 slapd_daemon (ports=0x7fff04c632e0) at ldap/servers/slapd/daemon.c:1137
#14 0x000000000041f0df in main (argc=7, argv=0x7fff04c63678) at
ldap/servers/slapd/main.c:
This looks like a deadlock. Unfortunately, you didn't show what the
other threads are doing. Use
gdb -ex 'set confirm off' -ex 'set pagination off' -ex 'thread apply
all
bt full' -ex 'quit' /usr/sbin/ns-slapd
/var/log/dirsrv/slapd-INSTANCENAME/core.PID > stacktrace.`date +%s`.txt 2>&1
to get a full stack trace with all threads. Not sure if gcore writes
core files using /var/log/dirsrv/slapd-INSTANCENAME/core.PID - if not,
then use whatever core file gcore generates.
Also, if slapd is hung, it is not necessary to use gcore to generate a
core file for the sole purpose of getting a stack trace. You can use
gdb to get a stack trace of the running process like this:
gdb -ex 'set confirm off' -ex 'set pagination off' -ex 'thread apply
all
bt full' -ex 'quit' /usr/sbin/ns-slapd `pidof ns-slapd` >
stacktrace.`date +%s`.txt 2>&1
did a kill -9 of the server (normal kill had no effect). Server is now back up and
running.
/mrg
--
389 users mailing list
389-users(a)lists.fedoraproject.org
https://admin.fedoraproject.org/mailman/listinfo/389-users
11:26 a.m.
Here is the stack trace per your instructions…
i will be sure to get such traces in the future. got the gcore for just in case.
/mrg
On Oct 9, 2013, at 11:22 AM, Michael R. Gettes <gettes(a)gmail.com> wrote:
389-Directory/1.2.11.15 B2013.238.2155
Nothing in errors, nothing in access log files
uname -a
Linux XXXX 2.6.32-358.18.1.el6.x86_64 #1 SMP Fri Aug 2 17:04:38 EDT 2013 x86_64 x86_64
x86_64 GNU/Linux
yum list | grep 389
389-admin.x86_64 1.1.29-1.el6 @epel-x86_64-server-6
389-admin-console.noarch 1.1.8-1.el6 @epel-x86_64-server-6
389-admin-console-doc.noarch 1.1.8-1.el6 @epel-x86_64-server-6
389-adminutil.x86_64 1.1.15-1.el6 installed
389-console.noarch 1.1.7-3.el5 installed
389-ds.noarch 1.2.2-1.el6 @epel-x86_64-server-6
389-ds-base.x86_64 1.2.11.15-22.el6_4 @rhel-x86_64-server-6
389-ds-base-debuginfo.x86_64 1.2.11.15-22.el6_4 @rhel-x86_64-server-6-debuginfo
389-ds-base-libs.x86_64 1.2.11.15-22.el6_4 @rhel-x86_64-server-6
389-ds-console.noarch 1.2.6-1.el6 @epel-x86_64-server-6
389-ds-console-doc.noarch 1.2.6-1.el6 @epel-x86_64-server-6
389-dsgw.x86_64 1.1.10-1.el6 @epel-x86_64-server-6
389-admin.i686 1.1.29-1.el6 epel-x86_64-server-6
389-adminutil.i686 1.1.15-1.el6 epel-x86_64-server-6
389-adminutil-devel.i686 1.1.15-1.el6 epel-x86_64-server-6
389-adminutil-devel.x86_64 1.1.15-1.el6 epel-x86_64-server-6
389-ds-base-debuginfo.i686 1.2.11.15-22.el6_4 rhel-x86_64-server-6-debuginfo
389-ds-base-devel.i686 1.2.11.15-22.el6_4 rhel-x86_64-server-optional-6
389-ds-base-devel.x86_64 1.2.11.15-22.el6_4 rhel-x86_64-server-optional-6
389-ds-base-libs.i686 1.2.11.15-22.el6_4 rhel-x86_64-server-6
I have a gcore of ns-slapd
#0 __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:136
#1 0x00007f6f9c6af3be in _L_lock_995 () from /lib64/libpthread.so.0
#2 0x00007f6f9c6af326 in __pthread_mutex_lock (mutex=0x18c8850) at
pthread_mutex_lock.c:101
#3 0x00007f6f9cd050b9 in PR_Lock (lock=0x18c8850) at
../../../mozilla/nsprpub/pr/src/pthreads/ptsynch.c:174
#4 0x00007f6f9d390bf0 in nssCertificate_Destroy (c=0x1a0f3f0) at certificate.c:112
#5 0x00007f6f9d684e97 in ssl_ResetSecurityInfo (sec=0x2774888, doMemset=0) at
sslsecur.c:947
#6 0x00007f6f9d684f1b in ssl_DestroySecurityInfo (sec=0x2774888) at sslsecur.c:978
#7 0x00007f6f9d6891e5 in ssl_DestroySocketContents (ss=0x2774800) at sslsock.c:404
#8 0x00007f6f9d68a752 in ssl_FreeSocket (ss=0x2774800) at sslsock.c:465
#9 0x00007f6f9d6808b8 in ssl_DefClose (ss=0x2774800) at ssldef.c:206
#10 0x0000000000414301 in connection_cleanup (conn=0x7f6f80753670) at
ldap/servers/slapd/connection.c:167
#11 0x0000000000415371 in connection_table_move_connection_out_of_active_list
(ct=0x1ca7bc0, c=0x7f6f80753670) at ldap/servers/slapd/conntable.c:322
#12 0x0000000000417d66 in setup_pr_read_pds (ports=0x7fff04c632e0) at
ldap/servers/slapd/daemon.c:1702
#13 slapd_daemon (ports=0x7fff04c632e0) at ldap/servers/slapd/daemon.c:1137
#14 0x000000000041f0df in main (argc=7, argv=0x7fff04c63678) at
ldap/servers/slapd/main.c:
did a kill -9 of the server (normal kill had no effect). Server is now back up and
running.
/mrg
8:59 p.m.
On 10/09/2013 10:26 AM, Michael Gettes wrote:
Here is the stack trace per your instructions…
i will be sure to get such traces in the future. got the gcore for just in case.
/mrg
On Oct 9, 2013, at 11:22 AM, Michael R. Gettes <gettes(a)gmail.com> wrote:
> 389-Directory/1.2.11.15 B2013.238.2155
>
> Nothing in errors, nothing in access log files
>
> uname -a
> Linux XXXX 2.6.32-358.18.1.el6.x86_64 #1 SMP Fri Aug 2 17:04:38 EDT 2013 x86_64
x86_64 x86_64 GNU/Linux
>
>
> yum list | grep 389
> 389-admin.x86_64 1.1.29-1.el6 @epel-x86_64-server-6
> 389-admin-console.noarch 1.1.8-1.el6 @epel-x86_64-server-6
> 389-admin-console-doc.noarch 1.1.8-1.el6 @epel-x86_64-server-6
> 389-adminutil.x86_64 1.1.15-1.el6 installed
> 389-console.noarch 1.1.7-3.el5 installed
> 389-ds.noarch 1.2.2-1.el6 @epel-x86_64-server-6
> 389-ds-base.x86_64 1.2.11.15-22.el6_4 @rhel-x86_64-server-6
> 389-ds-base-debuginfo.x86_64 1.2.11.15-22.el6_4 @rhel-x86_64-server-6-debuginfo
> 389-ds-base-libs.x86_64 1.2.11.15-22.el6_4 @rhel-x86_64-server-6
> 389-ds-console.noarch 1.2.6-1.el6 @epel-x86_64-server-6
> 389-ds-console-doc.noarch 1.2.6-1.el6 @epel-x86_64-server-6
> 389-dsgw.x86_64 1.1.10-1.el6 @epel-x86_64-server-6
> 389-admin.i686 1.1.29-1.el6 epel-x86_64-server-6
> 389-adminutil.i686 1.1.15-1.el6 epel-x86_64-server-6
> 389-adminutil-devel.i686 1.1.15-1.el6 epel-x86_64-server-6
> 389-adminutil-devel.x86_64 1.1.15-1.el6 epel-x86_64-server-6
> 389-ds-base-debuginfo.i686 1.2.11.15-22.el6_4 rhel-x86_64-server-6-debuginfo
> 389-ds-base-devel.i686 1.2.11.15-22.el6_4 rhel-x86_64-server-optional-6
> 389-ds-base-devel.x86_64 1.2.11.15-22.el6_4 rhel-x86_64-server-optional-6
> 389-ds-base-libs.i686 1.2.11.15-22.el6_4 rhel-x86_64-server-6
>
> I have a gcore of ns-slapd
>
> #0 __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:136
> #1 0x00007f6f9c6af3be in _L_lock_995 () from /lib64/libpthread.so.0
> #2 0x00007f6f9c6af326 in __pthread_mutex_lock (mutex=0x18c8850) at
pthread_mutex_lock.c:101
> #3 0x00007f6f9cd050b9 in PR_Lock (lock=0x18c8850) at
../../../mozilla/nsprpub/pr/src/pthreads/ptsynch.c:174
> #4 0x00007f6f9d390bf0 in nssCertificate_Destroy (c=0x1a0f3f0) at certificate.c:112
> #5 0x00007f6f9d684e97 in ssl_ResetSecurityInfo (sec=0x2774888, doMemset=0) at
sslsecur.c:947
> #6 0x00007f6f9d684f1b in ssl_DestroySecurityInfo (sec=0x2774888) at sslsecur.c:978
> #7 0x00007f6f9d6891e5 in ssl_DestroySocketContents (ss=0x2774800) at sslsock.c:404
> #8 0x00007f6f9d68a752 in ssl_FreeSocket (ss=0x2774800) at sslsock.c:465
> #9 0x00007f6f9d6808b8 in ssl_DefClose (ss=0x2774800) at ssldef.c:206
> #10 0x0000000000414301 in connection_cleanup (conn=0x7f6f80753670) at
ldap/servers/slapd/connection.c:167
> #11 0x0000000000415371 in connection_table_move_connection_out_of_active_list
(ct=0x1ca7bc0, c=0x7f6f80753670) at ldap/servers/slapd/conntable.c:322
> #12 0x0000000000417d66 in setup_pr_read_pds (ports=0x7fff04c632e0) at
ldap/servers/slapd/daemon.c:1702
> #13 slapd_daemon (ports=0x7fff04c632e0) at ldap/servers/slapd/daemon.c:1137
> #14 0x000000000041f0df in main (argc=7, argv=0x7fff04c63678) at
ldap/servers/slapd/main.c:
>
> did a kill -9 of the server (normal kill had no effect). Server is now back up and
running.
>
> /mrg
Thanks. Please open a ticket. This looks like
https://fedorahosted.org/389/ticket/348
Looks like we need to put a mutex around ldap_sasl_bind in addition to
the mutex around ldap_initialize.
--
389 users mailing list
389-users(a)lists.fedoraproject.org
https://admin.fedoraproject.org/mailman/listinfo/389-users
7:55 a.m.
Done.
/mrg
On Oct 10, 2013, at 21:59, Rich Megginson <rmeggins(a)redhat.com> wrote:
On 10/09/2013 10:26 AM, Michael Gettes wrote:
> Here is the stack trace per your instructions…
>
> i will be sure to get such traces in the future. got the gcore for just in case.
>
> /mrg
>
>
>
>
> On Oct 9, 2013, at 11:22 AM, Michael R. Gettes <gettes(a)gmail.com> wrote:
>
>> 389-Directory/1.2.11.15 B2013.238.2155
>>
>> Nothing in errors, nothing in access log files
>>
>> uname -a
>> Linux XXXX 2.6.32-358.18.1.el6.x86_64 #1 SMP Fri Aug 2 17:04:38 EDT 2013 x86_64
x86_64 x86_64 GNU/Linux
>>
>>
>> yum list | grep 389
>> 389-admin.x86_64 1.1.29-1.el6 @epel-x86_64-server-6
>> 389-admin-console.noarch 1.1.8-1.el6 @epel-x86_64-server-6
>> 389-admin-console-doc.noarch 1.1.8-1.el6 @epel-x86_64-server-6
>> 389-adminutil.x86_64 1.1.15-1.el6 installed
>> 389-console.noarch 1.1.7-3.el5 installed
>> 389-ds.noarch 1.2.2-1.el6 @epel-x86_64-server-6
>> 389-ds-base.x86_64 1.2.11.15-22.el6_4 @rhel-x86_64-server-6
>> 389-ds-base-debuginfo.x86_64 1.2.11.15-22.el6_4
@rhel-x86_64-server-6-debuginfo
>> 389-ds-base-libs.x86_64 1.2.11.15-22.el6_4 @rhel-x86_64-server-6
>> 389-ds-console.noarch 1.2.6-1.el6 @epel-x86_64-server-6
>> 389-ds-console-doc.noarch 1.2.6-1.el6 @epel-x86_64-server-6
>> 389-dsgw.x86_64 1.1.10-1.el6 @epel-x86_64-server-6
>> 389-admin.i686 1.1.29-1.el6 epel-x86_64-server-6
>> 389-adminutil.i686 1.1.15-1.el6 epel-x86_64-server-6
>> 389-adminutil-devel.i686 1.1.15-1.el6 epel-x86_64-server-6
>> 389-adminutil-devel.x86_64 1.1.15-1.el6 epel-x86_64-server-6
>> 389-ds-base-debuginfo.i686 1.2.11.15-22.el6_4 rhel-x86_64-server-6-debuginfo
>> 389-ds-base-devel.i686 1.2.11.15-22.el6_4 rhel-x86_64-server-optional-6
>> 389-ds-base-devel.x86_64 1.2.11.15-22.el6_4 rhel-x86_64-server-optional-6
>> 389-ds-base-libs.i686 1.2.11.15-22.el6_4 rhel-x86_64-server-6
>>
>> I have a gcore of ns-slapd
>>
>> #0 __lll_lock_wait () at
../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:136
>> #1 0x00007f6f9c6af3be in _L_lock_995 () from /lib64/libpthread.so.0
>> #2 0x00007f6f9c6af326 in __pthread_mutex_lock (mutex=0x18c8850) at
pthread_mutex_lock.c:101
>> #3 0x00007f6f9cd050b9 in PR_Lock (lock=0x18c8850) at
../../../mozilla/nsprpub/pr/src/pthreads/ptsynch.c:174
>> #4 0x00007f6f9d390bf0 in nssCertificate_Destroy (c=0x1a0f3f0) at
certificate.c:112
>> #5 0x00007f6f9d684e97 in ssl_ResetSecurityInfo (sec=0x2774888, doMemset=0) at
sslsecur.c:947
>> #6 0x00007f6f9d684f1b in ssl_DestroySecurityInfo (sec=0x2774888) at
sslsecur.c:978
>> #7 0x00007f6f9d6891e5 in ssl_DestroySocketContents (ss=0x2774800) at
sslsock.c:404
>> #8 0x00007f6f9d68a752 in ssl_FreeSocket (ss=0x2774800) at sslsock.c:465
>> #9 0x00007f6f9d6808b8 in ssl_DefClose (ss=0x2774800) at ssldef.c:206
>> #10 0x0000000000414301 in connection_cleanup (conn=0x7f6f80753670) at
ldap/servers/slapd/connection.c:167
>> #11 0x0000000000415371 in connection_table_move_connection_out_of_active_list
(ct=0x1ca7bc0, c=0x7f6f80753670) at ldap/servers/slapd/conntable.c:322
>> #12 0x0000000000417d66 in setup_pr_read_pds (ports=0x7fff04c632e0) at
ldap/servers/slapd/daemon.c:1702
>> #13 slapd_daemon (ports=0x7fff04c632e0) at ldap/servers/slapd/daemon.c:1137
>> #14 0x000000000041f0df in main (argc=7, argv=0x7fff04c63678) at
ldap/servers/slapd/main.c:
>>
>> did a kill -9 of the server (normal kill had no effect). Server is now back up
and running.
>>
>> /mrg
Thanks. Please open a ticket. This looks like https://fedorahosted.org/389/ticket/348
Looks like we need to put a mutex around ldap_sasl_bind in addition to the mutex around
ldap_initialize.
>
>
> --
> 389 users mailing list
> 389-users(a)lists.fedoraproject.org
> https://admin.fedoraproject.org/mailman/listinfo/389-users
3847
days inactive
3849
days old
389-users@lists.fedoraproject.org
4 comments
2 participants
participants (2)
-
Michael Gettes -
Rich Megginson