3:45 p.m.
hi all,
we are running centos76 with ipa-server-4.6.4-10.el7 (one master and one
replica; the upgrade went fine on both) and we have a problem with pki
tomcat. (we are not sure since when this occurs, but it might be from
after the update)
ipactl status is ok on both master and replica, pki-tomcatd is running
(ports 8080, 8443, 8005 and 8009 are listening)
running 'ipa host-disable' fails with
Certificate operation cannot be completed: Unable to communicate with
CMS (500)
and the only hints i can find are in the
/var/log/pki/pki-tomcat/localhost.2019-01-08.log file (the .../ca/debug
has nothing relevant).
i pasted the backtrace below.
any help only how to further investiagte or debug are welcome.
stijn
> SEVERE: Servlet.service() for servlet [Resteasy] in context with path [/ca] threw
exception
> org.jboss.resteasy.spi.UnhandledException: Response is committed, can't handle
exception
> at
org.jboss.resteasy.core.SynchronousDispatcher.writeException(SynchronousDispatcher.java:148)
> at
org.jboss.resteasy.core.SynchronousDispatcher.writeResponse(SynchronousDispatcher.java:432)
> at
org.jboss.resteasy.core.SynchronousDispatcher.invoke(SynchronousDispatcher.java:376)
> at
org.jboss.resteasy.core.SynchronousDispatcher.invoke(SynchronousDispatcher.java:179)
> at
org.jboss.resteasy.plugins.server.servlet.ServletContainerDispatcher.service(ServletContainerDispatcher.java:220)
> at
org.jboss.resteasy.plugins.server.servlet.HttpServletDispatcher.service(HttpServletDispatcher.java:56)
> at
org.jboss.resteasy.plugins.server.servlet.HttpServletDispatcher.service(HttpServletDispatcher.java:51)
> at javax.servlet.http.HttpServlet.service(HttpServlet.java:731)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498)
> at org.apache.catalina.security.SecurityUtil$1.run(SecurityUtil.java:288)
> at org.apache.catalina.security.SecurityUtil$1.run(SecurityUtil.java:285)
> at java.security.AccessController.doPrivileged(Native Method)
> at javax.security.auth.Subject.doAsPrivileged(Subject.java:549)
> at org.apache.catalina.security.SecurityUtil.execute(SecurityUtil.java:320)
> at org.apache.catalina.security.SecurityUtil.doAsPrivilege(SecurityUtil.java:175)
> at
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:297)
> at
org.apache.catalina.core.ApplicationFilterChain.access$000(ApplicationFilterChain.java:55)
> at
org.apache.catalina.core.ApplicationFilterChain$1.run(ApplicationFilterChain.java:191)
> at
org.apache.catalina.core.ApplicationFilterChain$1.run(ApplicationFilterChain.java:187)
> at java.security.AccessController.doPrivileged(Native Method)
> at
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:186)
> at org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:52)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498)
> at org.apache.catalina.security.SecurityUtil$1.run(SecurityUtil.java:288)
> at org.apache.catalina.security.SecurityUtil$1.run(SecurityUtil.java:285)
> at java.security.AccessController.doPrivileged(Native Method)
> at javax.security.auth.Subject.doAsPrivileged(Subject.java:549)
> at org.apache.catalina.security.SecurityUtil.execute(SecurityUtil.java:320)
> at org.apache.catalina.security.SecurityUtil.doAsPrivilege(SecurityUtil.java:260)
> at
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:237)
> at
org.apache.catalina.core.ApplicationFilterChain.access$000(ApplicationFilterChain.java:55)
> at
org.apache.catalina.core.ApplicationFilterChain$1.run(ApplicationFilterChain.java:191)
> at
org.apache.catalina.core.ApplicationFilterChain$1.run(ApplicationFilterChain.java:187)
> at java.security.AccessController.doPrivileged(Native Method)
> at
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:186)
> at
org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:218)
> at
org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:110)
> at
org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:506)
> at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:169)
> at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:103)
> at org.apache.catalina.valves.AccessLogValve.invoke(AccessLogValve.java:962)
> at
org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:116)
> at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:445)
> at org.apache.coyote.ajp.AjpProcessor.process(AjpProcessor.java:190)
> at
org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java:637)
> at org.apache.tomcat.util.net.JIoEndpoint$SocketProcessor.run(JIoEndpoint.java:316)
> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> at
org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
> at java.lang.Thread.run(Thread.java:748)
> Caused by: org.jboss.resteasy.plugins.providers.jaxb.JAXBMarshalException:
javax.xml.bind.MarshalException
> - with linked exception:
> [org.apache.catalina.connector.ClientAbortException: java.net.SocketException: Broken
pipe (Write failed)]
> at
org.jboss.resteasy.plugins.providers.jaxb.AbstractJAXBProvider.writeTo(AbstractJAXBProvider.java:128)
> at
org.jboss.resteasy.core.interception.AbstractWriterInterceptorContext.writeTo(AbstractWriterInterceptorContext.java:129)
> at
org.jboss.resteasy.core.interception.ServerWriterInterceptorContext.writeTo(ServerWriterInterceptorContext.java:62)
> at
org.jboss.resteasy.core.interception.AbstractWriterInterceptorContext.proceed(AbstractWriterInterceptorContext.java:118)
> at
org.jboss.resteasy.plugins.interceptors.encoding.GZIPEncodingInterceptor.aroundWriteTo(GZIPEncodingInterceptor.java:100)
> at
org.jboss.resteasy.core.interception.AbstractWriterInterceptorContext.proceed(AbstractWriterInterceptorContext.java:122)
> at
org.jboss.resteasy.core.ServerResponseWriter.writeNomapResponse(ServerResponseWriter.java:99)
> at
org.jboss.resteasy.core.SynchronousDispatcher.writeResponse(SynchronousDispatcher.java:427)
> ... 54 more
> Caused by: javax.xml.bind.MarshalException
> - with linked exception:
> [org.apache.catalina.connector.ClientAbortException: java.net.SocketException: Broken
pipe (Write failed)]
> at
com.sun.xml.internal.bind.v2.runtime.MarshallerImpl.write(MarshallerImpl.java:313)
> at
com.sun.xml.internal.bind.v2.runtime.MarshallerImpl.marshal(MarshallerImpl.java:236)
> at
javax.xml.bind.helpers.AbstractMarshallerImpl.marshal(AbstractMarshallerImpl.java:95)
> at
org.jboss.resteasy.plugins.providers.jaxb.AbstractJAXBProvider.writeTo(AbstractJAXBProvider.java:124)
> ... 61 more
> Caused by: org.apache.catalina.connector.ClientAbortException:
java.net.SocketException: Broken pipe (Write failed)
> at org.apache.catalina.connector.OutputBuffer.realWriteBytes(OutputBuffer.java:410)
> at org.apache.tomcat.util.buf.ByteChunk.flushBuffer(ByteChunk.java:480)
> at org.apache.tomcat.util.buf.ByteChunk.append(ByteChunk.java:366)
> at org.apache.catalina.connector.OutputBuffer.writeBytes(OutputBuffer.java:435)
> at org.apache.catalina.connector.OutputBuffer.write(OutputBuffer.java:423)
> at
org.apache.catalina.connector.CoyoteOutputStream.write(CoyoteOutputStream.java:91)
> at
org.jboss.resteasy.plugins.server.servlet.HttpServletResponseWrapper$DeferredOutputStream.write(HttpServletResponseWrapper.java:46)
> at
org.jboss.resteasy.util.CommitHeaderOutputStream.write(CommitHeaderOutputStream.java:71)
> at
com.sun.xml.internal.bind.v2.runtime.output.UTF8XmlOutput.write(UTF8XmlOutput.java:396)
> at com.sun.xml.internal.bind.v2.runtime.output.Encoded.write(Encoded.java:152)
> at
com.sun.xml.internal.bind.v2.runtime.output.UTF8XmlOutput.doText(UTF8XmlOutput.java:308)
> at
com.sun.xml.internal.bind.v2.runtime.output.UTF8XmlOutput.text(UTF8XmlOutput.java:290)
> at
com.sun.xml.internal.bind.v2.runtime.XMLSerializer.leafElement(XMLSerializer.java:313)
> at
com.sun.xml.internal.bind.v2.model.impl.RuntimeBuiltinLeafInfoImpl$StringImplImpl.writeLeafElement(RuntimeBuiltinLeafInfoImpl.java:1036)
> at
com.sun.xml.internal.bind.v2.model.impl.RuntimeBuiltinLeafInfoImpl$StringImplImpl.writeLeafElement(RuntimeBuiltinLeafInfoImpl.java:1015)
> at
com.sun.xml.internal.bind.v2.runtime.reflect.TransducedAccessor$CompositeTransducedAccessorImpl.writeLeafElement(TransducedAccessor.java:239)
> at
com.sun.xml.internal.bind.v2.runtime.property.SingleElementLeafProperty.serializeBody(SingleElementLeafProperty.java:115)
> at
com.sun.xml.internal.bind.v2.runtime.ClassBeanInfoImpl.serializeBody(ClassBeanInfoImpl.java:345)
> at
com.sun.xml.internal.bind.v2.runtime.XMLSerializer.childAsXsiType(XMLSerializer.java:681)
> at
com.sun.xml.internal.bind.v2.runtime.property.ArrayElementNodeProperty.serializeItem(ArrayElementNodeProperty.java:54)
> at
com.sun.xml.internal.bind.v2.runtime.property.ArrayElementProperty.serializeListBody(ArrayElementProperty.java:157)
> at
com.sun.xml.internal.bind.v2.runtime.property.ArrayERProperty.serializeBody(ArrayERProperty.java:144)
> at
com.sun.xml.internal.bind.v2.runtime.ClassBeanInfoImpl.serializeBody(ClassBeanInfoImpl.java:350)
> at
com.sun.xml.internal.bind.v2.runtime.ClassBeanInfoImpl.serializeBody(ClassBeanInfoImpl.java:336)
> at
com.sun.xml.internal.bind.v2.runtime.XMLSerializer.childAsSoleContent(XMLSerializer.java:578)
> at
com.sun.xml.internal.bind.v2.runtime.ClassBeanInfoImpl.serializeRoot(ClassBeanInfoImpl.java:326)
> at
com.sun.xml.internal.bind.v2.runtime.XMLSerializer.childAsRoot(XMLSerializer.java:479)
> at
com.sun.xml.internal.bind.v2.runtime.MarshallerImpl.write(MarshallerImpl.java:308)
> ... 64 more
> Caused by: java.net.SocketException: Broken pipe (Write failed)
> at java.net.SocketOutputStream.socketWrite0(Native Method)
> at java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:111)
> at java.net.SocketOutputStream.write(SocketOutputStream.java:155)
> at org.apache.coyote.ajp.AjpProcessor.output(AjpProcessor.java:298)
> at
org.apache.coyote.ajp.AbstractAjpProcessor$SocketOutputBuffer.doWrite(AbstractAjpProcessor.java:1275)
> at org.apache.coyote.Response.doWrite(Response.java:499)
> at org.apache.catalina.connector.OutputBuffer.realWriteBytes(OutputBuffer.java:405)
> ... 91 more