Sorry for the dump size, but not sure if the below from /var/log/pki/pki-tomcat/localhost.date.log helps:

15-Nov-2017 12:14:50.557 SEVERE [localhost-startStop-1] org.apache.catalina.core.ApplicationContext.log StandardWrapper.Throwable
 java.lang.NullPointerException
at com.netscape.cmscore.selftests.SelfTestSubsystem.shutdown(SelfTestSubsystem.java:1886)
at com.netscape.cmscore.apps.CMSEngine.shutdownSubsystems(CMSEngine.java:2118)
at com.netscape.cmscore.apps.CMSEngine.shutdown(CMSEngine.java:2013)
at com.netscape.certsrv.apps.CMS.shutdown(CMS.java:234)
at com.netscape.certsrv.apps.CMS.start(CMS.java:1630)
at com.netscape.cms.servlet.base.CMSStartServlet.init(CMSStartServlet.java:114)
at javax.servlet.GenericServlet.init(GenericServlet.java:158)
at org.apache.catalina.core.StandardWrapper.initServlet(StandardWrapper.java:1227)
at org.apache.catalina.core.StandardWrapper.loadServlet(StandardWrapper.java:1140)
at org.apache.catalina.core.StandardWrapper.load(StandardWrapper.java:1027)
at org.apache.catalina.core.StandardContext.loadOnStartup(StandardContext.java:5038)
at org.apache.catalina.core.StandardContext.startInternal(StandardContext.java:5348)
at org.apache.catalina.util.LifecycleBase.start(LifecycleBase.java:145)
at org.apache.catalina.core.ContainerBase.addChildInternal(ContainerBase.java:753)
at org.apache.catalina.core.ContainerBase.addChild(ContainerBase.java:729)
at org.apache.catalina.core.StandardHost.addChild(StandardHost.java:717)
at org.apache.catalina.startup.HostConfig.deployDescriptor(HostConfig.java:621)
at org.apache.catalina.startup.HostConfig$DeployDescriptor.run(HostConfig.java:1835)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)

15-Nov-2017 12:14:50.558 SEVERE [localhost-startStop-1] org.apache.catalina.core.StandardContext.loadOnStartup Servlet [castart] in web application [/ca] threw load() exception
 java.lang.NullPointerException
at com.netscape.cmscore.selftests.SelfTestSubsystem.shutdown(SelfTestSubsystem.java:1886)
at com.netscape.cmscore.apps.CMSEngine.shutdownSubsystems(CMSEngine.java:2118)
at com.netscape.cmscore.apps.CMSEngine.shutdown(CMSEngine.java:2013)
at com.netscape.certsrv.apps.CMS.shutdown(CMS.java:234)
at com.netscape.certsrv.apps.CMS.start(CMS.java:1630)
at com.netscape.cms.servlet.base.CMSStartServlet.init(CMSStartServlet.java:114)
at javax.servlet.GenericServlet.init(GenericServlet.java:158)
at org.apache.catalina.core.StandardWrapper.initServlet(StandardWrapper.java:1227)
at org.apache.catalina.core.StandardWrapper.loadServlet(StandardWrapper.java:1140)
at org.apache.catalina.core.StandardWrapper.load(StandardWrapper.java:1027)
at org.apache.catalina.core.StandardContext.loadOnStartup(StandardContext.java:5038)
at org.apache.catalina.core.StandardContext.startInternal(StandardContext.java:5348)
at org.apache.catalina.util.LifecycleBase.start(LifecycleBase.java:145)
at org.apache.catalina.core.ContainerBase.addChildInternal(ContainerBase.java:753)
at org.apache.catalina.core.ContainerBase.addChild(ContainerBase.java:729)
at org.apache.catalina.core.StandardHost.addChild(StandardHost.java:717)
at org.apache.catalina.startup.HostConfig.deployDescriptor(HostConfig.java:621)
at org.apache.catalina.startup.HostConfig$DeployDescriptor.run(HostConfig.java:1835)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)

15-Nov-2017 12:14:54.509 SEVERE [http-bio-8443-exec-1] org.apache.catalina.core.StandardHostValve.invoke Exception Processing /ca/rest/account/login
 javax.ws.rs.ServiceUnavailableException: Subsystem unavailable
at com.netscape.cms.tomcat.ProxyRealm.findSecurityConstraints(ProxyRealm.java:138)
at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:498)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:141)
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:79)
at org.apache.catalina.valves.AbstractAccessLogValve.invoke(AbstractAccessLogValve.java:620)
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:88)
at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:502)
at org.apache.coyote.http11.AbstractHttp11Processor.process(AbstractHttp11Processor.java:1132)
at org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java:684)
at org.apache.tomcat.util.net.JIoEndpoint$SocketProcessor.run(JIoEndpoint.java:283)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
at java.lang.Thread.run(Thread.java:748)

15-Nov-2017 13:05:55.874 SEVERE [localhost-startStop-1] org.apache.catalina.core.ApplicationContext.log StandardWrapper.Throwable
 java.lang.NullPointerException
at com.netscape.cmscore.selftests.SelfTestSubsystem.shutdown(SelfTestSubsystem.java:1886)
at com.netscape.cmscore.apps.CMSEngine.shutdownSubsystems(CMSEngine.java:2118)
at com.netscape.cmscore.apps.CMSEngine.shutdown(CMSEngine.java:2013)
at com.netscape.certsrv.apps.CMS.shutdown(CMS.java:234)
at com.netscape.certsrv.apps.CMS.start(CMS.java:1630)
at com.netscape.cms.servlet.base.CMSStartServlet.init(CMSStartServlet.java:114)
at javax.servlet.GenericServlet.init(GenericServlet.java:158)
at org.apache.catalina.core.StandardWrapper.initServlet(StandardWrapper.java:1227)
at org.apache.catalina.core.StandardWrapper.loadServlet(StandardWrapper.java:1140)
at org.apache.catalina.core.StandardWrapper.load(StandardWrapper.java:1027)
at org.apache.catalina.core.StandardContext.loadOnStartup(StandardContext.java:5038)
at org.apache.catalina.core.StandardContext.startInternal(StandardContext.java:5348)
at org.apache.catalina.util.LifecycleBase.start(LifecycleBase.java:145)
at org.apache.catalina.core.ContainerBase.addChildInternal(ContainerBase.java:753)
at org.apache.catalina.core.ContainerBase.addChild(ContainerBase.java:729)
at org.apache.catalina.core.StandardHost.addChild(StandardHost.java:717)
at org.apache.catalina.startup.HostConfig.deployDescriptor(HostConfig.java:621)
at org.apache.catalina.startup.HostConfig$DeployDescriptor.run(HostConfig.java:1835)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)

15-Nov-2017 13:05:55.875 SEVERE [localhost-startStop-1] org.apache.catalina.core.StandardContext.loadOnStartup Servlet [castart] in web application [/ca] threw load() exception
 java.lang.NullPointerException
at com.netscape.cmscore.selftests.SelfTestSubsystem.shutdown(SelfTestSubsystem.java:1886)
at com.netscape.cmscore.apps.CMSEngine.shutdownSubsystems(CMSEngine.java:2118)
at com.netscape.cmscore.apps.CMSEngine.shutdown(CMSEngine.java:2013)
at com.netscape.certsrv.apps.CMS.shutdown(CMS.java:234)
at com.netscape.certsrv.apps.CMS.start(CMS.java:1630)
at com.netscape.cms.servlet.base.CMSStartServlet.init(CMSStartServlet.java:114)
at javax.servlet.GenericServlet.init(GenericServlet.java:158)
at org.apache.catalina.core.StandardWrapper.initServlet(StandardWrapper.java:1227)
at org.apache.catalina.core.StandardWrapper.loadServlet(StandardWrapper.java:1140)
at org.apache.catalina.core.StandardWrapper.load(StandardWrapper.java:1027)
at org.apache.catalina.core.StandardContext.loadOnStartup(StandardContext.java:5038)
at org.apache.catalina.core.StandardContext.startInternal(StandardContext.java:5348)
at org.apache.catalina.util.LifecycleBase.start(LifecycleBase.java:145)
at org.apache.catalina.core.ContainerBase.addChildInternal(ContainerBase.java:753)
at org.apache.catalina.core.ContainerBase.addChild(ContainerBase.java:729)
at org.apache.catalina.core.StandardHost.addChild(StandardHost.java:717)
at org.apache.catalina.startup.HostConfig.deployDescriptor(HostConfig.java:621)
at org.apache.catalina.startup.HostConfig$DeployDescriptor.run(HostConfig.java:1835)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)

15-Nov-2017 13:05:59.706 SEVERE [http-bio-8443-exec-1] org.apache.catalina.core.StandardHostValve.invoke Exception Processing /ca/rest/account/login
 javax.ws.rs.ServiceUnavailableException: Subsystem unavailable
at com.netscape.cms.tomcat.ProxyRealm.findSecurityConstraints(ProxyRealm.java:138)
at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:498)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:141)
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:79)
at org.apache.catalina.valves.AbstractAccessLogValve.invoke(AbstractAccessLogValve.java:620)
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:88)
at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:502)
at org.apache.coyote.http11.AbstractHttp11Processor.process(AbstractHttp11Processor.java:1132)
at org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java:684)
at org.apache.tomcat.util.net.JIoEndpoint$SocketProcessor.run(JIoEndpoint.java:283)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
at java.lang.Thread.run(Thread.java:748)


On 15 November 2017 at 13:23, David Harvey <davidcharvey@googlemail.com> wrote:
Hi wisdom of the list,

I know I am an edge case with running on ubuntu, but hoped someone might be able to shed some light.

A bit of background.  I'm trying to test upgrades without potentially hosing my existing services, so I have cloned the VM, given it a new IP address, updated hosts file and pointed DNS somewhere that doesn't know about the real IPA services (8.8.8.8) so it won't try and sync or replicate.

Attempting to upgrade hits a snags or two, some described in bugs already like the pki version number confusing the apt scripts https://bugs.launchpad.net/ubuntu/+source/freeipa/+bug/1703051 ). The one I can't work around however is below.

It seems deeply unhappy, and restarting the services result in the dogtag-pki web page being available until a login attempt is made (as occurs during the ipa-server-upgrade) after which point it bombs with a 500 error.

Could the below caused by https://bugs.launchpad.net/ubuntu/+source/freeipa/+bug/1716842 ?

Any advice appreciated, as I think even when 18.04 hits with the proposed updates to rely on to tomcat 8.5, I'll still need to upgrade via 17.10 which seems currently fraught!  If it relates to my method of cloning the VM, is there a better way of testing upgrades without potentially hosing the existing live systems?


Thanks in advance,

David

2017-11-15T13:05:59Z DEBUG approved_usage = SSL Server intended_usage = SSL Server
2017-11-15T13:05:59Z DEBUG cert valid True for "CN=ipa1.my.net,O=THOMAC.NET"
2017-11-15T13:05:59Z DEBUG handshake complete, peer = IPADDRESS
2017-11-15T13:05:59Z DEBUG Protocol: TLS1.2
2017-11-15T13:05:59Z DEBUG Cipher: TLS_RSA_WITH_AES_128_CBC_SHA
2017-11-15T13:05:59Z DEBUG response status 500
2017-11-15T13:05:59Z DEBUG response headers {'content-length': '2292', 'content-language': 'en', 'server': 'Apache-Coyote/1.1', 'connection': 'close', 'date': 'Wed, 15 Nov 2017 13:05:59 GMT', 'content-type': 'text/html;charset=utf-8'}
2017-11-15T13:05:59Z DEBUG response body '<!DOCTYPE html><html><head><title>Apache Tomcat/8.0.46 (Ubuntu) - Error report</title><style type="text/css">H1 {font-family:Tahoma,Arial,sans-serif;color:white;background-color:#525D76;font-size:22px;} H2 {font-family:Tahoma,Arial,sans-serif;color:white;background-color:#525D76;font-size:16px;} H3 {font-family:Tahoma,Arial,sans-serif;color:white;background-color:#525D76;font-size:14px;} BODY {font-family:Tahoma,Arial,sans-serif;color:black;background-color:white;} B {font-family:Tahoma,Arial,sans-serif;color:white;background-color:#525D76;} P {font-family:Tahoma,Arial,sans-serif;background:white;color:black;font-size:12px;}A {color : black;}A.name {color : black;}.line {height: 1px; background-color: #525D76; border: none;}</style> </head><body><h1>HTTP Status 500 - Subsystem unavailable</h1><div class="line"></div><p><b>type</b> Exception report</p><p><b>message</b> <u>Subsystem unavailable</u></p><p><b>description</b> <u>The server encountered an internal error that prevented it from fulfilling this request.</u></p><p><b>exception</b></p><pre>javax.ws.rs.ServiceUnavailableException: Subsystem unavailable\n\tcom.netscape.cms.tomcat.ProxyRealm.findSecurityConstraints(ProxyRealm.java:138)\n\torg.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:498)\n\torg.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:79)\n\torg.apache.catalina.valves.AbstractAccessLogValve.invoke(AbstractAccessLogValve.java:620)\n\torg.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:502)\n\torg.apache.coyote.http11.AbstractHttp11Processor.process(AbstractHttp11Processor.java:1132)\n\torg.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java:684)\n\torg.apache.tomcat.util.net.JIoEndpoint$SocketProcessor.run(JIoEndpoint.java:283)\n\tjava.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\n\tjava.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\n\torg.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)\n\tjava.lang.Thread.run(Thread.java:748)\n</pre><p><b>note</b> <u>The full stack trace of the root cause is available in the Apache Tomcat/8.0.46 (Ubuntu) logs.</u></p><hr class="line"><h3>Apache Tomcat/8.0.46 (Ubuntu)</h3></body></html>'
2017-11-15T13:05:59Z ERROR IPA server upgrade failed: Inspect /var/log/ipaupgrade.log and run command ipa-server-upgrade manually.
2017-11-15T13:05:59Z DEBUG   File "/usr/lib/python2.7/dist-packages/ipapython/admintool.py", line 172, in execute
    return_value = self.run()
  File "/usr/lib/python2.7/dist-packages/ipaserver/install/ipa_server_upgrade.py", line 46, in run
    server.upgrade()
  File "/usr/lib/python2.7/dist-packages/ipaserver/install/server/upgrade.py", line 1878, in upgrade
    upgrade_configuration()
  File "/usr/lib/python2.7/dist-packages/ipaserver/install/server/upgrade.py", line 1797, in upgrade_configuration
    ca_enable_ldap_profile_subsystem(ca)
  File "/usr/lib/python2.7/dist-packages/ipaserver/install/server/upgrade.py", line 347, in ca_enable_ldap_profile_subsystem
    cainstance.migrate_profiles_to_ldap()
  File "/usr/lib/python2.7/dist-packages/ipaserver/install/cainstance.py", line 1981, in migrate_profiles_to_ldap
    _create_dogtag_profile(profile_id, profile_data, overwrite=False)
  File "/usr/lib/python2.7/dist-packages/ipaserver/install/cainstance.py", line 1987, in _create_dogtag_profile
    with api.Backend.ra_certprofile as profile_api:
  File "/usr/lib/python2.7/dist-packages/ipaserver/plugins/dogtag.py", line 1294, in __enter__
    raise errors.RemoteRetrieveError(reason=_('Failed to authenticate to CA REST API'))

2017-11-15T13:05:59Z DEBUG The ipa-server-upgrade command failed, exception: RemoteRetrieveError: Failed to authenticate to CA REST API
2017-11-15T13:05:59Z ERROR Unexpected error - see /var/log/ipaupgrade.log for details:
RemoteRetrieveError: Failed to authenticate to CA REST API