Stack trace on fc21-vm05.phx.ovirt.org
Hi, trying to investigate why http://jenkins.ovirt.org/job/vdsm_master-libgfapi_create-rpms-el7-x86_64_mer... is taking more than 5 hours to complete, I've gone to http://jenkins.ovirt.org/computer/fc21-vm05.phx.ovirt.org/script and executed: println "free -h".execute().text This is the result: Stack trace hudson.remoting.ChannelClosedException: channel is already closed at hudson.remoting.Channel.send(Channel.java:540) at hudson.remoting.Request.call(Request.java:129) at hudson.remoting.Channel.call(Channel.java:742) at hudson.util.RemotingDiagnostics.executeGroovy(RemotingDiagnostics.java:108) at jenkins.model.Jenkins._doScript(Jenkins.java:3537) at hudson.model.Computer._doScript(Computer.java:1179) at hudson.model.Computer.doScript(Computer.java:1168) at sun.reflect.GeneratedMethodAccessor8245.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:616) at org.kohsuke.stapler.Function$InstanceFunction.invoke(Function.java:298) at org.kohsuke.stapler.Function.bindAndInvoke(Function.java:161) at org.kohsuke.stapler.Function.bindAndInvokeAndServeResponse(Function.java:96) at org.kohsuke.stapler.MetaClass$1.doDispatch(MetaClass.java:120) at org.kohsuke.stapler.NameBasedDispatcher.dispatch(NameBasedDispatcher.java:53) at org.kohsuke.stapler.Stapler.tryInvoke(Stapler.java:733) at org.kohsuke.stapler.Stapler.invoke(Stapler.java:863) at org.kohsuke.stapler.MetaClass$12.dispatch(MetaClass.java:390) at org.kohsuke.stapler.Stapler.tryInvoke(Stapler.java:733) at org.kohsuke.stapler.Stapler.invoke(Stapler.java:863) at org.kohsuke.stapler.MetaClass$4.doDispatch(MetaClass.java:210) at org.kohsuke.stapler.NameBasedDispatcher.dispatch(NameBasedDispatcher.java:53) at org.kohsuke.stapler.Stapler.tryInvoke(Stapler.java:733) at org.kohsuke.stapler.Stapler.invoke(Stapler.java:863) at org.kohsuke.stapler.Stapler.invoke(Stapler.java:636) at org.kohsuke.stapler.Stapler.service(Stapler.java:225) at javax.servlet.http.HttpServlet.service(HttpServlet.java:848) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:686) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1494) at hudson.util.PluginServletFilter$1.doFilter(PluginServletFilter.java:96) at hudson.plugins.greenballs.GreenBallFilter.doFilter(GreenBallFilter.java:58) at hudson.util.PluginServletFilter$1.doFilter(PluginServletFilter.java:99) at hudson.plugins.audit_trail.AuditTrailFilter.doFilter(AuditTrailFilter.java:89) at hudson.util.PluginServletFilter$1.doFilter(PluginServletFilter.java:99) at net.bull.javamelody.MonitoringFilter.doFilter(MonitoringFilter.java:202) at net.bull.javamelody.MonitoringFilter.doFilter(MonitoringFilter.java:180) at net.bull.javamelody.PluginMonitoringFilter.doFilter(PluginMonitoringFilter.java:85) at org.jvnet.hudson.plugins.monitoring.HudsonMonitoringFilter.doFilter(HudsonMonitoringFilter.java:89) at hudson.util.PluginServletFilter$1.doFilter(PluginServletFilter.java:99) at hudson.util.PluginServletFilter.doFilter(PluginServletFilter.java:88) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1482) at hudson.security.csrf.CrumbFilter.doFilter(CrumbFilter.java:48) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1482) at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:84) at hudson.security.UnwrapSecurityExceptionFilter.doFilter(UnwrapSecurityExceptionFilter.java:51) at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:87) at jenkins.security.ExceptionTranslationFilter.doFilter(ExceptionTranslationFilter.java:117) at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:87) at org.acegisecurity.providers.anonymous.AnonymousProcessingFilter.doFilter(AnonymousProcessingFilter.java:125) at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:87) at org.acegisecurity.ui.rememberme.RememberMeProcessingFilter.doFilter(RememberMeProcessingFilter.java:142) at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:87) at org.acegisecurity.ui.AbstractProcessingFilter.doFilter(AbstractProcessingFilter.java:271) at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:87) at jenkins.security.BasicHeaderProcessor.doFilter(BasicHeaderProcessor.java:86) at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:87) at org.acegisecurity.context.HttpSessionContextIntegrationFilter.doFilter(HttpSessionContextIntegrationFilter.java:249) at hudson.security.HttpSessionContextIntegrationFilter2.doFilter(HttpSessionContextIntegrationFilter2.java:67) at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:87) at hudson.security.ChainedServletFilter.doFilter(ChainedServletFilter.java:76) at hudson.security.HudsonFilter.doFilter(HudsonFilter.java:164) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1482) at org.kohsuke.stapler.compression.CompressionFilter.doFilter(CompressionFilter.java:46) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1482) at hudson.util.CharacterEncodingFilter.doFilter(CharacterEncodingFilter.java:81) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1474) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:499) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:533) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:231) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1086) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:428) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1020) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116) at org.eclipse.jetty.server.Server.handle(Server.java:370) at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489) at org.eclipse.jetty.server.AbstractHttpConnection.content(AbstractHttpConnection.java:960) at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.content(AbstractHttpConnection.java:1021) at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:865) at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:240) at org.eclipse.jetty.server.AsyncHttpConnection.handle(AsyncHttpConnection.java:82) at org.eclipse.jetty.io.nio.SelectChannelEndPoint.handle(SelectChannelEndPoint.java:668) at org.eclipse.jetty.io.nio.SelectChannelEndPoint$1.run(SelectChannelEndPoint.java:52) at winstone.BoundedExecutorService$1.run(BoundedExecutorService.java:77) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1146) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:679) Caused by: java.io.IOException at hudson.remoting.Channel.close(Channel.java:1030) at hudson.slaves.ChannelPinger$1.onDead(ChannelPinger.java:114) at hudson.remoting.PingThread.ping(PingThread.java:120) at hudson.remoting.PingThread.run(PingThread.java:81) Caused by: java.util.concurrent.TimeoutException: Ping started on 1422979675741 hasn't completed at 1422979915741 ... 2 more Can you check what's going on? -- Sandro Bonazzola Better technology. Faster innovation. Powered by community collaboration. See how it works at redhat.com
Hi, trying to investigate why http://jenkins.ovirt.org/job/vdsm_master-libgfapi_create-rpms-el7-x86_64_= merged/176/ is taking more than 5 hours to complete, I've gone to http://jenkins.ovirt.org/computer/fc21-vm05.phx.ovirt.org/script =20 and executed: println "free -h".execute().text =20 This is the result: Stack trace =20 hudson.remoting.ChannelClosedException: channel is already closed at hudson.remoting.Channel.send(Channel.java:540) at hudson.remoting.Request.call(Request.java:129) at hudson.remoting.Channel.call(Channel.java:742) at hudson.util.RemotingDiagnostics.executeGroovy(RemotingDiagnostics.jav= a:108) at jenkins.model.Jenkins._doScript(Jenkins.java:3537) at hudson.model.Computer._doScript(Computer.java:1179) at hudson.model.Computer.doScript(Computer.java:1168) at sun.reflect.GeneratedMethodAccessor8245.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAcces= sorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:616) at org.kohsuke.stapler.Function$InstanceFunction.invoke(Function.java:29=
at org.kohsuke.stapler.Function.bindAndInvoke(Function.java:161) at org.kohsuke.stapler.Function.bindAndInvokeAndServeResponse(Function.j= ava:96) at org.kohsuke.stapler.MetaClass$1.doDispatch(MetaClass.java:120) at org.kohsuke.stapler.NameBasedDispatcher.dispatch(NameBasedDispatcher.= java:53) at org.kohsuke.stapler.Stapler.tryInvoke(Stapler.java:733) at org.kohsuke.stapler.Stapler.invoke(Stapler.java:863) at org.kohsuke.stapler.MetaClass$12.dispatch(MetaClass.java:390) at org.kohsuke.stapler.Stapler.tryInvoke(Stapler.java:733) at org.kohsuke.stapler.Stapler.invoke(Stapler.java:863) at org.kohsuke.stapler.MetaClass$4.doDispatch(MetaClass.java:210) at org.kohsuke.stapler.NameBasedDispatcher.dispatch(NameBasedDispatcher.= java:53) at org.kohsuke.stapler.Stapler.tryInvoke(Stapler.java:733) at org.kohsuke.stapler.Stapler.invoke(Stapler.java:863) at org.kohsuke.stapler.Stapler.invoke(Stapler.java:636) at org.kohsuke.stapler.Stapler.service(Stapler.java:225) at javax.servlet.http.HttpServlet.service(HttpServlet.java:848) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:686) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(Servlet= Handler.java:1494) at hudson.util.PluginServletFilter$1.doFilter(PluginServletFilter.java:9=
at hudson.plugins.greenballs.GreenBallFilter.doFilter(GreenBallFilter.ja= va:58) at hudson.util.PluginServletFilter$1.doFilter(PluginServletFilter.java:9=
at hudson.plugins.audit_trail.AuditTrailFilter.doFilter(AuditTrailFilter= =2Ejava:89) at hudson.util.PluginServletFilter$1.doFilter(PluginServletFilter.java:9=
at net.bull.javamelody.MonitoringFilter.doFilter(MonitoringFilter.java:2=
at net.bull.javamelody.MonitoringFilter.doFilter(MonitoringFilter.java:1=
at net.bull.javamelody.PluginMonitoringFilter.doFilter(PluginMonitoringF= ilter.java:85) at org.jvnet.hudson.plugins.monitoring.HudsonMonitoringFilter.doFilter(H= udsonMonitoringFilter.java:89) at hudson.util.PluginServletFilter$1.doFilter(PluginServletFilter.java:9=
at hudson.util.PluginServletFilter.doFilter(PluginServletFilter.java:88) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(Servlet= Handler.java:1482) at hudson.security.csrf.CrumbFilter.doFilter(CrumbFilter.java:48) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(Servlet= Handler.java:1482) at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.= java:84) at hudson.security.UnwrapSecurityExceptionFilter.doFilter(UnwrapSecurity= ExceptionFilter.java:51) at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.= java:87) at jenkins.security.ExceptionTranslationFilter.doFilter(ExceptionTransla= tionFilter.java:117) at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.= java:87) at org.acegisecurity.providers.anonymous.AnonymousProcessingFilter.doFil= ter(AnonymousProcessingFilter.java:125) at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.= java:87) at org.acegisecurity.ui.rememberme.RememberMeProcessingFilter.doFilter(R= ememberMeProcessingFilter.java:142) at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.= java:87) at org.acegisecurity.ui.AbstractProcessingFilter.doFilter(AbstractProces= singFilter.java:271) at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.= java:87) at jenkins.security.BasicHeaderProcessor.doFilter(BasicHeaderProcessor.j= ava:86) at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.= java:87) at org.acegisecurity.context.HttpSessionContextIntegrationFilter.doFilte= r(HttpSessionContextIntegrationFilter.java:249) at hudson.security.HttpSessionContextIntegrationFilter2.doFilter(HttpSes= sionContextIntegrationFilter2.java:67) at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.= java:87) at hudson.security.ChainedServletFilter.doFilter(ChainedServletFilter.ja= va:76) at hudson.security.HudsonFilter.doFilter(HudsonFilter.java:164) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(Servlet= Handler.java:1482) at org.kohsuke.stapler.compression.CompressionFilter.doFilter(Compressio= nFilter.java:46) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(Servlet= Handler.java:1482) at hudson.util.CharacterEncodingFilter.doFilter(CharacterEncodingFilter.= java:81) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(Servlet= Handler.java:1474) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java= :499) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.j= ava:137) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.jav= a:533) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandl= er.java:231) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandl= er.java:1086) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:=
--IQvoI1rdlCKiYEYW Content-Type: text/plain; charset=us-ascii Content-Disposition: inline Content-Transfer-Encoding: quoted-printable We just went out (still reviewing the consequences) of another outage, this time was the storage server. After yesterdays network issue, the storage servers needed to change their netmask, something that should not have ad any consequence but it seems tha= t it triggered a resource failover to the slave, but as the primary still was ac= tive (the ip was reachable) the failover was aborted and lead to a resource inconsistency that after a few seconds made the floating ip go away, leaving the nfs service unexposed and making the hosts stall. The cluster is up and running again and all the changes have been made to t= he storage servers, so no further interventions there are needed. Sorry for all the inconvenience. On 02/03, Sandro Bonazzola wrote: 8) 6) 9) 9) 02) 80) 9) 428)
at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandle= r.java:193) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandle= r.java:1020) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.j= ava:135) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper= =2Ejava:116) at org.eclipse.jetty.server.Server.handle(Server.java:370) at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(Abstrac= tHttpConnection.java:489) at org.eclipse.jetty.server.AbstractHttpConnection.content(AbstractHttpC= onnection.java:960) at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.conten= t(AbstractHttpConnection.java:1021) at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:865) at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:240) at org.eclipse.jetty.server.AsyncHttpConnection.handle(AsyncHttpConnecti= on.java:82) at org.eclipse.jetty.io.nio.SelectChannelEndPoint.handle(SelectChannelEn= dPoint.java:668) at org.eclipse.jetty.io.nio.SelectChannelEndPoint$1.run(SelectChannelEnd= Point.java:52) at winstone.BoundedExecutorService$1.run(BoundedExecutorService.java:77) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.= java:1146) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor= =2Ejava:615) at java.lang.Thread.run(Thread.java:679) Caused by: java.io.IOException at hudson.remoting.Channel.close(Channel.java:1030) at hudson.slaves.ChannelPinger$1.onDead(ChannelPinger.java:114) at hudson.remoting.PingThread.ping(PingThread.java:120) at hudson.remoting.PingThread.run(PingThread.java:81) Caused by: java.util.concurrent.TimeoutException: Ping started on 1422979= 675741 hasn't completed at 1422979915741 ... 2 more =20 =20 Can you check what's going on? =20 --=20 Sandro Bonazzola Better technology. Faster innovation. Powered by community collaboration. See how it works at redhat.com _______________________________________________ Infra mailing list Infra@ovirt.org http://lists.ovirt.org/mailman/listinfo/infra
--=20 David Caro Red Hat S.L. Continuous Integration Engineer - EMEA ENG Virtualization R&D Tel.: +420 532 294 605 Email: dcaro@redhat.com Web: www.redhat.com RHT Global #: 82-62605 --IQvoI1rdlCKiYEYW Content-Type: application/pgp-signature -----BEGIN PGP SIGNATURE----- Version: GnuPG v1 iQEcBAEBAgAGBQJU0PlrAAoJEEBxx+HSYmnDOsoH/joICSfqBV0O/74jSJ/JJ7t/ eyuxfaY1Utyd2RPc+RjJcNHSbWZ0wGVfSQ4I5QbD25iuUyLCse6wA/1paLHmdiEq jg2/34nhYYb+lWataGwVYDZd267JDKxzkwP095J4j/jI1yTLMO1xFQBUPvdXWCWM zNZAbivCjwcFWyevBrQbMXkST0pHY8cQfHKqX5UEnaZrBIUpRnGkmj64aTRVK3Jd uiCqpkQMqny+YeAKXH6NIZk0UUOq6uOaelqZWxBhzQ2ld+RVB0QNGcY566JnL6Uu QPu5pAnStROVsCP2EKflj1NV7WYxJwQXNV1wDj5N9VppeMt/OjoxhvtbPPHajBo= =EInE -----END PGP SIGNATURE----- --IQvoI1rdlCKiYEYW--
participants (2)
-
David Caro -
Sandro Bonazzola