Hi All,
In our production server we are getting very high CPU utilization around 100 % . We are getting a lot of CPU alarms for it . We are having 3.75GB of RAM . Please find the added stack trace.
Current time: Sat Dec 22 13:58:12 EST 2012
Server Information
Servoy version 6.1.3 -build 1424
Port used by RMI Registry: 1099
Repository version 42
Current time: Sat Dec 22 13:58:12 EST 2012
Uptime: 1 hour 59 minutes 37 seconds
JDK Information
java.vm.name=Java HotSpot™ 64-Bit Server VM
java.version=1.7.0_05
java.vm.info=mixed mode
java.vm.vendor=Oracle Corporation
Operating System Information
os.name=Windows Server 2008 R2
os.version=6.1
os.arch=amd64
System Information
Heap Space Memory: allocated=172816K, used=104792K, max=1267072K
Non-Heap Space Memory: allocated=73856K, used=73629K, max=180224K
Thread: pool-1-thread-6, state: WAITING, total cpu time: 0.0ms, total user time: 0.0ms
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.park(Unknown Source)
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(Unknown Source)
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(Unknown Source)
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(Unknown Source)
java.util.concurrent.ThreadPoolExecutor.getTask(Unknown Source)
java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
java.lang.Thread.run(Unknown Source)
Thread: pool-4-thread-19, state: WAITING, total cpu time: 156.001ms, total user time: 109.2007ms
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.park(Unknown Source)
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(Unknown Source)
java.util.concurrent.LinkedBlockingQueue.take(Unknown Source)
java.util.concurrent.ThreadPoolExecutor.getTask(Unknown Source)
java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
java.lang.Thread.run(Unknown Source)
Thread: pool-6-thread-1, state: WAITING, total cpu time: 0.0ms, total user time: 0.0ms
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.park(Unknown Source)
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(Unknown Source)
java.util.concurrent.LinkedBlockingQueue.take(Unknown Source)
java.util.concurrent.ThreadPoolExecutor.getTask(Unknown Source)
java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
java.lang.Thread.run(Unknown Source)
Thread: pool-2-thread-2, state: WAITING, total cpu time: 62.4004ms, total user time: 62.4004ms
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.park(Unknown Source)
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(Unknown Source)
java.util.concurrent.LinkedBlockingQueue.take(Unknown Source)
java.util.concurrent.ThreadPoolExecutor.getTask(Unknown Source)
java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
java.lang.Thread.run(Unknown Source)
Thread: pool-1-thread-5, state: TIMED_WAITING, total cpu time: 0.0ms, total user time: 0.0ms
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.parkNanos(Unknown Source)
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Unknown Source)
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(Unknown Source)
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(Unknown Source)
java.util.concurrent.ThreadPoolExecutor.getTask(Unknown Source)
java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
java.lang.Thread.run(Unknown Source)
Thread: http-8080-12, state: RUNNABLE, total cpu time: 67907.2353ms, total user time: 65894.8224ms
java.net.SocketInputStream.socketRead0(Native Method)
java.net.SocketInputStream.read(Unknown Source)
java.net.SocketInputStream.read(Unknown Source)
org.apache.coyote.http11.InternalInputBuffer.fill(InternalInputBuffer.java:735)
org.apache.coyote.http11.InternalInputBuffer.parseRequestLine(InternalInputBuffer.java:366)
org.apache.coyote.http11.Http11Processor.process(Http11Processor.java:814)
org.apache.coyote.http11.Http11Protocol$Http11ConnectionHandler.process(Http11Protocol.java:588)
org.apache.tomcat.util.net.JIoEndpoint$Worker.run(JIoEndpoint.java:489)
java.lang.Thread.run(Unknown Source)
Thread: http-8080-11, state: RUNNABLE, total cpu time: 530.4034ms, total user time: 452.4029ms
sun.management.ThreadImpl.getThreadInfo1(Native Method)
sun.management.ThreadImpl.getThreadInfo(Unknown Source)
sun.management.ThreadImpl.getThreadInfo(Unknown Source)
com.servoy.j2db.server.servlets.ConfigServlet.Zs(ConfigServlet.java:4160)
com.servoy.j2db.server.servlets.ConfigServlet.service(ConfigServlet.java:865)
javax.servlet.http.HttpServlet.service(HttpServlet.java:717)
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:290)
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:233)
org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:191)
org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:127)
org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:102)
org.apache.catalina.valves.AccessLogValve.invoke(AccessLogValve.java:554)
org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:109)
org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:298)
org.apache.coyote.http11.Http11Processor.process(Http11Processor.java:859)
org.apache.coyote.http11.Http11Protocol$Http11ConnectionHandler.process(Http11Protocol.java:588)
org.apache.tomcat.util.net.JIoEndpoint$Worker.run(JIoEndpoint.java:489)
java.lang.Thread.run(Unknown Source)
Thread: http-8080-10, state: WAITING, total cpu time: 2620.8168ms, total user time: 2418.0155ms
java.lang.Object.wait(Native Method)
java.lang.Object.wait(Object.java:503)
org.apache.tomcat.util.net.JIoEndpoint$Worker.await(JIoEndpoint.java:458)
org.apache.tomcat.util.net.JIoEndpoint$Worker.run(JIoEndpoint.java:484)
java.lang.Thread.run(Unknown Source)
Thread: http-8080-9, state: WAITING, total cpu time: 296.4019ms, total user time: 187.2012ms
java.lang.Object.wait(Native Method)
java.lang.Object.wait(Object.java:503)
org.apache.tomcat.util.net.JIoEndpoint$Worker.await(JIoEndpoint.java:458)
org.apache.tomcat.util.net.JIoEndpoint$Worker.run(JIoEndpoint.java:484)
java.lang.Thread.run(Unknown Source)
Thread: http-8080-8, state: WAITING, total cpu time: 499.2032ms, total user time: 452.4029ms
java.lang.Object.wait(Native Method)
java.lang.Object.wait(Object.java:503)
org.apache.tomcat.util.net.JIoEndpoint$Worker.await(JIoEndpoint.java:458)
org.apache.tomcat.util.net.JIoEndpoint$Worker.run(JIoEndpoint.java:484)
java.lang.Thread.run(Unknown Source)
Thread: http-8080-7, state: WAITING, total cpu time: 9172.8588ms, total user time: 8782.8563ms
java.lang.Object.wait(Native Method)
java.lang.Object.wait(Object.java:503)
org.apache.tomcat.util.net.JIoEndpoint$Worker.await(JIoEndpoint.java:458)
org.apache.tomcat.util.net.JIoEndpoint$Worker.run(JIoEndpoint.java:484)
java.lang.Thread.run(Unknown Source)
Thread: http-8080-6, state: WAITING, total cpu time: 0.0ms, total user time: 0.0ms
java.lang.Object.wait(Native Method)
java.lang.Object.wait(Object.java:503)
org.apache.tomcat.util.net.JIoEndpoint$Worker.await(JIoEndpoint.java:458)
org.apache.tomcat.util.net.JIoEndpoint$Worker.run(JIoEndpoint.java:484)
java.lang.Thread.run(Unknown Source)
Thread: http-8080-5, state: WAITING, total cpu time: 15.6001ms, total user time: 15.6001ms
java.lang.Object.wait(Native Method)
java.lang.Object.wait(Object.java:503)
org.apache.tomcat.util.net.JIoEndpoint$Worker.await(JIoEndpoint.java:458)
org.apache.tomcat.util.net.JIoEndpoint$Worker.run(JIoEndpoint.java:484)
java.lang.Thread.run(Unknown Source)
Thread: pool-1-thread-4, state: WAITING, total cpu time: 0.0ms, total user time: 0.0ms
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.park(Unknown Source)
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(Unknown Source)
java.util.concurrent.LinkedBlockingQueue.take(Unknown Source)
java.util.concurrent.ThreadPoolExecutor.getTask(Unknown Source)
java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
java.lang.Thread.run(Unknown Source)
Thread: pool-1-thread-3, state: WAITING, total cpu time: 0.0ms, total user time: 0.0ms
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.park(Unknown Source)
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(Unknown Source)
java.util.concurrent.LinkedBlockingQueue.take(Unknown Source)
java.util.concurrent.ThreadPoolExecutor.getTask(Unknown Source)
java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
java.lang.Thread.run(Unknown Source)
Thread: http-8080-4, state: WAITING, total cpu time: 280.8018ms, total user time: 280.8018ms
java.lang.Object.wait(Native Method)
java.lang.Object.wait(Object.java:503)
org.apache.tomcat.util.net.JIoEndpoint$Worker.await(JIoEndpoint.java:458)
org.apache.tomcat.util.net.JIoEndpoint$Worker.run(JIoEndpoint.java:484)
java.lang.Thread.run(Unknown Source)
Thread: http-8080-3, state: WAITING, total cpu time: 514.8033ms, total user time: 436.8028ms
java.lang.Object.wait(Native Method)
java.lang.Object.wait(Object.java:503)
org.apache.tomcat.util.net.JIoEndpoint$Worker.await(JIoEndpoint.java:458)
org.apache.tomcat.util.net.JIoEndpoint$Worker.run(JIoEndpoint.java:484)
java.lang.Thread.run(Unknown Source)
Thread: http-8080-2, state: WAITING, total cpu time: 31.2002ms, total user time: 31.2002ms
java.lang.Object.wait(Native Method)
java.lang.Object.wait(Object.java:503)
org.apache.tomcat.util.net.JIoEndpoint$Worker.await(JIoEndpoint.java:458)
org.apache.tomcat.util.net.JIoEndpoint$Worker.run(JIoEndpoint.java:484)
java.lang.Thread.run(Unknown Source)
Thread: http-8080-1, state: WAITING, total cpu time: 78.0005ms, total user time: 62.4004ms
java.lang.Object.wait(Native Method)
java.lang.Object.wait(Object.java:503)
org.apache.tomcat.util.net.JIoEndpoint$Worker.await(JIoEndpoint.java:458)
org.apache.tomcat.util.net.JIoEndpoint$Worker.run(JIoEndpoint.java:484)
java.lang.Thread.run(Unknown Source)
Thread: http-8080-Acceptor-0, state: RUNNABLE, total cpu time: 0.0ms, total user time: 0.0ms
java.net.DualStackPlainSocketImpl.accept0(Native Method)
java.net.DualStackPlainSocketImpl.socketAccept(Unknown Source)
java.net.AbstractPlainSocketImpl.accept(Unknown Source)
java.net.PlainSocketImpl.accept(Unknown Source)
java.net.ServerSocket.implAccept(Unknown Source)
java.net.ServerSocket.accept(Unknown Source)
org.apache.tomcat.util.net.DefaultServerSocketFactory.acceptSocket(DefaultServerSocketFactory.java:61)
org.apache.tomcat.util.net.JIoEndpoint$Acceptor.run(JIoEndpoint.java:352)
java.lang.Thread.run(Unknown Source)
Thread: ContainerBackgroundProcessor[StandardEngine[Catalina]], state: TIMED_WAITING, total cpu time: 0.0ms, total user time: 0.0ms
java.lang.Thread.sleep(Native Method)
org.apache.catalina.core.ContainerBase$ContainerBackgroundProcessor.run(ContainerBase.java:1627)
java.lang.Thread.run(Unknown Source)
Thread: ModificationWatcher Task, state: TIMED_WAITING, total cpu time: 858.0055ms, total user time: 639.6041ms
java.lang.Thread.sleep(Native Method)
org.apache.wicket.util.time.Duration.sleep(Duration.java:445)
org.apache.wicket.util.thread.Task$1.run(Task.java:130)
java.lang.Thread.run(Unknown Source)
Thread: Timer-0, state: TIMED_WAITING, total cpu time: 0.0ms, total user time: 0.0ms
java.lang.Object.wait(Native Method)
java.util.TimerThread.mainLoop(Unknown Source)
java.util.TimerThread.run(Unknown Source)
Thread: scheduler_QuartzSchedulerThread, state: TIMED_WAITING, total cpu time: 109.2007ms, total user time: 109.2007ms
java.lang.Object.wait(Native Method)
org.quartz.core.QuartzSchedulerThread.run(QuartzSchedulerThread.java:411)
Thread: scheduler_Worker-1, state: TIMED_WAITING, total cpu time: 31.2002ms, total user time: 31.2002ms
java.lang.Object.wait(Native Method)
org.quartz.simpl.SimpleThreadPool$WorkerThread.run(SimpleThreadPool.java:543)
Thread: pool-1-thread-2, state: WAITING, total cpu time: 0.0ms, total user time: 0.0ms
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.park(Unknown Source)
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(Unknown Source)
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(Unknown Source)
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(Unknown Source)
java.util.concurrent.ThreadPoolExecutor.getTask(Unknown Source)
java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
java.lang.Thread.run(Unknown Source)
Thread: AWT-Windows, state: RUNNABLE, total cpu time: 0.0ms, total user time: 0.0ms
sun.awt.windows.WToolkit.eventLoop(Native Method)
sun.awt.windows.WToolkit.run(Unknown Source)
java.lang.Thread.run(Unknown Source)
Thread: Java2D Disposer, state: WAITING, total cpu time: 0.0ms, total user time: 0.0ms
java.lang.Object.wait(Native Method)
java.lang.ref.ReferenceQueue.remove(Unknown Source)
java.lang.ref.ReferenceQueue.remove(Unknown Source)
sun.java2d.Disposer.run(Unknown Source)
java.lang.Thread.run(Unknown Source)
Thread: GC Daemon, state: TIMED_WAITING, total cpu time: 0.0ms, total user time: 0.0ms
java.lang.Object.wait(Native Method)
sun.misc.GC$Daemon.run(Unknown Source)
Thread: RMI Reaper, state: WAITING, total cpu time: 0.0ms, total user time: 0.0ms
java.lang.Object.wait(Native Method)
java.lang.ref.ReferenceQueue.remove(Unknown Source)
java.lang.ref.ReferenceQueue.remove(Unknown Source)
sun.rmi.transport.ObjectTable$Reaper.run(Unknown Source)
java.lang.Thread.run(Unknown Source)
Thread: pool-1-thread-1, state: WAITING, total cpu time: 202.8013ms, total user time: 187.2012ms
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.park(Unknown Source)
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(Unknown Source)
java.util.concurrent.LinkedBlockingQueue.take(Unknown Source)
java.util.concurrent.ThreadPoolExecutor.getTask(Unknown Source)
java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
java.lang.Thread.run(Unknown Source)
Thread: RMI TCP Accept-1099, state: WAITING, total cpu time: 0.0ms, total user time: 0.0ms
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.park(Unknown Source)
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(Unknown Source)
java.util.concurrent.ArrayBlockingQueue.take(Unknown Source)
com.servoy.j2db.rmi.ServerTwoWaySocketFactory$TwoWayServerSocket.accept(ServerTwoWaySocketFactory.java:603)
sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(Unknown Source)
sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(Unknown Source)
java.lang.Thread.run(Unknown Source)
Thread: 2 way Server Acceptor, state: RUNNABLE, total cpu time: 0.0ms, total user time: 0.0ms
java.net.DualStackPlainSocketImpl.accept0(Native Method)
java.net.DualStackPlainSocketImpl.socketAccept(Unknown Source)
java.net.AbstractPlainSocketImpl.accept(Unknown Source)
java.net.PlainSocketImpl.accept(Unknown Source)
java.net.ServerSocket.implAccept(Unknown Source)
sun.security.ssl.SSLServerSocketImpl.accept(Unknown Source)
com.servoy.j2db.rmi.ServerTwoWaySocketFactory$TwoWayServerSocket.run(ServerTwoWaySocketFactory.java:621)
java.lang.Thread.run(Unknown Source)
Thread: pool-2-thread-1, state: TIMED_WAITING, total cpu time: 15.6001ms, total user time: 15.6001ms
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.parkNanos(Unknown Source)
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(Unknown Source)
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(Unknown Source)
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(Unknown Source)
java.util.concurrent.ThreadPoolExecutor.getTask(Unknown Source)
java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
java.lang.Thread.run(Unknown Source)
Thread: PingScanner, state: TIMED_WAITING, total cpu time: 0.0ms, total user time: 0.0ms
java.lang.Object.wait(Native Method)
com.servoy.j2db.dataprocessing.Zn.run(Zn.java:38)
java.lang.Thread.run(Unknown Source)
Thread: MessageScheduler, state: TIMED_WAITING, total cpu time: 0.0ms, total user time: 0.0ms
java.lang.Object.wait(Native Method)
com.servoy.j2db.dataprocessing.Zp.run(Zp.java:3)
java.lang.Thread.run(Unknown Source)
Thread: DestroyJavaVM, state: RUNNABLE, total cpu time: 655.2042ms, total user time: 546.0035ms
Thread: Wrapper-Connection, state: RUNNABLE, total cpu time: 15.6001ms, total user time: 15.6001ms
java.net.SocketInputStream.socketRead0(Native Method)
java.net.SocketInputStream.read(Unknown Source)
java.net.SocketInputStream.read(Unknown Source)
java.net.SocketInputStream.read(Unknown Source)
java.io.DataInputStream.readByte(Unknown Source)
org.tanukisoftware.wrapper.WrapperManager.handleSocket(WrapperManager.java:4203)
org.tanukisoftware.wrapper.WrapperManager.run(WrapperManager.java:4558)
java.lang.Thread.run(Unknown Source)
Thread: Wrapper-Control-Event-Monitor, state: TIMED_WAITING, total cpu time: 15.6001ms, total user time: 15.6001ms
java.lang.Thread.sleep(Native Method)
org.tanukisoftware.wrapper.WrapperManager$3.run(WrapperManager.java:816)
Thread: Attach Listener, state: RUNNABLE, total cpu time: 0.0ms, total user time: 0.0ms
Thread: Signal Dispatcher, state: RUNNABLE, total cpu time: 0.0ms, total user time: 0.0ms
Thread: Finalizer, state: WAITING, total cpu time: 109.2007ms, total user time: 109.2007ms
java.lang.Object.wait(Native Method)
java.lang.ref.ReferenceQueue.remove(Unknown Source)
java.lang.ref.ReferenceQueue.remove(Unknown Source)
java.lang.ref.Finalizer$FinalizerThread.run(Unknown Source)
Thread: Reference Handler, state: WAITING, total cpu time: 62.4004ms, total user time: 62.4004ms
java.lang.Object.wait(Native Method)
java.lang.Object.wait(Object.java:503)
java.lang.ref.Reference$ReferenceHandler.run(Unknown Source)
Please provide some suggestions on this.
Thanks,
Ashutos.