TeamCity Professional 8.0.3 (build 27540)
OS: Linux 3.2.0-40-virtual, amd64
JVM args: -Djava.util.logging.config.file=/home/ubuntu/TeamCity/conf/logging.properties -Djava.util.logging.manager=org.apache.juli.ClassLoaderLogManager -Xmx512m -XX:MaxPermSize=270m -Dlog4j.configuration=file:/home/ubuntu/TeamCity/bin/../conf/teamcity-server-log4j.xml -Dteamcity_logs=../logs/ -Djava.awt.headless=true -Djava.endorsed.dirs=/home/ubuntu/TeamCity/endorsed -Dcatalina.base=/home/ubuntu/TeamCity -Dcatalina.home=/home/ubuntu/TeamCity -Djava.io.tmpdir=/home/ubuntu/TeamCity/temp 
Database type: HSQL2

2013-09-19 16:38:31
Full thread dump Java HotSpot(TM) 64-Bit Server VM (24.0-b56 mixed mode):

"Alarm (NotificationThread-HangingBuilds)" daemon group="main" prio=5 tid=31 nid=31 waiting 
   java.lang.Thread.State: TIMED_WAITING
    at sun.misc.Unsafe.park(Native Method)
    at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
    at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
    at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1090)
    at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807)
    at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
    at java.lang.Thread.run(Thread.java:724)

"Normal executor 4" group="main" prio=5 tid=58 nid=58 waiting 
   java.lang.Thread.State: WAITING
    at sun.misc.Unsafe.park(Native Method)
    at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
    at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
    at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
    at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
    at java.lang.Thread.run(Thread.java:724)

"Scheduled executor 2" group="main" prio=5 tid=26 nid=26 waiting 
   java.lang.Thread.State: WAITING
    at sun.misc.Unsafe.park(Native Method)
    at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
    at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
    at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1085)
    at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807)
    at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
    at java.lang.Thread.run(Thread.java:724)

"SearchService-BackgroundIndexing" daemon group="main" prio=1 tid=68 nid=68 waiting 
   java.lang.Thread.State: TIMED_WAITING
    at java.lang.Thread.sleep(Native Method)
    at jetbrains.buildServer.serverSide.search.BackgroundIndexer.run(BackgroundIndexer.java:33)
    at java.lang.Thread.run(Thread.java:724)

"Agent messages processing thread 1" daemon group="main" prio=5 tid=25 nid=25 waiting 
   java.lang.Thread.State: TIMED_WAITING
    at java.lang.Object.wait(Native Method)
    at jetbrains.buildServer.serverSide.impl.BuildAgentMessagesQueue$2.run(BuildAgentMessagesQueue.java:0)
    at java.lang.Thread.run(Thread.java:724)

"Scheduled executor 4" group="main" prio=5 tid=29 nid=29 waiting 
   java.lang.Thread.State: WAITING
    at sun.misc.Unsafe.park(Native Method)
    at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
    at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
    at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1085)
    at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807)
    at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
    at java.lang.Thread.run(Thread.java:724)

"Scheduled executor 5" group="main" prio=5 tid=30 nid=30 waiting 
   java.lang.Thread.State: WAITING
    at sun.misc.Unsafe.park(Native Method)
    at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
    at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
    at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1085)
    at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807)
    at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
    at java.lang.Thread.run(Thread.java:724)

"Signal Dispatcher" daemon group="system" prio=9 tid=4 nid=4 runnable 
   java.lang.Thread.State: RUNNABLE

"Store issue-errors Spool Thread" daemon group="main" prio=5 tid=35 nid=35 waiting 
   java.lang.Thread.State: TIMED_WAITING
    at java.lang.Thread.sleep(Native Method)
    at net.sf.ehcache.store.DiskStore.spoolAndExpiryThreadMain(DiskStore.java:620)
    at net.sf.ehcache.store.DiskStore.access$900(DiskStore.java:66)
    at net.sf.ehcache.store.DiskStore$SpoolAndExpiryThread.run(DiskStore.java:1074)

"Low prio executor 1" daemon group="main" prio=4 tid=42 nid=42 waiting 
   java.lang.Thread.State: WAITING
    at sun.misc.Unsafe.park(Native Method)
    at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
    at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
    at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
    at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
    at java.lang.Thread.run(Thread.java:724)

"Amazon connection: profile 'EC2 TC Win IE9'{id=cp6} 1" daemon group="main" prio=5 tid=62 nid=62 waiting 
   java.lang.Thread.State: TIMED_WAITING
    at sun.misc.Unsafe.park(Native Method)
    at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
    at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
    at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1090)
    at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807)
    at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
    at java.lang.Thread.run(Thread.java:724)

"http-nio-8111-exec-1" daemon group="main" prio=5 tid=310 nid=310 waiting 
   java.lang.Thread.State: WAITING
    at sun.misc.Unsafe.park(Native Method)
    at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
    at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
    at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
    at org.apache.tomcat.util.threads.TaskQueue.take(TaskQueue.java:104)
    at org.apache.tomcat.util.threads.TaskQueue.take(TaskQueue.java:32)
    at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
    at java.lang.Thread.run(Thread.java:724)

"Normal executor 1" group="main" prio=5 tid=40 nid=40 waiting 
   java.lang.Thread.State: WAITING
    at sun.misc.Unsafe.park(Native Method)
    at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
    at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
    at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
    at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
    at java.lang.Thread.run(Thread.java:724)

"http-nio-8111-exec-7" daemon group="main" prio=5 tid=316 nid=316 waiting 
   java.lang.Thread.State: WAITING
    at sun.misc.Unsafe.park(Native Method)
    at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
    at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
    at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
    at org.apache.tomcat.util.threads.TaskQueue.take(TaskQueue.java:104)
    at org.apache.tomcat.util.threads.TaskQueue.take(TaskQueue.java:32)
    at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
    at java.lang.Thread.run(Thread.java:724)

"http-nio-8111-ClientPoller-0" daemon group="main" prio=5 tid=18 nid=18 runnable 
   java.lang.Thread.State: RUNNABLE
    at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
    at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:257)
    at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
    at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
    at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
    at org.apache.tomcat.util.net.NioEndpoint$Poller.run(NioEndpoint.java:1162)
    at java.lang.Thread.run(Thread.java:724)

"http-nio-8111-exec-2" daemon group="main" prio=5 tid=311 nid=311 waiting 
   java.lang.Thread.State: WAITING
    at sun.misc.Unsafe.park(Native Method)
    at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
    at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
    at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
    at org.apache.tomcat.util.threads.TaskQueue.take(TaskQueue.java:104)
    at org.apache.tomcat.util.threads.TaskQueue.take(TaskQueue.java:32)
    at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
    at java.lang.Thread.run(Thread.java:724)

"GC Daemon" daemon group="system" prio=2 tid=10 nid=10 waiting 
   java.lang.Thread.State: TIMED_WAITING
    at java.lang.Object.wait(Native Method)
    at sun.misc.GC$Daemon.run(GC.java:117)

"VCS Periodical executor 3" group="main" prio=5 tid=53 nid=53 waiting 
   java.lang.Thread.State: TIMED_WAITING
    at sun.misc.Unsafe.park(Native Method)
    at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
    at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
    at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
    at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
    at java.lang.Thread.run(Thread.java:724)

"Amazon connection: profile 'EC2 TC Ubuntu Agents'{id=cp1} 1" daemon group="main" prio=5 tid=64 nid=64 waiting 
   java.lang.Thread.State: TIMED_WAITING
    at sun.misc.Unsafe.park(Native Method)
    at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
    at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
    at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1090)
    at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807)
    at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
    at java.lang.Thread.run(Thread.java:724)

"Ping" daemon group="main" prio=5 tid=46 nid=46 waiting 
   java.lang.Thread.State: TIMED_WAITING
    at java.lang.Object.wait(Native Method)
    at jetbrains.buildServer.serverSide.impl.BuildServerRunner$BuildServerWorker.run(BuildServerRunner.java:47)
    at java.lang.Thread.run(Thread.java:724)

"Memory Usage Watcher" daemon group="main" prio=5 tid=69 nid=69 waiting 
   java.lang.Thread.State: TIMED_WAITING
    at java.lang.Thread.sleep(Native Method)
    at jetbrains.buildServer.web.MemoryUsageMonitor$1.run(MemoryUsageMonitor.java:13)
    at java.lang.Thread.run(Thread.java:724)

"http-nio-8111-exec-4" daemon group="main" prio=5 tid=313 nid=313 waiting 
   java.lang.Thread.State: WAITING
    at sun.misc.Unsafe.park(Native Method)
    at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
    at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
    at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
    at org.apache.tomcat.util.threads.TaskQueue.take(TaskQueue.java:104)
    at org.apache.tomcat.util.threads.TaskQueue.take(TaskQueue.java:32)
    at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
    at java.lang.Thread.run(Thread.java:724)

"Scheduled executor 3" group="main" prio=5 tid=28 nid=28 waiting 
   java.lang.Thread.State: TIMED_WAITING
    at sun.misc.Unsafe.park(Native Method)
    at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
    at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
    at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1090)
    at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807)
    at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
    at java.lang.Thread.run(Thread.java:724)

"Usage Statistics Collector" daemon group="main" prio=5 tid=36 nid=36 waiting 
   java.lang.Thread.State: WAITING
    at java.lang.Object.wait(Native Method)
    at java.lang.Object.wait(Object.java:503)
    at jetbrains.buildServer.usageStatistics.impl.UsageStatisticsCollectorImpl.waitForEvent(UsageStatisticsCollectorImpl.java:199)
    at jetbrains.buildServer.usageStatistics.impl.UsageStatisticsCollectorImpl.run(UsageStatisticsCollectorImpl.java:132)
    at java.lang.Thread.run(Thread.java:724)

"Build estimates updater 1" daemon group="main" prio=5 tid=44 nid=44 waiting 
   java.lang.Thread.State: TIMED_WAITING
    at java.lang.Object.wait(Native Method)
    at jetbrains.buildServer.serverSide.impl.timeEstimation.CachingBuildEstimator$2.run(CachingBuildEstimator.java:11)
    at java.lang.Thread.run(Thread.java:724)

"Agent messages processing thread 0" daemon group="main" prio=5 tid=24 nid=24 waiting 
   java.lang.Thread.State: TIMED_WAITING
    at java.lang.Object.wait(Native Method)
    at jetbrains.buildServer.serverSide.impl.BuildAgentMessagesQueue$2.run(BuildAgentMessagesQueue.java:0)
    at java.lang.Thread.run(Thread.java:724)

"http-nio-8111-Acceptor-0" daemon group="main" prio=5 tid=19 nid=19 runnable 
   java.lang.Thread.State: RUNNABLE
    at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
    at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:241)
    at org.apache.tomcat.util.net.NioEndpoint$Acceptor.run(NioEndpoint.java:793)
    at java.lang.Thread.run(Thread.java:724)

"Test failure rate persisting thread" daemon group="main" prio=5 tid=32 nid=32 waiting 
   java.lang.Thread.State: WAITING
    at sun.misc.Unsafe.park(Native Method)
    at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
    at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
    at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
    at jetbrains.buildServer.serverSide.stat.TestFailureRateCollector.executeCommands(TestFailureRateCollector.java:145)
    at jetbrains.buildServer.serverSide.stat.TestFailureRateCollector.processPersistingQueue(TestFailureRateCollector.java:80)
    at jetbrains.buildServer.serverSide.stat.TestFailureRateCollector.access$400(TestFailureRateCollector.java:109)
    at jetbrains.buildServer.serverSide.stat.TestFailureRateCollector$3.run(TestFailureRateCollector.java)
    at java.lang.Thread.run(Thread.java:724)

"ContainerBackgroundProcessor[StandardEngine[Catalina]]" daemon group="main" prio=5 tid=17 nid=17 waiting 
   java.lang.Thread.State: TIMED_WAITING
    at java.lang.Thread.sleep(Native Method)
    at org.apache.catalina.core.ContainerBase$ContainerBackgroundProcessor.run(ContainerBase.java:1508)
    at java.lang.Thread.run(Thread.java:724)

"java-sdk-http-connection-reaper" daemon group="main" prio=5 tid=61 nid=61 waiting 
   java.lang.Thread.State: TIMED_WAITING
    at java.lang.Thread.sleep(Native Method)
    at com.amazonaws.http.IdleConnectionReaper.run(IdleConnectionReaper.java:75)

"Scheduled executor 1" group="main" prio=5 tid=23 nid=23 waiting 
   java.lang.Thread.State: WAITING
    at sun.misc.Unsafe.park(Native Method)
    at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
    at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
    at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1085)
    at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807)
    at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
    at java.lang.Thread.run(Thread.java:724)

"Change Observer 1" daemon group="main" prio=5 tid=14 nid=14 waiting 
   java.lang.Thread.State: TIMED_WAITING
    at sun.misc.Unsafe.park(Native Method)
    at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
    at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
    at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1090)
    at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807)
    at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
    at java.lang.Thread.run(Thread.java:724)

"http-nio-8111-exec-3" daemon group="main" prio=5 tid=312 nid=312 waiting 
   java.lang.Thread.State: WAITING
    at sun.misc.Unsafe.park(Native Method)
    at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
    at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
    at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
    at org.apache.tomcat.util.threads.TaskQueue.take(TaskQueue.java:104)
    at org.apache.tomcat.util.threads.TaskQueue.take(TaskQueue.java:32)
    at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
    at java.lang.Thread.run(Thread.java:724)

"Flush Queue" daemon group="main" prio=5 tid=47 nid=47 waiting 
   java.lang.Thread.State: TIMED_WAITING
    at java.util.ArrayList.<init>(ArrayList.java:167)
    at org.apache.commons.dbcp.AbandonedTrace.getTrace(AbandonedTrace.java:173)
    at org.apache.commons.dbcp.DelegatingStatement.close(DelegatingStatement.java:159)
    at org.apache.commons.dbcp.DelegatingStatement.close(DelegatingStatement.java:168)
    at org.apache.commons.dbcp.DelegatingStatement.close(DelegatingStatement.java:168)
    at jetbrains.buildServer.serverSide.db.DatabaseUtil.close(DatabaseUtil.java:874)
    at jetbrains.buildServer.serverSide.db.queries.GenericQuery.underPreparedStatement(GenericQuery.java:540)
    at jetbrains.buildServer.serverSide.db.queries.GenericQuery.access$700(GenericQuery.java:28)
    at jetbrains.buildServer.serverSide.db.queries.GenericQuery$10.apply(GenericQuery.java:524)
    at jetbrains.buildServer.serverSide.db.DBFunctions.withDB(DBFunctions.java:2423)
    at jetbrains.buildServer.serverSide.db.queries.GenericQuery.underPreparedStatement(GenericQuery.java:521)
    at jetbrains.buildServer.serverSide.db.queries.GenericQuery.doExecuteQuery(GenericQuery.java:444)
    at jetbrains.buildServer.serverSide.db.queries.GenericQuery.access$100(GenericQuery.java:28)
    at jetbrains.buildServer.serverSide.db.queries.GenericQuery$1.run(GenericQuery.java:136)
    at jetbrains.buildServer.serverSide.db.DBFacade$5.doInConnection(DBFacade.java:393)
    at jetbrains.buildServer.serverSide.db.DBFacade$6.doInConnection(DBFacade.java:417)
    at org.springframework.jdbc.core.JdbcTemplate.execute(JdbcTemplate.java:341)
    at jetbrains.buildServer.serverSide.db.DBFacade._runSql(DBFacade.java:413)
    at jetbrains.buildServer.serverSide.db.DBFacade.runSql(DBFacade.java:391)
    at jetbrains.buildServer.serverSide.db.queries.GenericQuery.execute(GenericQuery.java:139)
    at jetbrains.buildServer.clouds.server.impl.DBCloudStateManagerImpl$CloudImageTable.isImageWithAgent(DBCloudStateManagerImpl.java:130)
    at jetbrains.buildServer.clouds.server.impl.DBCloudStateManagerImpl.isImageWithAgent(DBCloudStateManagerImpl.java:35)
    at jetbrains.buildServer.clouds.server.impl.CloudManagerFacade.isImageWithAgent(CloudManagerFacade.java:96)
    at jetbrains.buildServer.clouds.server.impl.SecuredCloudManager.isImageWithAgent(SecuredCloudManager.java:82)
    at jetbrains.buildServer.clouds.server.impl.virtual.VirtualAgentsProviderImpl.getStartableAgents(VirtualAgentsProviderImpl.java:50)
    at jetbrains.buildServer.serverSide.impl.virtualAgent.VirtualAgentsManagerImpl.getStartableAgentTypes(VirtualAgentsManagerImpl.java:11)
    at jetbrains.buildServer.serverSide.impl.FlushQueueVirtualAction.distribute(FlushQueueVirtualAction.java:55)
    at jetbrains.buildServer.serverSide.impl.FlushQueueTemplate.flushRealAgents(FlushQueueTemplate.java:30)
    at jetbrains.buildServer.serverSide.impl.FlushQueueTemplate.access$100(FlushQueueTemplate.java:28)
    at jetbrains.buildServer.serverSide.impl.FlushQueueTemplate$2.run(FlushQueueTemplate.java:2)
    at jetbrains.buildServer.serverSide.impl.FlushQueueAction$QueueAction.run(FlushQueueAction.java:1)
    at jetbrains.buildServer.serverSide.impl.cleanup.ServerCleanupManagerImpl.executeWithInactiveCleanup(ServerCleanupManagerImpl.java:54)
    at jetbrains.buildServer.serverSide.impl.cleanup.ServerCleanupManagerImpl$$FastClassByCGLIB$$ba2c8525.invoke(<generated>)
    at net.sf.cglib.proxy.MethodProxy.invoke(MethodProxy.java:149)
    at org.springframework.aop.framework.Cglib2AopProxy$CglibMethodInvocation.invokeJoinpoint(Cglib2AopProxy.java:688)
    at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:150)
    at jetbrains.buildServer.serverSide.impl.auth.TeamCityMethodSecurityInterceptor.invoke(TeamCityMethodSecurityInterceptor.java:17)
    at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:172)
    at org.springframework.aop.framework.Cglib2AopProxy$FixedChainStaticTargetInterceptor.intercept(Cglib2AopProxy.java:572)
    at jetbrains.buildServer.serverSide.impl.cleanup.ServerCleanupManagerImpl$$EnhancerByCGLIB$$e3825840.executeWithInactiveCleanup(<generated>)
    at jetbrains.buildServer.serverSide.impl.QueueFlushImpl.flushQueue(QueueFlushImpl.java:13)
    at jetbrains.buildServer.serverSide.impl.BuildServerImpl.flushQueue(BuildServerImpl.java:94)
    at jetbrains.buildServer.serverSide.impl.BuildServerRunner$2.doSomething(BuildServerRunner.java)
    at jetbrains.buildServer.serverSide.impl.BuildServerRunner$BuildServerWorker.runAction(BuildServerRunner.java:37)
    at jetbrains.buildServer.serverSide.impl.BuildServerRunner$BuildServerWorker.run(BuildServerRunner.java:4)
    at java.lang.Thread.run(Thread.java:724)

"Reference Handler" daemon group="system" prio=10 tid=2 nid=2 waiting 
   java.lang.Thread.State: WAITING
    at java.lang.Object.wait(Native Method)
    at java.lang.Object.wait(Object.java:503)
    at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:133)

"http-nio-8111-exec-5" daemon group="main" prio=5 tid=314 nid=314 waiting 
   java.lang.Thread.State: WAITING
    at sun.misc.Unsafe.park(Native Method)
    at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
    at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
    at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
    at org.apache.tomcat.util.threads.TaskQueue.take(TaskQueue.java:104)
    at org.apache.tomcat.util.threads.TaskQueue.take(TaskQueue.java:32)
    at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
    at java.lang.Thread.run(Thread.java:724)

"Agent Distribution Monitor 1" group="main" prio=5 tid=57 nid=57 waiting 
   java.lang.Thread.State: TIMED_WAITING
    at sun.misc.Unsafe.park(Native Method)
    at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
    at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
    at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1090)
    at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807)
    at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
    at java.lang.Thread.run(Thread.java:724)

"http-nio-8111-exec-8" daemon group="main" prio=5 tid=317 nid=317 waiting 
   java.lang.Thread.State: WAITING
    at sun.misc.Unsafe.park(Native Method)
    at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
    at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
    at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
    at org.apache.tomcat.util.threads.TaskQueue.take(TaskQueue.java:104)
    at org.apache.tomcat.util.threads.TaskQueue.take(TaskQueue.java:32)
    at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
    at java.lang.Thread.run(Thread.java:724)

"Store tables Spool Thread" daemon group="main" prio=5 tid=43 nid=43 waiting 
   java.lang.Thread.State: TIMED_WAITING
    at java.lang.Thread.sleep(Native Method)
    at net.sf.ehcache.store.DiskStore.spoolAndExpiryThreadMain(DiskStore.java:620)
    at net.sf.ehcache.store.DiskStore.access$900(DiskStore.java:66)
    at net.sf.ehcache.store.DiskStore$SpoolAndExpiryThread.run(DiskStore.java:1074)

"DBMaintenanceScheduler" daemon group="main" prio=5 tid=38 nid=38 waiting 
   java.lang.Thread.State: TIMED_WAITING
    at java.lang.Object.wait(Native Method)
    at java.util.TimerThread.mainLoop(Timer.java:552)
    at java.util.TimerThread.run(Timer.java:505)

"http-nio-8111-exec-9" daemon group="main" prio=5 tid=318 nid=318 waiting 
   java.lang.Thread.State: WAITING
    at sun.misc.Unsafe.park(Native Method)
    at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
    at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
    at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
    at org.apache.tomcat.util.threads.TaskQueue.take(TaskQueue.java:104)
    at org.apache.tomcat.util.threads.TaskQueue.take(TaskQueue.java:32)
    at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
    at java.lang.Thread.run(Thread.java:724)

"http-nio-8111-exec-10" daemon group="main" prio=5 tid=319 nid=319 waiting 
   java.lang.Thread.State: WAITING
    at sun.misc.Unsafe.park(Native Method)
    at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
    at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
    at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
    at org.apache.tomcat.util.threads.TaskQueue.take(TaskQueue.java:104)
    at org.apache.tomcat.util.threads.TaskQueue.take(TaskQueue.java:32)
    at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
    at java.lang.Thread.run(Thread.java:724)

"NuGet Packages Version Checker 3" daemon group="main" prio=5 tid=140 nid=140 waiting 
   java.lang.Thread.State: WAITING
    at sun.misc.Unsafe.park(Native Method)
    at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
    at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
    at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1085)
    at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807)
    at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
    at java.lang.Thread.run(Thread.java:724)

"Store buildsToReindex Spool Thread" daemon group="main" prio=5 tid=60 nid=60 waiting 
   java.lang.Thread.State: TIMED_WAITING
    at java.lang.Thread.sleep(Native Method)
    at net.sf.ehcache.store.DiskStore.spoolAndExpiryThreadMain(DiskStore.java:620)
    at net.sf.ehcache.store.DiskStore.access$900(DiskStore.java:66)
    at net.sf.ehcache.store.DiskStore$SpoolAndExpiryThread.run(DiskStore.java:1074)

"VCS Periodical executor 18" group="main" prio=5 tid=254 nid=254 waiting 
   java.lang.Thread.State: TIMED_WAITING
    at sun.misc.Unsafe.park(Native Method)
    at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
    at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
    at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
    at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
    at java.lang.Thread.run(Thread.java:724)

"Normal executor 2" group="main" prio=5 tid=41 nid=41 waiting 
   java.lang.Thread.State: WAITING
    at sun.misc.Unsafe.park(Native Method)
    at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
    at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
    at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
    at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
    at java.lang.Thread.run(Thread.java:724)

"Finalizer" daemon group="system" prio=8 tid=3 nid=3 waiting 
   java.lang.Thread.State: WAITING
    at java.lang.Object.wait(Native Method)
    at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:135)
    at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:151)
    at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:189)

"NuGet Packages Version Checker 2" daemon group="main" prio=5 tid=110 nid=110 waiting 
   java.lang.Thread.State: WAITING
    at sun.misc.Unsafe.park(Native Method)
    at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
    at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
    at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1085)
    at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807)
    at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
    at java.lang.Thread.run(Thread.java:724)

"Abandoned connection cleanup thread" daemon group="main" prio=5 tid=21 nid=21 waiting 
   java.lang.Thread.State: WAITING
    at java.lang.Object.wait(Native Method)
    at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:135)
    at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:151)
    at com.mysql.jdbc.NonRegisteringDriver$1.run(NonRegisteringDriver.java:93)

"Amazon connection: profile 'EC2 TC Win IE8'{id=cp2} 1" daemon group="main" prio=5 tid=63 nid=63 waiting 
   java.lang.Thread.State: TIMED_WAITING
    at sun.misc.Unsafe.park(Native Method)
    at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
    at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
    at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1090)
    at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807)
    at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
    at java.lang.Thread.run(Thread.java:724)

"VCS Periodical executor 19" group="main" prio=5 tid=267 nid=267 waiting 
   java.lang.Thread.State: TIMED_WAITING
    at sun.misc.Unsafe.park(Native Method)
    at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
    at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
    at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
    at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
    at java.lang.Thread.run(Thread.java:724)

"Store issues Spool Thread" daemon group="main" prio=5 tid=34 nid=34 waiting 
   java.lang.Thread.State: TIMED_WAITING
    at java.lang.Thread.sleep(Native Method)
    at net.sf.ehcache.store.DiskStore.spoolAndExpiryThreadMain(DiskStore.java:620)
    at net.sf.ehcache.store.DiskStore.access$900(DiskStore.java:66)
    at net.sf.ehcache.store.DiskStore$SpoolAndExpiryThread.run(DiskStore.java:1074)

"VCS Periodical executor 20" group="main" prio=5 tid=281 nid=281 waiting 
   java.lang.Thread.State: TIMED_WAITING
    at sun.misc.Unsafe.park(Native Method)
    at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
    at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
    at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
    at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
    at java.lang.Thread.run(Thread.java:724)

"Alarm (Disk space watcher)" daemon group="main" prio=5 tid=33 nid=33 waiting 
   java.lang.Thread.State: TIMED_WAITING
    at sun.misc.Unsafe.park(Native Method)
    at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
    at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
    at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1090)
    at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807)
    at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
    at java.lang.Thread.run(Thread.java:724)

"TeamCityStartup" daemon group="main" prio=5 tid=16 nid=16 waiting 
   java.lang.Thread.State: TIMED_WAITING
    at java.lang.Object.wait(Native Method)
    at jetbrains.buildServer.maintenance.StartupContext.waitForStageLock(StartupContext.java:122)
    at jetbrains.buildServer.maintenance.StartupContext.waitForFollowingStage(StartupContext.java:143)
    at jetbrains.buildServer.maintenance.StartupProcessor.processTeamCityLifecycle(StartupProcessor.java:469)
    at jetbrains.buildServer.maintenance.StartupProcessor.access$000(StartupProcessor.java:320)
    at jetbrains.buildServer.maintenance.StartupProcessor$1.run(StartupProcessor.java:2)
    at java.lang.Thread.run(Thread.java:724)

"Vcs Monitor" daemon group="main" prio=5 tid=48 nid=48 waiting 
   java.lang.Thread.State: TIMED_WAITING
    at java.lang.Object.wait(Native Method)
    at jetbrains.buildServer.serverSide.impl.BuildServerRunner$BuildServerWorker.run(BuildServerRunner.java:47)
    at java.lang.Thread.run(Thread.java:724)

"Store Subversion Externals Spool Thread" daemon group="main" prio=5 tid=37 nid=37 waiting 
   java.lang.Thread.State: TIMED_WAITING
    at java.lang.Thread.sleep(Native Method)
    at net.sf.ehcache.store.DiskStore.spoolAndExpiryThreadMain(DiskStore.java:620)
    at net.sf.ehcache.store.DiskStore.access$900(DiskStore.java:66)
    at net.sf.ehcache.store.DiskStore$SpoolAndExpiryThread.run(DiskStore.java:1074)

"main" group="main" prio=5 tid=1 nid=1 runnable 
   java.lang.Thread.State: RUNNABLE
    at java.net.PlainSocketImpl.socketAccept(Native Method)
    at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:398)
    at java.net.ServerSocket.implAccept(ServerSocket.java:530)
    at java.net.ServerSocket.accept(ServerSocket.java:498)
    at org.apache.catalina.core.StandardServer.await(StandardServer.java:452)
    at org.apache.catalina.startup.Catalina.await(Catalina.java:766)
    at org.apache.catalina.startup.Catalina.start(Catalina.java:712)
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.lang.reflect.Method.invoke(Method.java:606)
    at org.apache.catalina.startup.Bootstrap.start(Bootstrap.java:322)
    at org.apache.catalina.startup.Bootstrap.main(Bootstrap.java:456)

"Amazon connection: profile 'EC2 TC Win Chrome'{id=cp3} 1" daemon group="main" prio=5 tid=66 nid=66 waiting 
   java.lang.Thread.State: TIMED_WAITING
    at sun.misc.Unsafe.park(Native Method)
    at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
    at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
    at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1090)
    at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807)
    at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
    at java.lang.Thread.run(Thread.java:724)

"Store contentCache Spool Thread" daemon group="main" prio=5 tid=27 nid=27 waiting 
   java.lang.Thread.State: TIMED_WAITING
    at java.lang.Thread.sleep(Native Method)
    at net.sf.ehcache.store.DiskStore.spoolAndExpiryThreadMain(DiskStore.java:620)
    at net.sf.ehcache.store.DiskStore.access$900(DiskStore.java:66)
    at net.sf.ehcache.store.DiskStore$SpoolAndExpiryThread.run(DiskStore.java:1074)

"VCS Periodical executor 17" group="main" prio=5 tid=240 nid=240 waiting 
   java.lang.Thread.State: TIMED_WAITING
    at sun.misc.Unsafe.park(Native Method)
    at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
    at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
    at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
    at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
    at java.lang.Thread.run(Thread.java:724)

"Amazon connection: profile 'EC2 TC Win FF'{id=cp4} 1" daemon group="main" prio=5 tid=65 nid=65 waiting 
   java.lang.Thread.State: TIMED_WAITING
    at sun.misc.Unsafe.park(Native Method)
    at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
    at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
    at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1090)
    at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807)
    at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
    at java.lang.Thread.run(Thread.java:724)

"NuGet Packages Version Checker 1" daemon group="main" prio=5 tid=67 nid=67 waiting 
   java.lang.Thread.State: TIMED_WAITING
    at sun.misc.Unsafe.park(Native Method)
    at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
    at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
    at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1090)
    at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807)
    at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
    at java.lang.Thread.run(Thread.java:724)

"Alarm (VersionChecker)" daemon group="main" prio=5 tid=70 nid=70 waiting 
   java.lang.Thread.State: TIMED_WAITING
    at sun.misc.Unsafe.park(Native Method)
    at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
    at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
    at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1090)
    at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807)
    at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
    at java.lang.Thread.run(Thread.java:724)

"ObsoleteBuildsKiller 1" group="main" prio=5 tid=50 nid=50 waiting 
   java.lang.Thread.State: TIMED_WAITING
    at sun.misc.Unsafe.park(Native Method)
    at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
    at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
    at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1090)
    at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807)
    at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
    at java.lang.Thread.run(Thread.java:724)

"Thread-3" daemon group="main" prio=5 tid=15 nid=15 waiting 
   java.lang.Thread.State: TIMED_WAITING
    at java.lang.Thread.sleep(Native Method)
    at org.apache.log4j.helpers.FileWatchdog.run(FileWatchdog.java:103)

"VCS Periodical executor 22" group="main" prio=5 tid=320 nid=320 waiting 
   java.lang.Thread.State: TIMED_WAITING
    at sun.misc.Unsafe.park(Native Method)
    at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
    at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
    at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
    at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
    at java.lang.Thread.run(Thread.java:724)

"NuGet Packages Version Checker 4" daemon group="main" prio=5 tid=168 nid=168 waiting 
   java.lang.Thread.State: WAITING
    at sun.misc.Unsafe.park(Native Method)
    at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
    at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
    at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1085)
    at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807)
    at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
    at java.lang.Thread.run(Thread.java:724)

"HSQLDB Timer @3cc5d56a" daemon group="main" prio=5 tid=22 nid=22 waiting 
   java.lang.Thread.State: TIMED_WAITING
    at java.lang.Object.wait(Native Method)
    at org.hsqldb.lib.HsqlTimer$TaskQueue.park(Unknown Source)
    at org.hsqldb.lib.HsqlTimer.nextTask(Unknown Source)
    at org.hsqldb.lib.HsqlTimer$TaskRunner.run(Unknown Source)
    at java.lang.Thread.run(Thread.java:724)

"Alarm (GetRunningBuildsAction Alarm)" daemon group="main" prio=5 tid=334 nid=334 waiting 
   java.lang.Thread.State: WAITING
    at sun.misc.Unsafe.park(Native Method)
    at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
    at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
    at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1079)
    at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807)
    at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
    at java.lang.Thread.run(Thread.java:724)

"Build Triggers Monitor" daemon group="main" prio=5 tid=49 nid=49 waiting 
   java.lang.Thread.State: TIMED_WAITING
    at java.lang.Object.wait(Native Method)
    at jetbrains.buildServer.serverSide.impl.BuildServerRunner$BuildServerWorker.run(BuildServerRunner.java:47)
    at java.lang.Thread.run(Thread.java:724)

"NioBlockingSelector.BlockPoller-1" daemon group="main" prio=5 tid=11 nid=11 runnable 
   java.lang.Thread.State: RUNNABLE
    at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
    at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:257)
    at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
    at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
    at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
    at org.apache.tomcat.util.net.NioBlockingSelector$BlockPoller.run(NioBlockingSelector.java:327)

"Normal executor 3" group="main" prio=5 tid=45 nid=45 waiting 
   java.lang.Thread.State: WAITING
    at sun.misc.Unsafe.park(Native Method)
    at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
    at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
    at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
    at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
    at java.lang.Thread.run(Thread.java:724)


Memory usage:
   Code Cache: used = 9.70 MB committed = 9.87 MB max = 48.0 MB used/max = 20.22%
   Eden Space: used = 114.80 MB committed = 115.75 MB max = 136.50 MB used/max = 84.10%
   Survivor Space: used = 7.19 MB committed = 14.43 MB max = 17.06 MB used/max = 42.15%
   Tenured Gen: used = 201.94 MB committed = 289.02 MB max = 341.37 MB used/max = 59.16%
   Perm Gen: used = 88.17 MB committed = 88.37 MB max = 270.0 MB used/max = 32.66%
   Total: used = 421.82 MB max = 812.93 MB used/max = 51.89%
