Probably related also seeing some deadlock with threads stuck like so (note that these threads have a database connection so that they in turn cause other threads to block on acquiring one):

"217593032@qtp-868385441-156" prio=10 tid=0x0000000001af3800 nid=0x303c waiting on condition [0x000000004dd15000..0x000000004dd19d80]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00007fdc9d0e7c90> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:773)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireShared(AbstractQueuedSynchronizer.java:903)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireShared(AbstractQueuedSynchronizer.java:1219)
at java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.lock(ReentrantReadWriteLock.java:627)
at net.liftweb.util.ConcurrentLock.read(ConcurrentLock.scala:25)
at net.liftweb.util.InMemoryCache.get(TemplateCache.scala:77)
at net.liftweb.util.InMemoryCache.get(TemplateCache.scala:70)
at net.liftweb.http.TemplateFinder$.findAnyTemplate(LiftSession.scala:1292)
at net.liftweb.http.LiftSession.net$liftweb$http$LiftSession$$findVisibleTemplate(LiftSession.scala:661)
at net.liftweb.http.LiftSession$$anonfun$processTemplate$1$1.apply(LiftSession.scala:473)
at net.liftweb.http.LiftSession$$anonfun$processTemplate$1$1.apply(LiftSession.scala:473)
at net.liftweb.common.EmptyBox.or(Box.scala:378)
at net.liftweb.http.LiftSession.processTemplate$1(LiftSession.scala:473)
at net.liftweb.http.LiftSession$$anonfun$10$$anonfun$apply$34$$anonfun$apply$37.apply(LiftSession.scala:527)
at net.liftweb.http.LiftSession$$anonfun$10$$anonfun$apply$34$$anonfun$apply$37.apply(LiftSession.scala:527)
at net.liftweb.common.EmptyBox.or(Box.scala:378)
at net.liftweb.http.LiftSession$$anonfun$10$$anonfun$apply$34.apply(LiftSession.scala:526)
at net.liftweb.http.LiftSession$$anonfun$10$$anonfun$apply$34.apply(LiftSession.scala:526)
at net.liftweb.util.StackableMaker$class.doWith(Maker.scala:104)
at net.liftweb.http.Factory$FactoryMaker.doWith(Factory.scala:37)
at net.liftweb.util.StackableMaker$class.doWith(Maker.scala:98)
at net.liftweb.http.Factory$FactoryMaker.doWith(Factory.scala:37)
at net.liftweb.http.LiftSession$$anonfun$10.apply(LiftSession.scala:525)
at net.liftweb.http.LiftSession$$anonfun$10.apply(LiftSession.scala:520)
at net.liftweb.common.EmptyBox.or(Box.scala:378)
at net.liftweb.http.LiftSession.processRequest(LiftSession.scala:520)
at net.liftweb.http.LiftServlet.net$liftweb$http$LiftServlet$$dispatchStatefulRequest(LiftServlet.scala:269)
at net.liftweb.http.LiftServlet$$anonfun$3.apply(LiftServlet.scala:177)
at net.liftweb.http.LiftServlet$$anonfun$3.apply(LiftServlet.scala:177)
at net.liftweb.http.S$.net$liftweb$http$S$$wrapQuery(S.scala:971)
at net.liftweb.http.S$$anonfun$net$liftweb$http$S$$_nest2InnerInit$1$$anonfun$apply$25.apply(S.scala:1111)
at net.liftweb.http.S$.net$liftweb$http$S$$doAround(S.scala:908)
at net.liftweb.http.S$$anonfun$net$liftweb$http$S$$doAround$1.apply(S.scala:909)
at com.foursquare.model.ExtendedSession$myWrapper$.apply(ExtendedSession.scala:29)
at net.liftweb.http.S$.net$liftweb$http$S$$doAround(S.scala:909)
at net.liftweb.http.S$$anonfun$net$liftweb$http$S$$doAround$1.apply(S.scala:909)
at bootstrap.liftweb.Boot$$anon$1$$anonfun$apply$3.apply(Boot.scala:78)
at bootstrap.liftweb.Boot$$anon$1$$anonfun$apply$3.apply(Boot.scala:78)
at net.liftweb.mapper.DB$$anonfun$use$1.apply(DB.scala:570)
at net.liftweb.util.DynoVar$class.run(ThreadGlobal.scala:95)
at net.liftweb.mapper.DB$currentConn$.run(DB.scala:557)
at net.liftweb.mapper.DB$.use(DB.scala:567)
at bootstrap.liftweb.Boot$$anon$1.apply(Boot.scala:78)
at net.liftweb.http.S$.net$liftweb$http$S$$doAround(S.scala:909)
at net.liftweb.http.S$$anonfun$net$liftweb$http$S$$doAround$1.apply(S.scala:909)
at net.liftweb.mapper.DB$$anon$1$$anonfun$apply$10.recurseMe$1(DB.scala:188)
at net.liftweb.mapper.DB$$anon$1$$anonfun$apply$10$$anonfun$recurseMe$1$1.apply(DB.scala:195)
at net.liftweb.mapper.DB$$anon$1$$anonfun$apply$10$$anonfun$recurseMe$1$1.apply(DB.scala:195)
at net.liftweb.mapper.DB$$anonfun$use$1.apply(DB.scala:570)
at net.liftweb.util.DynoVar$class.run(ThreadGlobal.scala:95)
at net.liftweb.mapper.DB$currentConn$.run(DB.scala:557)
at net.liftweb.mapper.DB$.use(DB.scala:567)
at net.liftweb.mapper.DB$$anon$1$$anonfun$apply$10.recurseMe$1(DB.scala:195)
at net.liftweb.mapper.DB$$anon$1$$anonfun$apply$10.apply(DB.scala:197)
at net.liftweb.util.DynoVar$class.run(ThreadGlobal.scala:95)
at net.liftweb.mapper.DB$$anon$1$DepthCnt$.run(DB.scala:178)
at net.liftweb.mapper.DB$$anon$1.apply(DB.scala:181)
at net.liftweb.http.S$.net$liftweb$http$S$$doAround(S.scala:909)
at net.liftweb.http.S$$anonfun$net$liftweb$http$S$$_nest2InnerInit$1.apply(S.scala:1109)
at net.liftweb.util.ThreadGlobal.doWith(ThreadGlobal.scala:71)
at net.liftweb.http.S$.net$liftweb$http$S$$_nest2InnerInit(S.scala:1108)
at net.liftweb.http.S$$anonfun$net$liftweb$http$S$$_innerInit$1$$anonfun$apply$28$$anonfun$apply$29$$anonfun$apply$30$$anonfun$apply$31.apply(S.scala:1134)
at net.liftweb.util.ThreadGlobal.doWith(ThreadGlobal.scala:71)
at net.liftweb.http.S$$anonfun$net$liftweb$http$S$$_innerInit$1$$anonfun$apply$28$$anonfun$apply$29$$anonfun$apply$30.apply(S.scala:1133)
at net.liftweb.util.ThreadGlobal.doWith(ThreadGlobal.scala:71)
at net.liftweb.http.S$$anonfun$net$liftweb$http$S$$_innerInit$1$$anonfun$apply$28$$anonfun$apply$29.apply(S.scala:1132)
at net.liftweb.util.ThreadGlobal.doWith(ThreadGlobal.scala:71)
at net.liftweb.http.S$$anonfun$net$liftweb$http$S$$_innerInit$1$$anonfun$apply$28.apply(S.scala:1131)
at net.liftweb.util.ThreadGlobal.doWith(ThreadGlobal.scala:71)
at net.liftweb.http.S$$anonfun$net$liftweb$http$S$$_innerInit$1.apply(S.scala:1130)
at net.liftweb.util.ThreadGlobal.doWith(ThreadGlobal.scala:71)
at net.liftweb.http.S$.net$liftweb$http$S$$_innerInit(S.scala:1129)
at net.liftweb.http.S$$anonfun$_init$1$$anonfun$apply$38$$anonfun$apply$39$$anonfun$apply$40$$anonfun$apply$41$$anonfun$apply$42.apply(S.scala:1176)
at net.liftweb.util.ThreadGlobal.doWith(ThreadGlobal.scala:71)
at net.liftweb.http.S$$anonfun$_init$1$$anonfun$apply$38$$anonfun$apply$39$$anonfun$apply$40$$anonfun$apply$41.apply(S.scala:1175)
at net.liftweb.http.CoreRequestVarHandler$class.apply(Vars.scala:327)
at net.liftweb.http.RequestVarHandler$.apply(Vars.scala:250)
at net.liftweb.http.S$$anonfun$_init$1$$anonfun$apply$38$$anonfun$apply$39$$anonfun$apply$40.apply(S.scala:1174)
at net.liftweb.http.CoreRequestVarHandler$class.apply(Vars.scala:327)
at net.liftweb.http.TransientRequestVarHandler$.apply(Vars.scala:254)
at net.liftweb.http.S$$anonfun$_init$1$$anonfun$apply$38$$anonfun$apply$39.apply(S.scala:1173)
at net.liftweb.util.ThreadGlobal.doWith(ThreadGlobal.scala:71)
at net.liftweb.http.S$$anonfun$_init$1$$anonfun$apply$38.apply(S.scala:1172)
at net.liftweb.util.ThreadGlobal.doWith(ThreadGlobal.scala:71)
at net.liftweb.http.S$$anonfun$_init$1.apply(S.scala:1171)
at net.liftweb.util.ThreadGlobal.doWith(ThreadGlobal.scala:71)
at net.liftweb.http.S$._init(S.scala:1170)
at net.liftweb.http.S$.init(S.scala:822)
at net.liftweb.http.LiftServlet.doService(LiftServlet.scala:176)
at net.liftweb.http.LiftServlet$$anonfun$doIt$1$1.apply(LiftServlet.scala:75)
at net.liftweb.http.LiftServlet$$anonfun$doIt$1$1.apply(LiftServlet.scala:75)
at net.liftweb.util.TimeHelpers$class.calcTime(TimeHelpers.scala:247)
at net.liftweb.util.Helpers$.calcTime(Helpers.scala:34)
at net.liftweb.util.TimeHelpers$class.logTime(TimeHelpers.scala:256)
at net.liftweb.util.Helpers$.logTime(Helpers.scala:34)
at net.liftweb.http.LiftServlet.doIt$1(LiftServlet.scala:74)
at net.liftweb.http.LiftServlet.service(LiftServlet.scala:82)
at net.liftweb.http.provider.HTTPProvider$$anonfun$service$3.apply(HTTPProvider.scala:58)
at net.liftweb.http.provider.HTTPProvider$$anonfun$service$3.apply(HTTPProvider.scala:58)
at net.liftweb.util.ThreadGlobal.doWith(ThreadGlobal.scala:71)
at net.liftweb.http.URLRewriter$.doWith(Req.scala:537)
at net.liftweb.http.provider.HTTPProvider$class.service(HTTPProvider.scala:57)
at net.liftweb.http.LiftFilter.service(LiftServlet.scala:537)
at net.liftweb.http.provider.servlet.ServletFilterProvider$class.protected$service(ServletFilterProvider.scala:62)
at net.liftweb.http.LiftFilter.protected$service(LiftServlet.scala:537)
at net.liftweb.http.provider.servlet.ServletFilterProvider$$anonfun$doFilter$1$$anonfun$apply$1.apply(ServletFilterProvider.scala:62)
at net.liftweb.http.provider.servlet.ServletFilterProvider$$anonfun$doFilter$1$$anonfun$apply$1.apply(ServletFilterProvider.scala:57)
at net.liftweb.http.CoreRequestVarHandler$$anonfun$apply$9$$anonfun$apply$10$$anonfun$apply$11$$anonfun$apply$12.apply(Vars.scala:333)
at net.liftweb.util.ThreadGlobal.doWith(ThreadGlobal.scala:71)
at net.liftweb.http.CoreRequestVarHandler$$anonfun$apply$9$$anonfun$apply$10$$anonfun$apply$11.apply(Vars.scala:332)
at net.liftweb.util.ThreadGlobal.doWith(ThreadGlobal.scala:71)
at net.liftweb.http.CoreRequestVarHandler$$anonfun$apply$9$$anonfun$apply$10.apply(Vars.scala:331)
at net.liftweb.util.ThreadGlobal.doWith(ThreadGlobal.scala:71)
at net.liftweb.http.CoreRequestVarHandler$$anonfun$apply$9.apply(Vars.scala:330)
at net.liftweb.util.ThreadGlobal.doWith(ThreadGlobal.scala:71)
at net.liftweb.http.CoreRequestVarHandler$class.apply(Vars.scala:329)
at net.liftweb.http.RequestVarHandler$.apply(Vars.scala:250)
at net.liftweb.http.provider.servlet.ServletFilterProvider$$anonfun$doFilter$1.apply(ServletFilterProvider.scala:56)
at net.liftweb.http.provider.servlet.ServletFilterProvider$$anonfun$doFilter$1.apply(ServletFilterProvider.scala:56)
at net.liftweb.http.CoreRequestVarHandler$$anonfun$apply$9$$anonfun$apply$10$$anonfun$apply$11$$anonfun$apply$12.apply(Vars.scala:333)
at net.liftweb.util.ThreadGlobal.doWith(ThreadGlobal.scala:71)
at net.liftweb.http.CoreRequestVarHandler$$anonfun$apply$9$$anonfun$apply$10$$anonfun$apply$11.apply(Vars.scala:332)
at net.liftweb.util.ThreadGlobal.doWith(ThreadGlobal.scala:71)
at net.liftweb.http.CoreRequestVarHandler$$anonfun$apply$9$$anonfun$apply$10.apply(Vars.scala:331)
at net.liftweb.util.ThreadGlobal.doWith(ThreadGlobal.scala:71)
at net.liftweb.http.CoreRequestVarHandler$$anonfun$apply$9.apply(Vars.scala:330)
at net.liftweb.util.ThreadGlobal.doWith(ThreadGlobal.scala:71)
at net.liftweb.http.CoreRequestVarHandler$class.apply(Vars.scala:329)
at net.liftweb.http.TransientRequestVarHandler$.apply(Vars.scala:254)
at net.liftweb.http.provider.servlet.ServletFilterProvider$class.doFilter(ServletFilterProvider.scala:55)
at net.liftweb.http.LiftFilter.doFilter(LiftServlet.scala:537)
at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1148)
at org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:387)
at org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:216)
at org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:181)
at org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:765)
at org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:417)
at org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:230)
at org.mortbay.jetty.handler.HandlerCollection.handle(HandlerCollection.java:114)
at org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
at org.mortbay.jetty.Server.handle(Server.java:326)
at org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:534)
at org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConnection.java:864)
at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:539)
at org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:212)
at org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:404)
at org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:409)
at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:520)

And one final thread (in the same process as the above process locked) that appears to have the InMemoryCache lock. It's RUNNABLE to it's not deadlock but the process was consuming 100% CPU. Some sort of infinite loop maybe? Will try to get more data when I see this again.

"1539840322@qtp-868385441-181" prio=10 tid=0x00007fdc91257000 nid=0x3d10 runnable [0x00000000491ca000..0x00000000491cea80]
java.lang.Thread.State: RUNNABLE
at scala.Iterable$class.foreach(Iterable.scala:256)
at scala.runtime.BoxedArray.foreach(BoxedArray.scala:24)
at net.liftweb.common.LRUMap.update(LRU.scala:103)
at net.liftweb.util.InMemoryCache$$anonfun$set$1.apply(TemplateCache.scala:83)
at net.liftweb.util.InMemoryCache$$anonfun$set$1.apply(TemplateCache.scala:82)
at net.liftweb.util.ConcurrentLock.write(ConcurrentLock.scala:36)
at net.liftweb.util.InMemoryCache.set(TemplateCache.scala:82)
at net.liftweb.util.InMemoryCache.set(TemplateCache.scala:70)
at net.liftweb.util.TemplateCache$class.update(TemplateCache.scala:39)
at net.liftweb.util.InMemoryCache.update(TemplateCache.scala:70)
at net.liftweb.http.TemplateFinder$.findAnyTemplate(LiftSession.scala:1340)
at net.liftweb.http.LiftSession.net$liftweb$http$LiftSession$$findVisibleTemplate(LiftSession.scala:661)
at net.liftweb.http.LiftSession$$anonfun$processTemplate$1$1.apply(LiftSession.scala:473)
at net.liftweb.http.LiftSession$$anonfun$processTemplate$1$1.apply(LiftSession.scala:473)
at net.liftweb.common.EmptyBox.or(Box.scala:378)

Found the problem. The TemplateCache allows for concurrent reads, so the LRU ordering was being updated by multiple threads. I changed the model to synchronized (rather than concurrent read lock)

(In revision:e473346d9e2a2dd58c7a4917bcef59b45906cdda) Closes #367. The Template Cache allowed for multiple reads, but the reads caused an update to the LRU so the reads and writes have to be synchronized

Branch: master