外表join about mdrill HOT 9 CLOSED

2013-08-19 15:30:48 MdrillGroupBy [INFO] TopMaps groups.size=11778@11778 to 10010@10010,res.size=10010,remove=1768,timetaken=15,mergercount=1
2013-08-19 15:30:48 SolrCore [ERROR] Exception during facet counts:java.lang.ArrayIndexOutOfBoundsException

2013-08-19 15:30:48 SolrCore [ERROR] org.apache.solr.common.SolrException: java.lang.ArrayIndexOutOfBoundsException
at org.apache.solr.request.SimpleFacets.getFacetCounts(SimpleFacets.java:213)
at org.apache.solr.request.mdrill.FacetComponent.process(FacetComponent.java:79)
at org.apache.solr.handler.component.SearchHandler.handleRequestBody(SearchHandler.java:102)
at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:129)
at org.apache.solr.core.SolrCore.execute(SolrCore.java:1506)
at org.apache.solr.servlet.SolrDispatchFilter.execute(SolrDispatchFilter.java:264)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:198)
at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212)
at org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:399)
at org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182)
at org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)
at org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
at org.mortbay.jetty.Server.handle(Server.java:326)
at org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542)
at org.mortbay.jetty.HttpConnection$RequestHandler.content(HttpConnection.java:945)
at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:756)
at org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:218)
at org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:404)
at org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:410)
at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582)
Caused by: java.lang.ArrayIndexOutOfBoundsException

from mdrill.

2013-08-19 09:30:29 SolrCore [INFO] ####UnInverted#### SegmentReader begin
2013-08-19 09:30:29 SolrCore [ERROR] Exception during facet counts:java.lang.NullPointerException
at org.apache.lucene.index.SegmentReader.getpos(SegmentReader.java:493)
at org.apache.solr.request.uninverted.UnInvertedField.uninvert(UnInvertedField.java:520)
at org.apache.solr.request.uninverted.UnInvertedField.(UnInvertedField.java:131)
at org.apache.solr.request.uninverted.UnInvertedField.getUnInvertedField(UnInvertedField.java:761)
at org.apache.solr.request.mdrill.MdrillPorcessUtils$UnvertFields.(MdrillPorcessUtils.java:324)
at org.apache.solr.request.mdrill.MdrillGroupBy.execute(MdrillGroupBy.java:288)
at org.apache.solr.request.mdrill.MdrillGroupBy.getCross(MdrillGroupBy.java:168)
at org.apache.lucene.index.SegmentReader.invertScan(SegmentReader.java:517)
at org.apache.lucene.index.DirectoryReader.invertScan(DirectoryReader.java:591)
at org.apache.lucene.index.FilterIndexReader.invertScan(FilterIndexReader.java:317)
at org.apache.solr.request.mdrill.MdrillGroupBy.getBySchemaReader(MdrillGroupBy.java:134)
at org.apache.solr.request.SimpleFacets.getFacetFieldCounts(SimpleFacets.java:318)
at org.apache.solr.request.SimpleFacets.getFacetCounts(SimpleFacets.java:201)
at org.apache.solr.request.mdrill.FacetComponent.process(FacetComponent.java:79)
at org.apache.solr.handler.component.SearchHandler.handleRequestBody(SearchHandler.java:102)
at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:129)
at org.apache.solr.core.SolrCore.execute(SolrCore.java:1506)
at org.apache.solr.servlet.SolrDispatchFilter.execute(SolrDispatchFilter.java:264)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:198)
at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212)
at org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:399)
at org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182)
at org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)
at org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
at org.mortbay.jetty.Server.handle(Server.java:326)
at org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542)
at org.mortbay.jetty.HttpConnection$RequestHandler.content(HttpConnection.java:945)
at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:756)
at org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:218)
at org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:404)
at org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:410)
at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582)

2013-08-19 09:30:29 SolrCore [ERROR] org.apache.solr.common.SolrException: java.lang.NullPointerException
at org.apache.solr.request.SimpleFacets.getFacetCounts(SimpleFacets.java:213)

from mdrill.

2013-08-19 15:27:38 MdrillGroupBy [INFO] ##baseDocs.size## 116546@116546
2013-08-19 15:27:38 MdrillGroupBy [INFO] TopMaps groups.size=102400@102400 to 10010@10010,res.size=10010,remove=92390,timetaken=195,mergercount=1
2013-08-19 15:27:38 MdrillGroupBy [INFO] TopMaps groups.size=10876@10876 to 10010@10010,res.size=10010,remove=866,timetaken=11,mergercount=1
2013-08-19 15:27:38 SolrCore [ERROR] Exception during facet counts:java.lang.ArrayIndexOutOfBoundsException: 33554431
at org.apache.solr.request.uninverted.NumberedTermEnum.skipTo(NumberedTermEnum.java:143)
at org.apache.solr.request.uninverted.UnInvertedField.getTermText(UnInvertedField.java:662)
at org.apache.solr.request.uninverted.UnInvertedFieldTermNumRead$TermNumReadSingleNotNull.tNumToString(UnInvertedFieldTermNumRead.java:78)
at org.apache.solr.request.uninverted.UnInvertedFieldTermNumRead$TermNumReadSingle.tNumToString(UnInvertedFieldTermNumRead.java:193)
at org.apache.solr.request.uninverted.UnInvertedField.tNumToString(UnInvertedField.java:632)
at org.apache.solr.request.mdrill.MdrillPorcessUtils$TermNumToString.fetchValues(MdrillPorcessUtils.java:96)
at org.apache.solr.request.mdrill.MdrillGroupBy.prefetch(MdrillGroupBy.java:445)
at org.apache.solr.request.mdrill.MdrillGroupBy.transGroupValue(MdrillGroupBy.java:459)
at org.apache.solr.request.mdrill.MdrillGroupBy.execute(MdrillGroupBy.java:334)
at org.apache.solr.request.mdrill.MdrillGroupBy.getCross(MdrillGroupBy.java:168)
at org.apache.lucene.index.SegmentReader.invertScan(SegmentReader.java:517)
at org.apache.lucene.index.DirectoryReader.invertScan(DirectoryReader.java:591)
at org.apache.lucene.index.FilterIndexReader.invertScan(FilterIndexReader.java:317)
at org.apache.solr.request.mdrill.MdrillGroupBy.getBySchemaReader(MdrillGroupBy.java:134)
at org.apache.solr.request.SimpleFacets.getFacetFieldCounts(SimpleFacets.java:318)
at org.apache.solr.request.SimpleFacets.getFacetCounts(SimpleFacets.java:201)
at org.apache.solr.request.mdrill.FacetComponent.process(FacetComponent.java:79)
at org.apache.solr.handler.component.SearchHandler.handleRequestBody(SearchHandler.java:102)
at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:129)
at org.apache.solr.core.SolrCore.execute(SolrCore.java:1506)
at org.apache.solr.servlet.SolrDispatchFilter.execute(SolrDispatchFilter.java:264)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:198)
at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212)
at org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:399)
at org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182)
at org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)
at org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
at org.mortbay.jetty.Server.handle(Server.java:326)
at org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542)
at org.mortbay.jetty.HttpConnection$RequestHandler.content(HttpConnection.java:945)
at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:756)
at org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:218)
at org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:404)
at org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:410)
at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582)

2013-08-19 15:27:38 SolrCore [ERROR] org.apache.solr.common.SolrException: java.lang.ArrayIndexOutOfBoundsException: 33554431
at org.apache.solr.request.SimpleFacets.getFacetCounts(SimpleFacets.java:213)
at org.apache.solr.request.mdrill.FacetComponent.process(FacetComponent.java:79)
at org.apache.solr.handler.component.SearchHandler.handleRequestBody(SearchHandler.java:102)
at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:129)

from mdrill.

java.io.IOException: Resource deadlock avoided
at sun.nio.ch.FileChannelImpl.lock0(Native Method)
at sun.nio.ch.FileChannelImpl.lock(FileChannelImpl.java:845)
at java.nio.channels.FileChannel.lock(FileChannel.java:860)
at org.apache.solr.request.join.HigoJoinUtils.maybeclear(HigoJoinUtils.java:196)
at org.apache.solr.request.join.HigoJoinUtils.pathToLocal(HigoJoinUtils.java:300)
at org.apache.solr.request.join.HigoJoinUtils.getSearch(HigoJoinUtils.java:85)
at org.apache.solr.request.join.HigoJoinInvert.open(HigoJoinInvert.java:61)
at org.apache.solr.request.mdrill.MdrillGroupBy.execute(MdrillGroupBy.java:299)
at org.apache.solr.request.mdrill.MdrillGroupBy.getCross(MdrillGroupBy.java:164)
at org.apache.lucene.index.SegmentReader.invertScan(SegmentReader.java:517)
at org.apache.lucene.index.DirectoryReader.invertScan(DirectoryReader.java:591)
at org.apache.lucene.index.FilterIndexReader.invertScan(FilterIndexReader.java:317)
at org.apache.solr.request.mdrill.MdrillGroupBy.getBySchemaReader(MdrillGroupBy.java:130)
at org.apache.solr.request.SimpleFacets.getFacetFieldCounts(SimpleFacets.java:315)
at org.apache.solr.request.SimpleFacets.getFacetCounts(SimpleFacets.java:204)
at org.apache.solr.request.mdrill.FacetComponent.process(FacetComponent.java:79)
at org.apache.solr.handler.component.SearchHandler.handleRequestBody(SearchHandler.java:102)
at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:129)
at org.apache.solr.core.SolrCore.execute(SolrCore.java:1506)
at org.apache.solr.servlet.SolrDispatchFilter.execute(SolrDispatchFilter.java:264)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:198)
at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212)
at org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:399)
at org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182)
at org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)
at org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
at org.mortbay.jetty.Server.handle(Server.java:326)
at org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542)
at org.mortbay.jetty.HttpConnection$RequestHandler.content(HttpConnection.java:945)
at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:756)
at org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:218)
at org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:404)
at org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:410)
at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582)
2013-08-19 20:43:14 SolrCore [ERROR] org.apache.solr.common.SolrException: java.io.IOException: Resource deadlock avoided
at org.apache.solr.request.SimpleFacets.getFacetCounts(SimpleFacets.java:210)
at org.apache.solr.request.mdrill.FacetComponent.process(FacetComponent.java:79)
at org.apache.solr.handler.component.SearchHandler.handleRequestBody(SearchHandler.java:102)
at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:129)
at org.apache.solr.core.SolrCore.execute(SolrCore.java:1506)
at org.apache.solr.servlet.SolrDispatchFilter.execute(SolrDispatchFilter.java:264)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:198)
at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212)
at org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:399)
at org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182)
at org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)
at org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
at org.mortbay.jetty.Server.handle(Server.java:326)
at org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542)
at org.mortbay.jetty.HttpConnection$RequestHandler.content(HttpConnection.java:945)
at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:756)
at org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:218)
at org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:404)
at org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:410)
at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582)
Caused by: java.io.IOException: Resource deadlock avoided
at sun.nio.ch.FileChannelImpl.lock0(Native Method)
at sun.nio.ch.FileChannelImpl.lock(FileChannelImpl.java:845)
at java.nio.channels.FileChannel.lock(FileChannel.java:860)
at org.apache.solr.request.join.HigoJoinUtils.maybeclear(HigoJoinUtils.java:196)
at org.apache.solr.request.join.HigoJoinUtils.pathToLocal(HigoJoinUtils.java:300)
at org.apache.solr.request.join.HigoJoinUtils.getSearch(HigoJoinUtils.java:85)
at org.apache.solr.request.join.HigoJoinInvert.open(HigoJoinInvert.java:61)
at org.apache.solr.request.mdrill.MdrillGroupBy.execute(MdrillGroupBy.java:299)

from mdrill.

    at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582)

Caused by: java.io.IOException: Resource deadlock avoided
at sun.nio.ch.FileChannelImpl.lock0(Native Method)
at sun.nio.ch.FileChannelImpl.lock(FileChannelImpl.java:845)
at java.nio.channels.FileChannel.lock(FileChannel.java:860)
at org.apache.solr.request.join.HigoJoinUtils.maybeclear(HigoJoinUtils.java:196)
at org.apache.solr.request.join.HigoJoinUtils.pathToLocal(HigoJoinUtils.java:300)
at org.apache.solr.request.join.HigoJoinUtils.getSearch(HigoJoinUtils.java:85)
at org.apache.solr.request.join.HigoJoinInvert.open(HigoJoinInvert.java:61)
at org.apache.solr.request.mdrill.MdrillGroupBy.execute(MdrillGroupBy.java:299)
at org.apache.solr.request.mdrill.MdrillGroupBy.getCross(MdrillGroupBy.java:164)
at org.apache.lucene.index.SegmentReader.invertScan(SegmentReader.java:517)
at org.apache.lucene.index.DirectoryReader.invertScan(DirectoryReader.java:591)
at org.apache.lucene.index.FilterIndexReader.invertScan(FilterIndexReader.java:317)
at org.apache.solr.request.mdrill.MdrillGroupBy.getBySchemaReader(MdrillGroupBy.java:130)
at org.apache.solr.request.SimpleFacets.getFacetFieldCounts(SimpleFacets.java:315)
at org.apache.solr.request.SimpleFacets.getFacetCounts(SimpleFacets.java:204)
... 19 more

2013-08-19 20:43:14 SolrCore [INFO] [rpt_p4padhoc_cust] webapp=/solr path=/select params={facet.cross=true&facet.cross.join=@&join.leftkey.15009154-4c14-4f4d-b30e-f4923d3022dd=user_id&facet=true&hadoop.merger.hashindex=33&facet.cross.offset=0&facet.cross.sort.fl=higoempty_sort_s&indexpartion=2013073&join.fl.bac81f87-7f4f-4b59-bc99-4c0ec4c735d6=cols_1_s&join.fl.bac81f87-7f4f-4b59-bc99-4c0ec4c735d6=cols_2_s&join.fl.bac81f87-7f4f-4b59-bc99-4c0ec4c735d6=cols_0_s&version=2&facet.cross.sort.desc=true&fl=higo_uuid,score&join.rightkey.15009154-4c14-4f4d-b3

from mdrill.

2013-08-20 00:01:20 MdrillGroupBy [ERROR] makePreGroup
java.lang.NumberFormatException: For input string: "null"
at java.lang.NumberFormatException.forInputString(NumberFormatException.java:48)
at java.lang.Long.parseLong(Long.java:403)
at java.lang.Long.parseLong(Long.java:461)
at org.apache.solr.schema.TrieField.readableToIndexed(TrieField.java:297)
at org.apache.solr.schema.TrieField.toInternal(TrieField.java:309)
at org.apache.solr.request.uninverted.UnInvertedField.getTermNum(UnInvertedField.java:673)
at org.apache.solr.request.mdrill.MdrillGroupBy.makePreGroup(MdrillGroupBy.java:388)
at org.apache.solr.request.mdrill.MdrillGroupBy.execute(MdrillGroupBy.java:317)
at org.apache.solr.request.mdrill.MdrillGroupBy.getCross(MdrillGroupBy.java:164)
at org.apache.lucene.index.SegmentReader.invertScan(SegmentReader.java:517)
at org.apache.lucene.index.DirectoryReader.invertScan(DirectoryReader.java:591)
at org.apache.lucene.index.FilterIndexReader.invertScan(FilterIndexReader.java:317)
at org.apache.solr.request.mdrill.MdrillGroupBy.getBySchemaReader(MdrillGroupBy.java:130)
at org.apache.solr.request.SimpleFacets.getFacetFieldCounts(SimpleFacets.java:315)
at org.apache.solr.request.SimpleFacets.getFacetCounts(SimpleFacets.java:204)
at org.apache.solr.request.mdrill.FacetComponent.process(FacetComponent.java:79)
at org.apache.solr.handler.component.SearchHandler.handleRequestBody(SearchHandler.java:102)
at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:129)
at org.apache.solr.core.SolrCore.execute(SolrCore.java:1506)
at org.apache.solr.servlet.SolrDispatchFilter.execute(SolrDispatchFilter.java:264)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:198)
at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212)
at org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:399)
at org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182)
at org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)
at org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
at org.mortbay.jetty.Server.handle(Server.java:326)
at org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542)
at org.mortbay.jetty.HttpConnection$RequestHandler.content(HttpConnection.java:945)

from mdrill.

    at java.lang.Thread.run(Thread.java:619)

2013-08-20 15:26:36 ConnectionState [ERROR] Connection timed out
org.apache.zookeeper.KeeperException$ConnectionLossException: KeeperErrorCode = ConnectionLoss
at com.netflix.curator.ConnectionState.getZooKeeper(ConnectionState.java:72)
at com.netflix.curator.CuratorZookeeperClient.getZooKeeper(CuratorZookeeperClient.java:74)
at com.netflix.curator.framework.imps.CuratorFrameworkImpl.getZooKeeper(CuratorFrameworkImpl.java:381)
at com.netflix.curator.framework.imps.ExistsBuilderImpl$2.call(ExistsBuilderImpl.java:145)
at com.netflix.curator.framework.imps.ExistsBuilderImpl$2.call(ExistsBuilderImpl.java:138)
at com.netflix.curator.RetryLoop.callWithRetry(RetryLoop.java:85)
at com.netflix.curator.framework.imps.ExistsBuilderImpl.pathInForeground(ExistsBuilderImpl.java:134)
at com.netflix.curator.framework.imps.ExistsBuilderImpl.forPath(ExistsBuilderImpl.java:125)
at com.netflix.curator.framework.imps.ExistsBuilderImpl.forPath(ExistsBuilderImpl.java:34)
at com.alipay.bluewhale.core.zk.Zookeeper.existsNode(Zookeeper.java:105)
at com.alipay.bluewhale.core.zk.Zookeeper.getData(Zookeeper.java:148)
at com.alipay.bluewhale.core.cluster.DistributedClusterState.get_data(DistributedClusterState.java:116)
at com.alipay.bluewhale.core.cluster.StormZkClusterState.assignment_info(StormZkClusterState.java:135)
at com.alipay.bluewhale.core.work.refresh.RefreshConnections.run(RefreshConnections.java:67)
at com.alipay.bluewhale.core.utils.AsyncLoopRunnable.run(AsyncLoopRunnable.java:54)
at java.lang.Thread.run(Thread.java:619)
2013-08-20 15:28:13 AsyncLoopThread [ERROR] uncaughtException
java.lang.OutOfMemoryError: GC overhead limit exceeded
at java.lang.Class.getDeclaredMethods0(Native Method)
at java.lang.Class.privateGetDeclaredMethods(Class.java:2427)
at java.lang.Class.getDeclaredMethod(Class.java:1935)
at java.io.ObjectStreamClass.getPrivateMethod(ObjectStreamClass.java:1382)
at java.io.ObjectStreamClass.access$1700(ObjectStreamClass.java:52)
at java.io.ObjectStreamClass$2.run(ObjectStreamClass.java:438)
at java.security.AccessController.doPrivileged(Native Method)
at java.io.ObjectStreamClass.(ObjectStreamClass.java:413)
at java.io.ObjectStreamClass.lookup(ObjectStreamClass.java:310)
at java.io.ObjectStreamClass.initNonProxy(ObjectStreamClass.java:547)
at java.io.ObjectInputStream.readNonProxyDesc(ObjectInputStream.java:1583)
at java.io.ObjectInputStream.readClassDesc(ObjectInputStream.java:1496)
at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1732)
at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1329)
at java.io.ObjectInputStream.readObject(ObjectInputStream.java:351)
at backtype.storm.utils.Utils.deserialize(Utils.java:56)
at backtype.storm.utils.LocalState.snapshot(LocalState.java:24)
at backtype.storm.utils.LocalState.put(LocalState.java:32)
at com.alipay.bluewhale.core.work.WorkCommon.doHeartbeat(WorkCommon.java:91)
at com.alipay.bluewhale.core.work.refresh.WorkerHeartbeatRunable.run(WorkerHeartbeatRunable.java:52)
at com.alipay.bluewhale.core.utils.AsyncLoopRunnable.run(AsyncLoopRunnable.java:54)
at java.lang.Thread.run(Thread.java:619)
@

from mdrill.

2013-08-20 13:40:33 SolrCore [ERROR] org.apache.solr.common.SolrException: java.lang.NullPointerException
at org.apache.solr.request.SimpleFacets.getFacetCounts(SimpleFacets.java:210)
at org.apache.solr.request.mdrill.FacetComponent.process(FacetComponent.java:79)
at org.apache.solr.handler.component.SearchHandler.handleRequestBody(SearchHandler.java:102)
at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:129)
at org.apache.solr.core.SolrCore.execute(SolrCore.java:1506)
at org.apache.solr.servlet.SolrDispatchFilter.execute(SolrDispatchFilter.java:264)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:198)
at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212)
at org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:399)
at org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182)
at org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)
at org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
at org.mortbay.jetty.Server.handle(Server.java:326)
at org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542)
at org.mortbay.jetty.HttpConnection$RequestHandler.content(HttpConnection.java:945)
at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:756)
at org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:218)
at org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:404)
at org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:410)
at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582)
Caused by: java.lang.NullPointerException
at org.apache.lucene.index.SegmentReader.getpos(SegmentReader.java:493)
at org.apache.solr.request.uninverted.UnInvertedField.uninvert(UnInvertedField.java:546)
at org.apache.solr.request.uninverted.UnInvertedField.(UnInvertedField.java:153)
at org.apache.solr.request.uninverted.UnInvertedField.getUnInvertedField(UnInvertedField.java:795)
at org.apache.solr.request.mdrill.MdrillPorcessUtils$UnvertFields.(MdrillPorcessUtils.java:324)
at org.apache.solr.request.mdrill.MdrillGroupBy.execute(MdrillGroupBy.java:284)
at org.apache.solr.request.mdrill.MdrillGroupBy.getCross(MdrillGroupBy.java:164)
at org.apache.lucene.index.SegmentReader.invertScan(SegmentReader.java:517)
at org.apache.lucene.index.DirectoryReader.invertScan(DirectoryReader.java:591)
at org.apache.lucene.index.FilterIndexReader.invertScan(FilterIndexReader.java:317)
at org.apache.solr.request.mdrill.MdrillGroupBy.getBySchemaReader(MdrillGroupBy.java:130)
at org.apache.solr.request.SimpleFacets.getFacetFieldCounts(SimpleFacets.java:315)
at org.apache.solr.request.SimpleFacets.getFacetCounts(SimpleFacets.java:204)
... 19 more

from mdrill.

013-08-22 15:43:41 HigoJoinInvert [INFO] ##joinright##7
2013-08-22 15:43:41 UnInvertedField [INFO] unlock /disk1/taobao/bluewhile/higojoin_work/b9fe9b45-0067-4851-abc1-73abcfa9ba65/store/cacheField/mdrill_uni_lock
2013-08-22 15:43:41 SimpleFacets [ERROR] getFacetCounts
java.io.IOException: Resource deadlock avoided
at sun.nio.ch.FileChannelImpl.lock0(Native Method)
at sun.nio.ch.FileChannelImpl.lock(FileChannelImpl.java:845)
at java.nio.channels.FileChannel.lock(FileChannel.java:860)
at org.apache.solr.request.uninverted.UnInvertedField.(UnInvertedField.java:97)
at org.apache.solr.request.uninverted.UnInvertedField.getUnInvertedField(UnInvertedField.java:770)
at org.apache.solr.request.uninverted.UnInvertedField.getUnInvertedField(UnInvertedField.java:751)
at org.apache.solr.request.mdrill.MdrillPorcessUtils$UnvertFields.(MdrillPorcessUtils.java:294)
at org.apache.solr.request.join.HigoJoinInvert.open(HigoJoinInvert.java:68)
at org.apache.solr.request.mdrill.MdrillDetail.execute(MdrillDetail.java:198)
at org.apache.solr.request.mdrill.MdrillDetail.getDetail(MdrillDetail.java:146)
at org.apache.lucene.index.SegmentReader.invertScan(SegmentReader.java:538)
at org.apache.lucene.index.DirectoryReader.invertScan(DirectoryReader.java:591)
at org.apache.lucene.index.FilterIndexReader.invertScan(FilterIndexReader.java:317)
at org.apache.solr.request.mdrill.MdrillDetail.getBySchemaReader(MdrillDetail.java:112)
at org.apache.solr.request.SimpleFacets.getFacetFieldCounts(SimpleFacets.java:311)
at org.apache.solr.request.SimpleFacets.getFacetCounts(SimpleFacets.java:204)
at org.apache.solr.request.mdrill.FacetComponent.process(FacetComponent.java:79)
at org.apache.solr.handler.component.SearchHandler.handleRequestBody(SearchHandler.java:102)
at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:129)
at org.apache.solr.core.SolrCore.execute(SolrCore.java:1506)
at org.apache.solr.servlet.SolrDispatchFilter.execute(SolrDispatchFilter.java:264)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:198)
at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212)
at org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:399)
at org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182)
at org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)
at org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
at org.mortbay.jetty.Server.handle(Server.java:326)
at org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542)
at org.mortbay.jetty.HttpConnection$RequestHandler.content(HttpConnection.java:945)
at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:756)
at org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:218)
at org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:404)
at org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:410)
at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582)
2013-08-22 15:43:41 SolrCore [ERROR] org.apache.solr.common.SolrException: java.io.IOException: Resource deadlock avoided
at org.apache.solr.request.SimpleFacets.getFacetCounts(SimpleFacets.java:210)
at org.apache.solr.request.mdrill.FacetComponent.process(FacetComponent.java:79)
at org.apache.solr.handler.component.SearchHandler.handleRequestBody(SearchHandler.java:102)
at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:129)
at org.apache.solr.core.SolrCore.execute(SolrCore.java:1506)
at org.apache.solr.servlet.SolrDispatchFilter.execute(SolrDispatchFilter.java:264)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:198)
at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212)
at org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:399)
at org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182)
at org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)
at org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
at org.mortbay.jetty.Server.handle(Server.java:326)
at org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542)
at org.mortbay.jetty.HttpConnection$RequestHandler.content(HttpConnection.java:945)
at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:756)
at org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:218)
at org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:404)
at org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:410)
at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582)
Caused by: java.io.IOException: Resource deadlock avoided
at sun.nio.ch.FileChannelImpl.lock0(Native Method)
at sun.nio.ch.FileChannelImpl.lock(FileChannelImpl.java:845)
at java.nio.channels.FileChannel.lock(FileChannel.java:860)
at org.apache.solr.request.uninverted.UnInvertedField.(UnInvertedField.java:97)
at org.apache.solr.request.uninverted.UnInvertedField.getUnInvertedField(UnInvertedField.java:770)
at org.apache.solr.request.uninverted.UnInvertedField.getUnInvertedField(UnInvertedField.java:751)
at org.apache.solr.request.mdrill.MdrillPorcessUtils$UnvertFields.(MdrillPorcessUtils.java:294)
at org.apache.solr.request.join.HigoJoinInvert.open(HigoJoinInvert.java:68)
at org.apache.solr.request.mdrill.MdrillDetail.execute(MdrillDetail.java:198)
at org.apache.solr.request.mdrill.MdrillDetail.getDetail(MdrillDetail.java:146)
at org.apache.lucene.index.SegmentReader.invertScan(SegmentReader.java:538)
at org.apache.lucene.index.DirectoryReader.invertScan(DirectoryReader.java:591)
at org.apache.lucene.index.FilterIndexReader.invertScan(FilterIndexReader.java:317)
at org.apache.solr.request.mdrill.MdrillDetail.getBySchemaReader(MdrillDetail.java:112)
at org.apache.solr.request.SimpleFacets.getFacetFieldCounts(SimpleFacets.java:311)
at org.apache.solr.request.SimpleFacets.getFacetCounts(SimpleFacets.java:204)
... 19 more

2013-08-22 15:43:41 Sol

from mdrill.