报错worker-6710.log:java.nio.channels.OverlappingFileLockException about mdrill HOT 6 OPEN

2013-09-29 10:07:30 TryLockFile [ERROR] trylock
java.nio.channels.OverlappingFileLockException
at sun.nio.ch.FileChannelImpl$SharedFileLockTable.checkList(FileChannelImpl.java:1173)
at sun.nio.ch.FileChannelImpl$SharedFileLockTable.add(FileChannelImpl.java:1075)
at sun.nio.ch.FileChannelImpl.lock(FileChannelImpl.java:837)
at java.nio.channels.FileChannel.lock(FileChannel.java:860)
at com.alimama.mdrill.utils.TryLockFile.trylock(TryLockFile.java:33)
at org.apache.solr.request.join.HigoJoinUtils.pathToLocal(HigoJoinUtils.java:300)
at org.apache.solr.request.join.HigoJoinUtils.getSearch(HigoJoinUtils.java:82)
at org.apache.solr.request.join.HigoJoinInvert.open(HigoJoinInvert.java:60)
at org.apache.solr.request.mdrill.MdrillParseGroupby$fetchContaioner.(MdrillParseGroupby.java:131)
at org.apache.solr.request.mdrill.MdrillParseGroupby.createContainer(MdrillParseGroupby.java:84)
at org.apache.solr.request.mdrill.MdrillGroupBy.get(MdrillGroupBy.java:65)
at org.apache.lucene.index.SegmentReader.invertScan(SegmentReader.java:559)
at org.apache.lucene.index.DirectoryReader.invertScan(DirectoryReader.java:591)
at org.apache.lucene.index.FilterIndexReader.invertScan(FilterIndexReader.java:317)
at org.apache.solr.request.mdrill.FacetComponent.getResult(FacetComponent.java:106)
at org.apache.solr.request.mdrill.FacetComponent.process(FacetComponent.java:81)
at org.apache.solr.handler.component.SearchHandler.handleRequestBody(SearchHandler.java:101)
at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:129)
at org.apache.solr.core.SolrCore.execute(SolrCore.java:1510)
at org.apache.solr.servlet.SolrDispatchFilter.execute(SolrDispatchFilter.java:264)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:198)
at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212)
at org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:399)
at org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182)
at org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)
at org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
at org.mortbay.jetty.Server.handle(Server.java:326)
at org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542)
at org.mortbay.jetty.HttpConnection$RequestHandler.content(HttpConnection.java:945)
at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:756)
at org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:218)
at org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:404)
at org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:410)
at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582)
2013-09-29 10:07:30 IndexUtils [INFO] higolog copyToLocal begin /group/tbdp-etao-adhoc/p4padhoc/download/offline/20130922/4a15221e-1d40-4816-aba3-c7641b99b5d2/index/part-00000,/disk4/taobao/mdrill/higojoin_work/939c6239-6acd-4643-9e1d-bfbd3136956b/store,/disk4/taobao/mdrill/higojoin_tmp/939c6239-6acd-4643-9e1d-bfbd3136956b/47
2013-09-29 10:07:30 IndexUtils [INFO] truncate /disk4/taobao/mdrill/higoj

from mdrill.

40-4816-aba3-c7641b99b5d2/index/part-00000,/disk4/taobao/mdrill/higojoin_work/939c6239-6acd-4643-9e1d-bfbd3136956b/store,/disk4/taobao/mdrill/higojoin_tmp/939c6239-6acd-4643-9e1d-bfbd3136956b/47
org.apache.hadoop.util.Shell$ExitCodeException: chmod: cannot access `/disk4/taobao/mdrill/higojoin_tmp/939c6239-6acd-4643-9e1d-bfbd3136956b/47/_m.tis': Not a directory

    at org.apache.hadoop.util.Shell.runCommand(Shell.java:233)
    at org.apache.hadoop.util.Shell.run(Shell.java:172)
    at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:324)
    at org.apache.hadoop.util.Shell.execCommand(Shell.java:376)
    at org.apache.hadoop.fs.RawLocalFileSystem.execCommand(RawLocalFileSystem.java:552)
    at org.apache.hadoop.fs.RawLocalFileSystem.setPermission(RawLocalFileSystem.java:544)
    at org.apache.hadoop.fs.FilterFileSystem.setPermission(FilterFileSystem.java:297)
    at org.apache.hadoop.fs.ChecksumFileSystem.create(ChecksumFileSystem.java:374)
    at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:517)
    at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:498)
    at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:405)
    at com.alimama.mdrill.utils.ZipUtils.unZip(ZipUtils.java:152)
    at com.alimama.mdrill.utils.IndexUtils.copyToLocal(IndexUtils.java:137)
    at org.apache.solr.request.join.HigoJoinUtils.pathToLocal(HigoJoinUtils.java:312)
    at org.apache.solr.request.join.HigoJoinUtils.getSearch(HigoJoinUtils.java:82)
    at org.apache.solr.request.join.HigoJoinInvert.open(HigoJoinInvert.java:60)
    at org.apache.solr.request.mdrill.MdrillParseGroupby$fetchContaioner.<init>(MdrillParseGroupby.java:131)
    at org.apache.solr.request.mdrill.MdrillParseGroupby.createContainer(MdrillParseGroupby.java:84)
    at org.apache.solr.request.mdrill.MdrillGroupBy.get(MdrillGroupBy.java:65)
    at org.apache.lucene.index.SegmentReader.invertScan(SegmentReader.java:559)
    at org.apache.lucene.index.DirectoryReader.invertScan(DirectoryReader.java:591)
    at org.apache.lucene.index.FilterIndexReader.invertScan(FilterIndexReader.java:317)
    at org.apache.solr.request.mdrill.FacetComponent.getResult(FacetComponent.java:106)
    at org.apache.solr.request.mdrill.FacetComponent.process(FacetComponent.java:81)
    at org.apache.solr.handler.component.SearchHandler.handleRequestBody(SearchHandler.java:101)
    at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:129)
    at org.apache.solr.core.SolrCore.execute(SolrCore.java:1510)
    at org.apache.solr.servlet.SolrDispatchFilter.execute(SolrDispatchFilter.java:264)
    at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:198)
    at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212)
    at org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:399)
    at org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182)
    at org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)
    at org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
    at org.mortbay.jetty.Server.handle(Server.java:326)
    at org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542)
    at org.mortbay.jetty.HttpConnection$RequestHandler.content(HttpConnection.java:945)
    at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:756)
    at org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:218)
    at org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:404)
    at org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:410)
    at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582)

2013-09-29 10:07:34 IndexUtils [ERROR] higolog copyToLocal error

from mdrill.

错误原因 文件锁 在同一个线程下是不生效的 ，会报错

        for(int i=0;i<1000;i++)
        {
            try
            {
                flout = fcout.lock();
                break;
            }catch(OverlappingFileLockException e){
                Thread.sleep(1000);
            }
        }

from mdrill.

JDK 1.4的文件锁"file locking"允许你以文件为共享资源，对访问进行同步化处理(allows you to synchronize access to a file as a shared resource)。但是，竞争文件的两个线程必须属于两个不同的JVM，或者一个是Java线程，另一个是操作系统的本地线程。由于Java的文件锁是直接映射操作系统的锁机制的，因此其它进程也能看到文件锁。

from mdrill.

2013-09-29 13:21:10 HigoJoinInvert [INFO] ##joinright##2001
2013-09-29 13:21:10 TryLockFile [ERROR] trylock
java.io.IOException: Resource deadlock avoided
at sun.nio.ch.FileChannelImpl.lock0(Native Method)
at sun.nio.ch.FileChannelImpl.lock(FileChannelImpl.java:845)
at java.nio.channels.FileChannel.lock(FileChannel.java:860)
at com.alimama.mdrill.utils.TryLockFile.trylock(TryLockFile.java:39)
at org.apache.solr.request.join.HigoJoinUtils.maybeclear(HigoJoinUtils.java:166)
at org.apache.solr.request.join.HigoJoinUtils.pathToLocal(HigoJoinUtils.java:276)
at org.apache.solr.request.join.HigoJoinUtils.getSearch(HigoJoinUtils.java:82)
at org.apache.solr.request.join.HigoJoinInvert.open(HigoJoinInvert.java:60)
at org.apache.solr.request.mdrill.MdrillParseGroupby$fetchContaioner.(MdrillParseGroupby.java:131)
at org.apache.solr.request.mdrill.MdrillParseGroupby.createContainer(MdrillParseGroupby.java:84)
at org.apache.solr.request.mdrill.MdrillGroupBy.get(MdrillGroupBy.java:65)
at org.apache.lucene.index.SegmentReader.invertScan(SegmentReader.java:559)
at org.apache.lucene.index.DirectoryReader.invertScan(DirectoryReader.java:591)
at org.apache.lucene.index.FilterIndexReader.invertScan(FilterIndexReader.java:317)
at org.apache.solr.request.mdrill.FacetComponent.getResult(FacetComponent.java:106)
at org.apache.solr.request.mdrill.FacetComponent.process(FacetComponent.java:81)
at org.apache.solr.handler.component.SearchHandler.handleRequestBody(SearchHandler.java:101)
at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:129)
at org.apache.solr.core.SolrCore.execute(SolrCore.java:1510)
at org.apache.solr.servlet.SolrDispatchFilter.execute(SolrDispatchFilter.java:264)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:198)
at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212)
at org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:399)
at org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182)
at org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)
at org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
at org.mortbay.jetty.Server.handle(Server.java:326)
at org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542)
at org.mortbay.jetty.HttpConnection$RequestHandler.content(HttpConnection.java:945)
at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:756)
at org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:218)
at org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:404)
at org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:410)
at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582)
2013-09-29 13:21:10 HigoJoinUtils [INFO] begin clean /disk2/taobao/mdrill/higojoin_work,600000,1380432070584

from mdrill.

还是有报错

2013-09-29 13:49:59 TryLockFile [ERROR] trylock
java.io.IOException: Resource deadlock avoided
at sun.nio.ch.FileChannelImpl.lock0(Native Method)
at sun.nio.ch.FileChannelImpl.lock(FileChannelImpl.java:845)
at java.nio.channels.FileChannel.lock(FileChannel.java:860)
at com.alimama.mdrill.utils.TryLockFile.trylock(TryLockFile.java:39)
at org.apache.solr.request.join.HigoJoinInvert.open(HigoJoinInvert.java:61)
at org.apache.solr.request.mdrill.MdrillParseGroupby$fetchContaioner.(MdrillParseGroupby.java:131)
at org.apache.solr.request.mdrill.MdrillParseGroupby.createContainer(MdrillParseGroupby.java:84)
at org.apache.solr.request.mdrill.MdrillGroupBy.get(MdrillGroupBy.java:65)
at org.apache.lucene.index.SegmentReader.invertScan(SegmentReader.java:559)
at org.apache.lucene.index.DirectoryReader.invertScan(DirectoryReader.java:591)
at org.apache.lucene.index.FilterIndexReader.invertScan(FilterIndexReader.java:317)
at org.apache.solr.request.mdrill.FacetComponent.getResult(FacetComponent.java:106)
at org.apache.solr.request.mdrill.FacetComponent.process(FacetComponent.java:81)
at org.apache.solr.handler.component.SearchHandler.handleRequestBody(SearchHandler.java:101)
at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:129)
at org.apache.solr.core.SolrCore.execute(SolrCore.java:1510)
at org.apache.solr.servlet.SolrDispatchFilter.execute(SolrDispatchFilter.java:264)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:198)
at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212)
at org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:399)
at org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182)
at org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)
at org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
at org.mortbay.jetty.Server.handle(Server.java:326)
at org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542)
at org.mortbay.jetty.HttpConnection$RequestHandler.content(HttpConnection.java:945)
at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:756)
at org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:218)
at org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:404)
at org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:410)
at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582)
2013-09-29 13:49:59 HigoJoinUtils [INFO] begin clean /disk6/taobao/mdrill/higojoin_work,600000,1380433799229
2013-09-29 13:49:59 HigoJoinUtils [INFO] ###joinpath###/disk6/taobao/mdrill/higojoin_work/0fed6521-0c6f-4be8-a709-c18f5dcb09ac
2013-09-29 13:49:59 SolrCore [INFO] getSearcher:join@1380432964106@/disk6/taobao/mdrill/higojoin_work/0fed6521-0c6f-4be8-a709-c18f5dcb09ac:/disk6/taobao/mdrill/higojoin_work/0fed6521-0c6f-4be8-a709-c18f5dcb09ac
2013-09-29 13:49:59 SolrCore [INFO] >>>>>LinkFSDirectory readOnlyOpen add links /disk6/taobao/mdrill/higojoin_work/0fed6521-0c6f-4be8-a709-c18f5dcb09ac/store
2013-09-29 13:49:59 SolrIndexSearcher [INFO] Opening Searcher@11b23bb1 partion_join@1380432964106@/disk6/taobao/mdrill/higojoin_work/0fed6521-0c6f-4be8-a709-c18f5dcb09ac
2013-09-29 13:49:59 HigoJoinInvert [INFO] ##fqlist.size()##1
2013-09-29 13:49:59 HigoJoinInvert [INFO] ##joinright##2001
2013-09-29 13:49:59 TryLockFile [ERROR] trylock
java.io.IOException: Resource deadlock avoided
at sun.nio.ch.FileChannelImpl.lock0(Native Method)
at sun.nio.ch.FileChannelImpl.lock(FileChannelImpl.java:845)
at java.nio.channels.FileChannel.lock(FileChannel.java:860)
at com.alimama.mdrill.utils.TryLockFile.trylock(TryLockFile.java:39)
at com.alimama.mdrill.buffer.TryLock.tryLock(TryLock.java:46)
at org.apache.solr.request.uninverted.UnInvertedField$1.call(UnInvertedField.java:519)
at org.apache.solr.request.uninverted.UnInvertedField$1.call(UnInvertedField.java:514)
at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303)
at java.util.concurrent.FutureTask.run(FutureTask.java:138)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:441)
at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303)
at java.util.concurrent.FutureTask.run(FutureTask.java:138)
at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:619)
2013-09-29 13:49:59 SolrCore [INFO] ####UnInverted#### begin

from mdrill.