Hi, I have this error when running a spark task.
Here is task json.
{
"type": "index_spark",
"dataSchema": {
"dataSource": "ad_statistics_history",
"parser": {
"type": "string",
"parseSpec": {
"format": "tsv",
"timestampSpec": {
"column": "time",
"format": "auto",
"missingValue": null
},
"dimensionsSpec": {
"dimensions": [
"time_frame",
"group_id",
"network_id",
"advertiser_id",
"lineitem_id",
"campaign_id",
"creative_id",
"payment_model",
"is_ad_default",
"website_id",
"section_id",
"channel_id",
"zone_id",
"placement_id",
"template_id",
"zone_format",
"topic_id",
"interest_id",
"inmarker_id",
"topic_curr_id",
"interest_curr_id",
"inmarket_curr_id",
"audience_id",
"location_id",
"os_id",
"os_version_id",
"browser_id",
"device_type",
"device_id",
"carrier_id",
"age_range_id",
"gender_id"
],
"dimensionExclusions": [
],
"spatialDimensions": []
},
"delimiter": ";",
"columns": [
"time",
"time_frame",
"group_id",
"network_id",
"advertiser_id",
"lineitem_id",
"campaign_id",
"creative_id",
"payment_model",
"is_ad_default",
"website_id",
"section_id",
"channel_id",
"zone_id",
"placement_id",
"template_id",
"zone_format",
"topic_id",
"interest_id",
"inmarker_id",
"topic_curr_id",
"interest_curr_id",
"inmarket_curr_id",
"audience_id",
"location_id",
"os_id",
"os_version_id",
"browser_id",
"device_type",
"device_id",
"carrier_id",
"age_range_id",
"gender_id",
"impression",
"viewable",
"click",
"click_fraud",
"revenue",
"proceeds",
"spent",
"user_sketches",
"ad_request",
"true_ad_request",
"pageview",
"converted_click",
"conversion",
"conv_value",
"f_converted_click",
"f_conversion",
"f_conv_value",
"a_converted_click",
"a_conversion",
"a_conv_value",
"l_converted_click",
"l_conversion",
"l_conv_value"
]
},
"encoding": "UTF-8"
},
"metricsSpec": [
{
"type": "count",
"name": "count"
},
{
"type": "hyperUnique",
"name": "user_sketches",
"fieldName": "user_sketches"
},
{
"type": "longSum",
"name": "pageview",
"fieldName": "pageview"
},
{
"type": "longSum",
"name": "impression",
"fieldName": "impression"
},
{
"type": "longSum",
"name": "viewable",
"fieldName": "viewable"
},
{
"type": "longSum",
"name": "click",
"fieldName": "click"
},
{
"type": "longSum",
"name": "click_fraud",
"fieldName": "click_fraud"
},
{
"type": "doubleSum",
"name": "revenue",
"fieldName": "revenue"
},
{
"type": "doubleSum",
"name": "proceeds",
"fieldName": "proceeds"
},
{
"type": "doubleSum",
"name": "spent",
"fieldName": "spent"
},
{
"type": "longSum",
"name": "ad_request",
"fieldName": "ad_request"
},
{
"type": "longSum",
"name": "converted_click",
"fieldName": "converted_click"
},
{
"type": "longSum",
"name": "conversion",
"fieldName": "conversion"
},
{
"type": "doubleSum",
"name": "conv_value",
"fieldName": "conv_value"
},
{
"type": "longSum",
"name": "f_converted_click",
"fieldName": "f_converted_click"
},
{
"type": "longSum",
"name": "f_conversion",
"fieldName": "f_conversion"
},
{
"type": "doubleSum",
"name": "f_conv_value",
"fieldName": "f_conv_value"
},
{
"type": "longSum",
"name": "a_converted_click",
"fieldName": "a_converted_click"
},
{
"type": "longSum",
"name": "a_conversion",
"fieldName": "f_conversion"
},
{
"type": "doubleSum",
"name": "a_conv_value",
"fieldName": "f_conv_value"
},
{
"type": "longSum",
"name": "l_converted_click",
"fieldName": "l_converted_click"
},
{
"type": "longSum",
"name": "l_conversion",
"fieldName": "l_conversion"
},
{
"type": "doubleSum",
"name": "l_conv_value",
"fieldName": "l_conv_value"
},
{
"type": "longSum",
"name": "true_ad_request",
"fieldName": "true_ad_request"
}
],
"granularitySpec": {
"type": "uniform",
"segmentGranularity": "HOUR",
"queryGranularity": "HOUR",
"intervals": [
"2016-10-19T01:00:00.000Z/2016-10-19T02:00:00.000Z"
]
}
},
"intervals": [
"2016-10-19T01:00:00.000Z/2016-10-19T02:00:00.000Z"
],
"paths": [
"/ad-statistic-hourly-utc/pageview/2016-10-19-01",
"/ad-statistic-hourly-utc/pageview-20/2016-10-19-01",
"/ad-statistic-hourly-utc/viewable/2016-10-19-01",
"/ad-statistic-hourly-utc/click/2016-10-19-01",
"/ad-statistic-hourly-utc/conversion/2016-10-19-01"
],
"targetPartitionSize": 500000000,
"properties": {
"java.util.logging.manager": "org.apache.logging.log4j.jul.LogManager",
"user.timezone": "UTC",
"org.jboss.logging.provider": "log4j2",
"file.encoding": "UTF-8"
},
"master": "local[8]",
"context": {},
"indexSpec": {
"bitmap": {
"type": "concise"
},
"dimensionCompression": "lz4",
"metricCompression": "lz4"
},
"hadoopDependencyCoordinates": [
"org.apache.spark:spark-core_2.10:1.6.1"
],
"dataSource": "ad_statistics_history"
}Total time for which application threads were stopped: 0.0004104 seconds, Stopping threads took: 0.0000443 seconds
2016-10-19T04:59:24,669 INFO [task-runner-0-priority-0] org.spark-project.jetty.server.handler.ContextHandler - stopped o.s.j.s.ServletContextHandler{/metrics/json,null}
2016-10-19T04:59:24,669 INFO [task-runner-0-priority-0] org.spark-project.jetty.server.handler.ContextHandler - stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null}
2016-10-19T04:59:24,669 INFO [task-runner-0-priority-0] org.spark-project.jetty.server.handler.ContextHandler - stopped o.s.j.s.ServletContextHandler{/api,null}
2016-10-19T04:59:24,669 INFO [task-runner-0-priority-0] org.spark-project.jetty.server.handler.ContextHandler - stopped o.s.j.s.ServletContextHandler{/,null}
2016-10-19T04:59:24,669 INFO [task-runner-0-priority-0] org.spark-project.jetty.server.handler.ContextHandler - stopped o.s.j.s.ServletContextHandler{/static,null}
2016-10-19T04:59:24,669 INFO [task-runner-0-priority-0] org.spark-project.jetty.server.handler.ContextHandler - stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null}
2016-10-19T04:59:24,670 INFO [task-runner-0-priority-0] org.spark-project.jetty.server.handler.ContextHandler - stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null}
2016-10-19T04:59:24,670 INFO [task-runner-0-priority-0] org.spark-project.jetty.server.handler.ContextHandler - stopped o.s.j.s.ServletContextHandler{/executors/json,null}
2016-10-19T04:59:24,670 INFO [task-runner-0-priority-0] org.spark-project.jetty.server.handler.ContextHandler - stopped o.s.j.s.ServletContextHandler{/executors,null}
2016-10-19T04:59:24,670 INFO [task-runner-0-priority-0] org.spark-project.jetty.server.handler.ContextHandler - stopped o.s.j.s.ServletContextHandler{/environment/json,null}
2016-10-19T04:59:24,670 INFO [task-runner-0-priority-0] org.spark-project.jetty.server.handler.ContextHandler - stopped o.s.j.s.ServletContextHandler{/environment,null}
2016-10-19T04:59:24,670 INFO [task-runner-0-priority-0] org.spark-project.jetty.server.handler.ContextHandler - stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null}
2016-10-19T04:59:24,670 INFO [task-runner-0-priority-0] org.spark-project.jetty.server.handler.ContextHandler - stopped o.s.j.s.ServletContextHandler{/storage/rdd,null}
2016-10-19T04:59:24,670 INFO [task-runner-0-priority-0] org.spark-project.jetty.server.handler.ContextHandler - stopped o.s.j.s.ServletContextHandler{/storage/json,null}
2016-10-19T04:59:24,670 INFO [task-runner-0-priority-0] org.spark-project.jetty.server.handler.ContextHandler - stopped o.s.j.s.ServletContextHandler{/storage,null}
2016-10-19T04:59:24,670 INFO [task-runner-0-priority-0] org.spark-project.jetty.server.handler.ContextHandler - stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null}
2016-10-19T04:59:24,670 INFO [task-runner-0-priority-0] org.spark-project.jetty.server.handler.ContextHandler - stopped o.s.j.s.ServletContextHandler{/stages/pool,null}
2016-10-19T04:59:24,670 INFO [task-runner-0-priority-0] org.spark-project.jetty.server.handler.ContextHandler - stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null}
2016-10-19T04:59:24,670 INFO [task-runner-0-priority-0] org.spark-project.jetty.server.handler.ContextHandler - stopped o.s.j.s.ServletContextHandler{/stages/stage,null}
2016-10-19T04:59:24,670 INFO [task-runner-0-priority-0] org.spark-project.jetty.server.handler.ContextHandler - stopped o.s.j.s.ServletContextHandler{/stages/json,null}
2016-10-19T04:59:24,670 INFO [task-runner-0-priority-0] org.spark-project.jetty.server.handler.ContextHandler - stopped o.s.j.s.ServletContextHandler{/stages,null}
2016-10-19T04:59:24,670 INFO [task-runner-0-priority-0] org.spark-project.jetty.server.handler.ContextHandler - stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null}
2016-10-19T04:59:24,670 INFO [task-runner-0-priority-0] org.spark-project.jetty.server.handler.ContextHandler - stopped o.s.j.s.ServletContextHandler{/jobs/job,null}
2016-10-19T04:59:24,670 INFO [task-runner-0-priority-0] org.spark-project.jetty.server.handler.ContextHandler - stopped o.s.j.s.ServletContextHandler{/jobs/json,null}
2016-10-19T04:59:24,670 INFO [task-runner-0-priority-0] org.spark-project.jetty.server.handler.ContextHandler - stopped o.s.j.s.ServletContextHandler{/jobs,null}
2016-10-19T04:59:24,726 INFO [task-runner-0-priority-0] org.apache.spark.ui.SparkUI - Stopped Spark web UI at http://10.199.0.20:4040
vmop [threads: total initially_running wait_to_block] [time: spin block sync cleanup vmop] page_trap_count
538.289: RevokeBias [ 290 0 0 ] [ 0 0 0 0 0 ] 0
Total time for which application threads were stopped: 0.0005247 seconds, Stopping threads took: 0.0000820 seconds
vmop [threads: total initially_running wait_to_block] [time: spin block sync cleanup vmop] page_trap_count
538.291: RevokeBias [ 289 0 0 ] [ 0 0 0 0 0 ] 0
Total time for which application threads were stopped: 0.0005893 seconds, Stopping threads took: 0.0000842 seconds
vmop [threads: total initially_running wait_to_block] [time: spin block sync cleanup vmop] page_trap_count
538.297: RevokeBias [ 283 0 2 ] [ 0 0 0 0 0 ] 0
Total time for which application threads were stopped: 0.0008298 seconds, Stopping threads took: 0.0002542 seconds
2016-10-19T04:59:24,745 INFO [dispatcher-event-loop-17] org.apache.spark.MapOutputTrackerMasterEndpoint - MapOutputTrackerMasterEndpoint stopped!
vmop [threads: total initially_running wait_to_block] [time: spin block sync cleanup vmop] page_trap_count
538.309: RevokeBias [ 277 0 2 ] [ 0 0 0 0 0 ] 0
Total time for which application threads were stopped: 0.0006842 seconds, Stopping threads took: 0.0001366 seconds
vmop [threads: total initially_running wait_to_block] [time: spin block sync cleanup vmop] page_trap_count
539.310: no vm operation [ 283 0 0 ] [ 0 0 0 0 0 ] 0
Total time for which application threads were stopped: 0.0011107 seconds, Stopping threads took: 0.0001984 seconds
vmop [threads: total initially_running wait_to_block] [time: spin block sync cleanup vmop] page_trap_count
540.513: RevokeBias [ 283 0 0 ] [ 0 0 0 0 0 ] 0
Total time for which application threads were stopped: 0.0010052 seconds, Stopping threads took: 0.0001971 seconds
vmop [threads: total initially_running wait_to_block] [time: spin block sync cleanup vmop] page_trap_count
540.514: RevokeBias [ 283 0 0 ] [ 0 0 0 0 0 ] 0
Total time for which application threads were stopped: 0.0005140 seconds, Stopping threads took: 0.0001181 seconds
vmop [threads: total initially_running wait_to_block] [time: spin block sync cleanup vmop] page_trap_count
540.515: RevokeBias [ 283 2 1 ] [ 0 0 0 0 0 ] 0
Total time for which application threads were stopped: 0.0006320 seconds, Stopping threads took: 0.0001637 seconds
vmop [threads: total initially_running wait_to_block] [time: spin block sync cleanup vmop] page_trap_count
540.516: RevokeBias [ 283 1 3 ] [ 0 0 0 0 0 ] 0
Total time for which application threads were stopped: 0.0005673 seconds, Stopping threads took: 0.0002083 seconds
vmop [threads: total initially_running wait_to_block] [time: spin block sync cleanup vmop] page_trap_count
540.516: RevokeBias [ 283 1 5 ] [ 0 0 0 0 0 ] 0
Total time for which application threads were stopped: 0.0005311 seconds, Stopping threads took: 0.0001341 seconds
vmop [threads: total initially_running wait_to_block] [time: spin block sync cleanup vmop] page_trap_count
540.519: RevokeBias [ 276 0 0 ] [ 0 0 0 0 0 ] 0
Total time for which application threads were stopped: 0.0005611 seconds, Stopping threads took: 0.0000896 seconds
vmop [threads: total initially_running wait_to_block] [time: spin block sync cleanup vmop] page_trap_count
541.520: no vm operation [ 275 0 0 ] [ 0 0 0 0 0 ] 0
Total time for which application threads were stopped: 0.0010081 seconds, Stopping threads took: 0.0002003 seconds
2016-10-19T04:59:29,120 INFO [task-runner-0-priority-0] org.apache.spark.storage.MemoryStore - MemoryStore cleared
vmop [threads: total initially_running wait_to_block] [time: spin block sync cleanup vmop] page_trap_count
542.681: RevokeBias [ 275 0 1 ] [ 0 0 0 0 0 ] 0
Total time for which application threads were stopped: 0.0009716 seconds, Stopping threads took: 0.0002078 seconds
2016-10-19T04:59:29,121 INFO [task-runner-0-priority-0] org.apache.spark.storage.BlockManager - BlockManager stopped
2016-10-19T04:59:29,122 INFO [task-runner-0-priority-0] org.apache.spark.storage.BlockManagerMaster - BlockManagerMaster stopped
vmop [threads: total initially_running wait_to_block] [time: spin block sync cleanup vmop] page_trap_count
542.684: RevokeBias [ 273 0 0 ] [ 0 0 0 0 0 ] 0
Total time for which application threads were stopped: 0.0003565 seconds, Stopping threads took: 0.0000569 seconds
2016-10-19T04:59:29,128 INFO [dispatcher-event-loop-22] org.apache.spark.scheduler.OutputCommitCoordinator$OutputCommitCoordinatorEndpoint - OutputCommitCoordinator stopped!
2016-10-19T04:59:29,139 INFO [sparkDriverActorSystem-akka.actor.default-dispatcher-14] akka.remote.RemoteActorRefProvider$RemotingTerminator - Shutting down remote daemon.
2016-10-19T04:59:29,142 INFO [sparkDriverActorSystem-akka.actor.default-dispatcher-14] akka.remote.RemoteActorRefProvider$RemotingTerminator - Remote daemon shut down; proceeding with flushing remote transports.
vmop [threads: total initially_running wait_to_block] [time: spin block sync cleanup vmop] page_trap_count
542.714: RevokeBias [ 233 0 1 ] [ 0 0 0 0 0 ] 0
Total time for which application threads were stopped: 0.0008804 seconds, Stopping threads took: 0.0001897 seconds
vmop [threads: total initially_running wait_to_block] [time: spin block sync cleanup vmop] page_trap_count
542.720: RevokeBias [ 233 0 0 ] [ 0 0 0 0 0 ] 0
Total time for which application threads were stopped: 0.0003667 seconds, Stopping threads took: 0.0000499 seconds
vmop [threads: total initially_running wait_to_block] [time: spin block sync cleanup vmop] page_trap_count
542.721: RevokeBias [ 233 0 0 ] [ 0 0 0 0 0 ] 0
Total time for which application threads were stopped: 0.0002772 seconds, Stopping threads took: 0.0000408 seconds
2016-10-19T04:59:29,161 INFO [task-runner-0-priority-0] org.apache.spark.SparkContext - Successfully stopped SparkContext
vmop [threads: total initially_running wait_to_block] [time: spin block sync cleanup vmop] page_trap_count
542.747: RevokeBias [ 228 0 1 ] [ 0 0 0 0 0 ] 0
Total time for which application threads were stopped: 0.0007918 seconds, Stopping threads took: 0.0001704 seconds
2016-10-19T04:59:29,188 INFO [sparkDriverActorSystem-akka.actor.default-dispatcher-14] akka.remote.RemoteActorRefProvider$RemotingTerminator - Remoting shut down.
2016-10-19T04:59:29,187 ERROR [task-runner-0-priority-0] io.druid.indexer.spark.SparkBatchIndexTask - Error running task [index_spark_ad_statistics_history_2016-10-19T01:00:00.000Z_2016-10-19T02:00:00.000Z_2016-10-19T04:50:26.408Z]
java.lang.RuntimeException: java.lang.reflect.InvocationTargetException
at com.google.common.base.Throwables.propagate(Throwables.java:160) ~[guava-16.0.1.jar:?]
at io.druid.indexing.common.task.HadoopTask.invokeForeignLoader(HadoopTask.java:204) ~[druid-indexing-service-0.9.2-rc2-SNAPSHOT.jar:0.9.2-rc2-SNAPSHOT]
at io.druid.indexer.spark.SparkBatchIndexTask.run(SparkBatchIndexTask.scala:152) [druid-spark-batch_2.10-0.9.1-3-SNAPSHOT.jar:0.9.1-3-SNAPSHOT]
at io.druid.indexing.overlord.ThreadPoolTaskRunner$ThreadPoolTaskRunnerCallable.call(ThreadPoolTaskRunner.java:436) [druid-indexing-service-0.9.2-rc2-SNAPSHOT.jar:0.9.2-rc2-SNAPSHOT]
at io.druid.indexing.overlord.ThreadPoolTaskRunner$ThreadPoolTaskRunnerCallable.call(ThreadPoolTaskRunner.java:408) [druid-indexing-service-0.9.2-rc2-SNAPSHOT.jar:0.9.2-rc2-SNAPSHOT]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) [?:1.8.0_77]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) [?:1.8.0_77]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [?:1.8.0_77]
at java.lang.Thread.run(Thread.java:745) [?:1.8.0_77]
Caused by: java.lang.reflect.InvocationTargetException
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_77]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_77]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_77]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_77]
at io.druid.indexing.common.task.HadoopTask.invokeForeignLoader(HadoopTask.java:201) ~[druid-indexing-service-0.9.2-rc2-SNAPSHOT.jar:0.9.2-rc2-SNAPSHOT]
... 7 more
Caused by: java.lang.RuntimeException: org.apache.spark.SparkException: Job aborted due to stage failure: Task 0 in stage 1.0 failed 1 times, most recent failure: Lost task 0.0 in stage 1.0 (TID 80, localhost): java.lang.IllegalArgumentException: Size exceeds Integer.MAX_VALUE
at sun.nio.ch.FileChannelImpl.map(FileChannelImpl.java:869)
at org.apache.spark.storage.DiskStore$$anonfun$getBytes$2.apply(DiskStore.scala:127)
at org.apache.spark.storage.DiskStore$$anonfun$getBytes$2.apply(DiskStore.scala:115)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1250)
at org.apache.spark.storage.DiskStore.getBytes(DiskStore.scala:129)
at org.apache.spark.storage.DiskStore.getBytes(DiskStore.scala:136)
at org.apache.spark.storage.BlockManager.doGetLocal(BlockManager.scala:503)
at org.apache.spark.storage.BlockManager.getLocal(BlockManager.scala:420)
at org.apache.spark.storage.BlockManager.get(BlockManager.scala:625)
at org.apache.spark.CacheManager.putInBlockManager(CacheManager.scala:154)
at org.apache.spark.CacheManager.getOrCompute(CacheManager.scala:78)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:268)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:306)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:270)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:306)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:270)
at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:73)
at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:41)
at org.apache.spark.scheduler.Task.run(Task.scala:89)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
Driver stacktrace:
at com.google.common.io.Closer.rethrow(Closer.java:149) ~[guava-16.0.1.jar:?]
at io.druid.indexer.spark.SparkBatchIndexTask$.runTask(SparkBatchIndexTask.scala:369) ~[?:?]
at io.druid.indexer.spark.SparkBatchIndexTask.runTask(SparkBatchIndexTask.scala) ~[?:?]
at io.druid.indexer.spark.Runner.runTask(Runner.java:29) ~[?:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_77]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_77]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_77]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_77]
at io.druid.indexing.common.task.HadoopTask.invokeForeignLoader(HadoopTask.java:201) ~[druid-indexing-service-0.9.2-rc2-SNAPSHOT.jar:0.9.2-rc2-SNAPSHOT]
... 7 more
Caused by: org.apache.spark.SparkException: Job aborted due to stage failure: Task 0 in stage 1.0 failed 1 times, most recent failure: Lost task 0.0 in stage 1.0 (TID 80, localhost): java.lang.IllegalArgumentException: Size exceeds Integer.MAX_VALUE
at sun.nio.ch.FileChannelImpl.map(FileChannelImpl.java:869)
at org.apache.spark.storage.DiskStore$$anonfun$getBytes$2.apply(DiskStore.scala:127)
at org.apache.spark.storage.DiskStore$$anonfun$getBytes$2.apply(DiskStore.scala:115)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1250)
at org.apache.spark.storage.DiskStore.getBytes(DiskStore.scala:129)
at org.apache.spark.storage.DiskStore.getBytes(DiskStore.scala:136)
at org.apache.spark.storage.BlockManager.doGetLocal(BlockManager.scala:503)
at org.apache.spark.storage.BlockManager.getLocal(BlockManager.scala:420)
at org.apache.spark.storage.BlockManager.get(BlockManager.scala:625)
at org.apache.spark.CacheManager.putInBlockManager(CacheManager.scala:154)
at org.apache.spark.CacheManager.getOrCompute(CacheManager.scala:78)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:268)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:306)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:270)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:306)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:270)
at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:73)
at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:41)
at org.apache.spark.scheduler.Task.run(Task.scala:89)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
Driver stacktrace:
at org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1431) ~[?:?]
at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1419) ~[?:?]
at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1418) ~[?:?]
at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) ~[?:?]
at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:47) ~[?:?]
at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:1418) ~[?:?]
at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:799) ~[?:?]
at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:799) ~[?:?]
at scala.Option.foreach(Option.scala:236) ~[?:?]
at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:799) ~[?:?]
at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:1640) ~[?:?]
at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1599) ~[?:?]
at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1588) ~[?:?]
at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:48) ~[?:?]
at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:620) ~[?:?]
at org.apache.spark.SparkContext.runJob(SparkContext.scala:1832) ~[?:?]
at org.apache.spark.SparkContext.runJob(SparkContext.scala:1845) ~[?:?]
at org.apache.spark.SparkContext.runJob(SparkContext.scala:1858) ~[?:?]
at org.apache.spark.SparkContext.runJob(SparkContext.scala:1929) ~[?:?]
at org.apache.spark.rdd.RDD$$anonfun$collect$1.apply(RDD.scala:927) ~[?:?]
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:150) ~[?:?]
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:111) ~[?:?]
at org.apache.spark.rdd.RDD.withScope(RDD.scala:316) ~[?:?]
at org.apache.spark.rdd.RDD.collect(RDD.scala:926) ~[?:?]
at io.druid.indexer.spark.SparkDruidIndexer$.loadData(SparkDruidIndexer.scala:173) ~[?:?]
at io.druid.indexer.spark.SparkBatchIndexTask$.runTask(SparkBatchIndexTask.scala:418) ~[?:?]
at io.druid.indexer.spark.SparkBatchIndexTask.runTask(SparkBatchIndexTask.scala) ~[?:?]
at io.druid.indexer.spark.Runner.runTask(Runner.java:29) ~[?:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_77]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_77]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_77]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_77]
at io.druid.indexing.common.task.HadoopTask.invokeForeignLoader(HadoopTask.java:201) ~[druid-indexing-service-0.9.2-rc2-SNAPSHOT.jar:0.9.2-rc2-SNAPSHOT]
... 7 more
Caused by: java.lang.IllegalArgumentException: Size exceeds Integer.MAX_VALUE
at sun.nio.ch.FileChannelImpl.map(FileChannelImpl.java:869) ~[?:1.8.0_77]
at org.apache.spark.storage.DiskStore$$anonfun$getBytes$2.apply(DiskStore.scala:127) ~[?:?]
at org.apache.spark.storage.DiskStore$$anonfun$getBytes$2.apply(DiskStore.scala:115) ~[?:?]
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1250) ~[?:?]
at org.apache.spark.storage.DiskStore.getBytes(DiskStore.scala:129) ~[?:?]
at org.apache.spark.storage.DiskStore.getBytes(DiskStore.scala:136) ~[?:?]
at org.apache.spark.storage.BlockManager.doGetLocal(BlockManager.scala:503) ~[?:?]
at org.apache.spark.storage.BlockManager.getLocal(BlockManager.scala:420) ~[?:?]
at org.apache.spark.storage.BlockManager.get(BlockManager.scala:625) ~[?:?]
at org.apache.spark.CacheManager.putInBlockManager(CacheManager.scala:154) ~[?:?]
at org.apache.spark.CacheManager.getOrCompute(CacheManager.scala:78) ~[?:?]
at org.apache.spark.rdd.RDD.iterator(RDD.scala:268) ~[?:?]
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38) ~[?:?]
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:306) ~[?:?]
at org.apache.spark.rdd.RDD.iterator(RDD.scala:270) ~[?:?]
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38) ~[?:?]
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:306) ~[?:?]
at org.apache.spark.rdd.RDD.iterator(RDD.scala:270) ~[?:?]
at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:73) ~[?:?]
at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:41) ~[?:?]
at org.apache.spark.scheduler.Task.run(Task.scala:89) ~[?:?]
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214) ~[?:?]
... 3 more
2016-10-19T04:59:29,221 INFO [task-runner-0-priority-0] io.druid.indexing.overlord.TaskRunnerUtils - Task [index_spark_ad_statistics_history_2016-10-19T01:00:00.000Z_2016-10-19T02:00:00.000Z_2016-10-19T04:50:26.408Z] status changed to [FAILED].
vmop [threads: total initially_running wait_to_block] [time: spin block sync cleanup vmop] page_trap_count
542.783: RevokeBias [ 229 0 0 ] [ 0 0 0 0 0 ] 0
Total time for which application threads were stopped: 0.0007305 seconds, Stopping threads took: 0.0001496 seconds
2016-10-19T04:59:29,224 INFO [task-runner-0-priority-0] io.druid.indexing.worker.executor.ExecutorLifecycle - Task completed with status: {
"id" : "index_spark_ad_statistics_history_2016-10-19T01:00:00.000Z_2016-10-19T02:00:00.000Z_2016-10-19T04:50:26.408Z",
"status" : "FAILED",
"duration" : 536981
}
React
Typescript
Django
Laravel
Facebook
Microsoft
Google
Alibaba
D3
Tencent