spark:
export JAVA_HOME=/home/share/jdk1.8.0_121
export JRE_HOME=${JAVA_HOME}/jre
export CLASSPATH=.:${JAVVA_HOME}/lib:${JRE_HOME}/lib
export PATH=${JAVA_HOME}/bin:$PATH
export SPARK_HOME=/home/share/spark-2.1.0-bin-hadoop2.7
export PATH=$PATH:$SPARK_HOME/bin:$SPARK_HOME/sbin
export SCALA_HOME=/home/elchen/Downloads/scala-2.11.8
export PATH=$PATH:$SCALA_HOME/bin
export PATH=$PATH:/usr/local/mongodb/bin/
export LD_LIBRARY_PATH=/usr/lib:/usr/lib64:/lib:/lib64:/usr/local/lib:/usr/local/lib64:$LD_LIBRARY_PATH
export ES_HOME=/usr/local/elasticsearch-5.0.0
jars:
http://search.maven.org/#search%7Cga%7C1%7Cg%3A%22org.apache.spark%22%20AND%20v%3A%222.1.0%22
http://search.maven.org/remotecontent?filepath=org/apache/spark/spark-streaming-kafka-0-8-assembly_2.11/2.1.0/spark-streaming-kafka-0-8-assembly_2.11-2.1.0.jar
spark-streaming-kafka-0-8-assembly_2.11-2.1.0.jar
kafka:
kafka_2.11-0.8.2.0
zookeeper-3.4.10
spark-submit --master spark://host-192-168-1-20.openstacklocal:7077 --packages org.apache.spark:spark-streaming-kafka-0-8_2.11:2.1.0 streaming/test_kafka.py
spark-submit --master spark://host-192-168-1-20.openstacklocal:7077 --jars /home/elchen/Downloads/spark-streaming-kafka-0-8-assembly_2.11-2.1.0.jar spark_streaming_kafka.py
Ivy Default Cache set to: /root/.ivy2/cache
The jars for the packages stored in: /root/.ivy2/jars
└─(elchen@host-192-168-1-20)$ spark-submit --master spark://host-192-168-1-20.openstacklocal:7077 --packages org.apache.spark:spark-streaming-kafka-0-8_2.11:2.1.0 streaming/test_kafka.py Ivy Default Cache set to: /root/.ivy2/cache The jars for the packages stored in: /root/.ivy2/jars :: loading settings :: url = jar:file:/home/share/spark-2.1.0-bin-hadoop2.7/jars/ivy-2.4.0.jar!/org/apache/ivy/core/settings/ivysettings.xml org.apache.spark#spark-streaming-kafka-0-8_2.11 added as a dependency :: resolving dependencies :: org.apache.spark#spark-submit-parent;1.0 confs: [default] found org.apache.spark#spark-streaming-kafka-0-8_2.11;2.1.0 in central found org.apache.kafka#kafka_2.11;0.8.2.1 in central found org.scala-lang.modules#scala-xml_2.11;1.0.2 in central found com.yammer.metrics#metrics-core;2.2.0 in central found org.slf4j#slf4j-api;1.7.16 in central found org.scala-lang.modules#scala-parser-combinators_2.11;1.0.2 in central found com.101tec#zkclient;0.3 in central found log4j#log4j;1.2.17 in central found org.apache.kafka#kafka-clients;0.8.2.1 in central found net.jpountz.lz4#lz4;1.3.0 in central found org.xerial.snappy#snappy-java;1.1.2.6 in central found org.apache.spark#spark-tags_2.11;2.1.0 in central found org.scalatest#scalatest_2.11;2.2.6 in central found org.scala-lang#scala-reflect;2.11.8 in central found org.spark-project.spark#unused;1.0.0 in central :: resolution report :: resolve 5679ms :: artifacts dl 20ms :: modules in use: com.101tec#zkclient;0.3 from central in [default] com.yammer.metrics#metrics-core;2.2.0 from central in [default] log4j#log4j;1.2.17 from central in [default] net.jpountz.lz4#lz4;1.3.0 from central in [default] org.apache.kafka#kafka-clients;0.8.2.1 from central in [default] org.apache.kafka#kafka_2.11;0.8.2.1 from central in [default] org.apache.spark#spark-streaming-kafka-0-8_2.11;2.1.0 from central in [default] org.apache.spark#spark-tags_2.11;2.1.0 from central in [default] org.scala-lang#scala-reflect;2.11.8 from central in [default] org.scala-lang.modules#scala-parser-combinators_2.11;1.0.2 from central in [default] org.scala-lang.modules#scala-xml_2.11;1.0.2 from central in [default] org.scalatest#scalatest_2.11;2.2.6 from central in [default] org.slf4j#slf4j-api;1.7.16 from central in [default] org.spark-project.spark#unused;1.0.0 from central in [default] org.xerial.snappy#snappy-java;1.1.2.6 from central in [default] --------------------------------------------------------------------- | | modules || artifacts | | conf | number| search|dwnlded|evicted|| number|dwnlded| --------------------------------------------------------------------- | default | 15 | 2 | 2 | 0 || 15 | 0 | --------------------------------------------------------------------- :: retrieving :: org.apache.spark#spark-submit-parent confs: [default] 0 artifacts copied, 15 already retrieved (0kB/22ms) Using Spark's default log4j profile: org/apache/spark/log4j-defaults.properties 17/05/29 18:19:30 INFO SparkContext: Running Spark version 2.1.0 17/05/29 18:19:30 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 17/05/29 18:19:30 INFO SecurityManager: Changing view acls to: root 17/05/29 18:19:30 INFO SecurityManager: Changing modify acls to: root 17/05/29 18:19:30 INFO SecurityManager: Changing view acls groups to: 17/05/29 18:19:30 INFO SecurityManager: Changing modify acls groups to: 17/05/29 18:19:30 INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(root); groups with view permissions: Set(); users with modify permissions: Set(root); groups with modify permissions: Set() 17/05/29 18:19:30 INFO Utils: Successfully started service 'sparkDriver' on port 38293. 17/05/29 18:19:30 INFO SparkEnv: Registering MapOutputTracker 17/05/29 18:19:30 INFO SparkEnv: Registering BlockManagerMaster 17/05/29 18:19:30 INFO BlockManagerMasterEndpoint: Using org.apache.spark.storage.DefaultTopologyMapper for getting topology information 17/05/29 18:19:30 INFO BlockManagerMasterEndpoint: BlockManagerMasterEndpoint up 17/05/29 18:19:30 INFO DiskBlockManager: Created local directory at /tmp/blockmgr-acf7b093-bb94-4633-92a5-ae0969dd90d6 17/05/29 18:19:30 INFO MemoryStore: MemoryStore started with capacity 366.3 MB 17/05/29 18:19:30 INFO SparkEnv: Registering OutputCommitCoordinator 17/05/29 18:19:31 INFO Utils: Successfully started service 'SparkUI' on port 4040. 17/05/29 18:19:31 INFO SparkUI: Bound SparkUI to 0.0.0.0, and started at http://192.168.1.20:4040 17/05/29 18:19:31 INFO SparkContext: Added JAR file:/root/.ivy2/jars/org.apache.spark_spark-streaming-kafka-0-8_2.11-2.1.0.jar at spark://192.168.1.20:38293/jars/org.apache.spark_spark-streaming-kafka-0-8_2.11-2.1.0.jar with timestamp 1496053171339 17/05/29 18:19:31 INFO SparkContext: Added JAR file:/root/.ivy2/jars/org.apache.kafka_kafka_2.11-0.8.2.1.jar at spark://192.168.1.20:38293/jars/org.apache.kafka_kafka_2.11-0.8.2.1.jar with timestamp 1496053171340 17/05/29 18:19:31 INFO SparkContext: Added JAR file:/root/.ivy2/jars/org.apache.spark_spark-tags_2.11-2.1.0.jar at spark://192.168.1.20:38293/jars/org.apache.spark_spark-tags_2.11-2.1.0.jar with timestamp 1496053171340 17/05/29 18:19:31 INFO SparkContext: Added JAR file:/root/.ivy2/jars/org.spark-project.spark_unused-1.0.0.jar at spark://192.168.1.20:38293/jars/org.spark-project.spark_unused-1.0.0.jar with timestamp 1496053171341 17/05/29 18:19:31 INFO SparkContext: Added JAR file:/root/.ivy2/jars/org.scala-lang.modules_scala-xml_2.11-1.0.2.jar at spark://192.168.1.20:38293/jars/org.scala-lang.modules_scala-xml_2.11-1.0.2.jar with timestamp 1496053171341 17/05/29 18:19:31 INFO SparkContext: Added JAR file:/root/.ivy2/jars/com.yammer.metrics_metrics-core-2.2.0.jar at spark://192.168.1.20:38293/jars/com.yammer.metrics_metrics-core-2.2.0.jar with timestamp 1496053171341 17/05/29 18:19:31 INFO SparkContext: Added JAR file:/root/.ivy2/jars/org.scala-lang.modules_scala-parser-combinators_2.11-1.0.2.jar at spark://192.168.1.20:38293/jars/org.scala-lang.modules_scala-parser-combinators_2.11-1.0.2.jar with timestamp 1496053171341 17/05/29 18:19:31 INFO SparkContext: Added JAR file:/root/.ivy2/jars/com.101tec_zkclient-0.3.jar at spark://192.168.1.20:38293/jars/com.101tec_zkclient-0.3.jar with timestamp 1496053171341 17/05/29 18:19:31 INFO SparkContext: Added JAR file:/root/.ivy2/jars/org.apache.kafka_kafka-clients-0.8.2.1.jar at spark://192.168.1.20:38293/jars/org.apache.kafka_kafka-clients-0.8.2.1.jar with timestamp 1496053171341 17/05/29 18:19:31 INFO SparkContext: Added JAR file:/root/.ivy2/jars/org.slf4j_slf4j-api-1.7.16.jar at spark://192.168.1.20:38293/jars/org.slf4j_slf4j-api-1.7.16.jar with timestamp 1496053171342 17/05/29 18:19:31 INFO SparkContext: Added JAR file:/root/.ivy2/jars/log4j_log4j-1.2.17.jar at spark://192.168.1.20:38293/jars/log4j_log4j-1.2.17.jar with timestamp 1496053171342 17/05/29 18:19:31 INFO SparkContext: Added JAR file:/root/.ivy2/jars/net.jpountz.lz4_lz4-1.3.0.jar at spark://192.168.1.20:38293/jars/net.jpountz.lz4_lz4-1.3.0.jar with timestamp 1496053171342 17/05/29 18:19:31 INFO SparkContext: Added JAR file:/root/.ivy2/jars/org.xerial.snappy_snappy-java-1.1.2.6.jar at spark://192.168.1.20:38293/jars/org.xerial.snappy_snappy-java-1.1.2.6.jar with timestamp 1496053171342 17/05/29 18:19:31 INFO SparkContext: Added JAR file:/root/.ivy2/jars/org.scalatest_scalatest_2.11-2.2.6.jar at spark://192.168.1.20:38293/jars/org.scalatest_scalatest_2.11-2.2.6.jar with timestamp 1496053171342 17/05/29 18:19:31 INFO SparkContext: Added JAR file:/root/.ivy2/jars/org.scala-lang_scala-reflect-2.11.8.jar at spark://192.168.1.20:38293/jars/org.scala-lang_scala-reflect-2.11.8.jar with timestamp 1496053171342 17/05/29 18:19:31 INFO SparkContext: Added file file:/home/elchen/pyspark/pyspark/streaming/test_kafka.py at spark://192.168.1.20:38293/files/test_kafka.py with timestamp 1496053171568 17/05/29 18:19:31 INFO Utils: Copying /home/elchen/pyspark/pyspark/streaming/test_kafka.py to /tmp/spark-ff0978ee-1039-4222-b80a-550ef0f885ed/userFiles-1fb0f005-86f4-4515-906d-12e549bcdee3/test_kafka.py 17/05/29 18:19:31 INFO SparkContext: Added file file:/root/.ivy2/jars/org.apache.spark_spark-streaming-kafka-0-8_2.11-2.1.0.jar at spark://192.168.1.20:38293/files/org.apache.spark_spark-streaming-kafka-0-8_2.11-2.1.0.jar with timestamp 1496053171577 17/05/29 18:19:31 INFO Utils: Copying /root/.ivy2/jars/org.apache.spark_spark-streaming-kafka-0-8_2.11-2.1.0.jar to /tmp/spark-ff0978ee-1039-4222-b80a-550ef0f885ed/userFiles-1fb0f005-86f4-4515-906d-12e549bcdee3/org.apache.spark_spark-streaming-kafka-0-8_2.11-2.1.0.jar 17/05/29 18:19:31 INFO SparkContext: Added file file:/root/.ivy2/jars/org.apache.kafka_kafka_2.11-0.8.2.1.jar at spark://192.168.1.20:38293/files/org.apache.kafka_kafka_2.11-0.8.2.1.jar with timestamp 1496053171584 17/05/29 18:19:31 INFO Utils: Copying /root/.ivy2/jars/org.apache.kafka_kafka_2.11-0.8.2.1.jar to /tmp/spark-ff0978ee-1039-4222-b80a-550ef0f885ed/userFiles-1fb0f005-86f4-4515-906d-12e549bcdee3/org.apache.kafka_kafka_2.11-0.8.2.1.jar 17/05/29 18:19:31 INFO SparkContext: Added file file:/root/.ivy2/jars/org.apache.spark_spark-tags_2.11-2.1.0.jar at spark://192.168.1.20:38293/files/org.apache.spark_spark-tags_2.11-2.1.0.jar with timestamp 1496053171595 17/05/29 18:19:31 INFO Utils: Copying /root/.ivy2/jars/org.apache.spark_spark-tags_2.11-2.1.0.jar to /tmp/spark-ff0978ee-1039-4222-b80a-550ef0f885ed/userFiles-1fb0f005-86f4-4515-906d-12e549bcdee3/org.apache.spark_spark-tags_2.11-2.1.0.jar 17/05/29 18:19:31 INFO SparkContext: Added file file:/root/.ivy2/jars/org.spark-project.spark_unused-1.0.0.jar at spark://192.168.1.20:38293/files/org.spark-project.spark_unused-1.0.0.jar with timestamp 1496053171599 17/05/29 18:19:31 INFO Utils: Copying /root/.ivy2/jars/org.spark-project.spark_unused-1.0.0.jar to /tmp/spark-ff0978ee-1039-4222-b80a-550ef0f885ed/userFiles-1fb0f005-86f4-4515-906d-12e549bcdee3/org.spark-project.spark_unused-1.0.0.jar 17/05/29 18:19:31 INFO SparkContext: Added file file:/root/.ivy2/jars/org.scala-lang.modules_scala-xml_2.11-1.0.2.jar at spark://192.168.1.20:38293/files/org.scala-lang.modules_scala-xml_2.11-1.0.2.jar with timestamp 1496053171604 17/05/29 18:19:31 INFO Utils: Copying /root/.ivy2/jars/org.scala-lang.modules_scala-xml_2.11-1.0.2.jar to /tmp/spark-ff0978ee-1039-4222-b80a-550ef0f885ed/userFiles-1fb0f005-86f4-4515-906d-12e549bcdee3/org.scala-lang.modules_scala-xml_2.11-1.0.2.jar 17/05/29 18:19:31 INFO SparkContext: Added file file:/root/.ivy2/jars/com.yammer.metrics_metrics-core-2.2.0.jar at spark://192.168.1.20:38293/files/com.yammer.metrics_metrics-core-2.2.0.jar with timestamp 1496053171609 17/05/29 18:19:31 INFO Utils: Copying /root/.ivy2/jars/com.yammer.metrics_metrics-core-2.2.0.jar to /tmp/spark-ff0978ee-1039-4222-b80a-550ef0f885ed/userFiles-1fb0f005-86f4-4515-906d-12e549bcdee3/com.yammer.metrics_metrics-core-2.2.0.jar 17/05/29 18:19:31 INFO SparkContext: Added file file:/root/.ivy2/jars/org.scala-lang.modules_scala-parser-combinators_2.11-1.0.2.jar at spark://192.168.1.20:38293/files/org.scala-lang.modules_scala-parser-combinators_2.11-1.0.2.jar with timestamp 1496053171613 17/05/29 18:19:31 INFO Utils: Copying /root/.ivy2/jars/org.scala-lang.modules_scala-parser-combinators_2.11-1.0.2.jar to /tmp/spark-ff0978ee-1039-4222-b80a-550ef0f885ed/userFiles-1fb0f005-86f4-4515-906d-12e549bcdee3/org.scala-lang.modules_scala-parser-combinators_2.11-1.0.2.jar 17/05/29 18:19:31 INFO SparkContext: Added file file:/root/.ivy2/jars/com.101tec_zkclient-0.3.jar at spark://192.168.1.20:38293/files/com.101tec_zkclient-0.3.jar with timestamp 1496053171618 17/05/29 18:19:31 INFO Utils: Copying /root/.ivy2/jars/com.101tec_zkclient-0.3.jar to /tmp/spark-ff0978ee-1039-4222-b80a-550ef0f885ed/userFiles-1fb0f005-86f4-4515-906d-12e549bcdee3/com.101tec_zkclient-0.3.jar 17/05/29 18:19:31 INFO SparkContext: Added file file:/root/.ivy2/jars/org.apache.kafka_kafka-clients-0.8.2.1.jar at spark://192.168.1.20:38293/files/org.apache.kafka_kafka-clients-0.8.2.1.jar with timestamp 1496053171623 17/05/29 18:19:31 INFO Utils: Copying /root/.ivy2/jars/org.apache.kafka_kafka-clients-0.8.2.1.jar to /tmp/spark-ff0978ee-1039-4222-b80a-550ef0f885ed/userFiles-1fb0f005-86f4-4515-906d-12e549bcdee3/org.apache.kafka_kafka-clients-0.8.2.1.jar 17/05/29 18:19:31 INFO SparkContext: Added file file:/root/.ivy2/jars/org.slf4j_slf4j-api-1.7.16.jar at spark://192.168.1.20:38293/files/org.slf4j_slf4j-api-1.7.16.jar with timestamp 1496053171629 17/05/29 18:19:31 INFO Utils: Copying /root/.ivy2/jars/org.slf4j_slf4j-api-1.7.16.jar to /tmp/spark-ff0978ee-1039-4222-b80a-550ef0f885ed/userFiles-1fb0f005-86f4-4515-906d-12e549bcdee3/org.slf4j_slf4j-api-1.7.16.jar 17/05/29 18:19:31 INFO SparkContext: Added file file:/root/.ivy2/jars/log4j_log4j-1.2.17.jar at spark://192.168.1.20:38293/files/log4j_log4j-1.2.17.jar with timestamp 1496053171635 17/05/29 18:19:31 INFO Utils: Copying /root/.ivy2/jars/log4j_log4j-1.2.17.jar to /tmp/spark-ff0978ee-1039-4222-b80a-550ef0f885ed/userFiles-1fb0f005-86f4-4515-906d-12e549bcdee3/log4j_log4j-1.2.17.jar 17/05/29 18:19:31 INFO SparkContext: Added file file:/root/.ivy2/jars/net.jpountz.lz4_lz4-1.3.0.jar at spark://192.168.1.20:38293/files/net.jpountz.lz4_lz4-1.3.0.jar with timestamp 1496053171641 17/05/29 18:19:31 INFO Utils: Copying /root/.ivy2/jars/net.jpountz.lz4_lz4-1.3.0.jar to /tmp/spark-ff0978ee-1039-4222-b80a-550ef0f885ed/userFiles-1fb0f005-86f4-4515-906d-12e549bcdee3/net.jpountz.lz4_lz4-1.3.0.jar 17/05/29 18:19:31 INFO SparkContext: Added file file:/root/.ivy2/jars/org.xerial.snappy_snappy-java-1.1.2.6.jar at spark://192.168.1.20:38293/files/org.xerial.snappy_snappy-java-1.1.2.6.jar with timestamp 1496053171648 17/05/29 18:19:31 INFO Utils: Copying /root/.ivy2/jars/org.xerial.snappy_snappy-java-1.1.2.6.jar to /tmp/spark-ff0978ee-1039-4222-b80a-550ef0f885ed/userFiles-1fb0f005-86f4-4515-906d-12e549bcdee3/org.xerial.snappy_snappy-java-1.1.2.6.jar 17/05/29 18:19:31 INFO SparkContext: Added file file:/root/.ivy2/jars/org.scalatest_scalatest_2.11-2.2.6.jar at spark://192.168.1.20:38293/files/org.scalatest_scalatest_2.11-2.2.6.jar with timestamp 1496053171653 17/05/29 18:19:31 INFO Utils: Copying /root/.ivy2/jars/org.scalatest_scalatest_2.11-2.2.6.jar to /tmp/spark-ff0978ee-1039-4222-b80a-550ef0f885ed/userFiles-1fb0f005-86f4-4515-906d-12e549bcdee3/org.scalatest_scalatest_2.11-2.2.6.jar 17/05/29 18:19:31 INFO SparkContext: Added file file:/root/.ivy2/jars/org.scala-lang_scala-reflect-2.11.8.jar at spark://192.168.1.20:38293/files/org.scala-lang_scala-reflect-2.11.8.jar with timestamp 1496053171665 17/05/29 18:19:31 INFO Utils: Copying /root/.ivy2/jars/org.scala-lang_scala-reflect-2.11.8.jar to /tmp/spark-ff0978ee-1039-4222-b80a-550ef0f885ed/userFiles-1fb0f005-86f4-4515-906d-12e549bcdee3/org.scala-lang_scala-reflect-2.11.8.jar 17/05/29 18:19:31 INFO StandaloneAppClient$ClientEndpoint: Connecting to master spark://host-192-168-1-20.openstacklocal:7077... 17/05/29 18:19:31 INFO TransportClientFactory: Successfully created connection to host-192-168-1-20.openstacklocal/192.168.1.20:7077 after 44 ms (0 ms spent in bootstraps) 17/05/29 18:19:32 INFO StandaloneSchedulerBackend: Connected to Spark cluster with app ID app-20170529181932-0016 17/05/29 18:19:32 INFO StandaloneAppClient$ClientEndpoint: Executor added: app-20170529181932-0016/0 on worker-20170528203111-192.168.1.20-42530 (192.168.1.20:42530) with 16 cores 17/05/29 18:19:32 INFO StandaloneSchedulerBackend: Granted executor ID app-20170529181932-0016/0 on hostPort 192.168.1.20:42530 with 16 cores, 1024.0 MB RAM 17/05/29 18:19:32 INFO Utils: Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 44983. 17/05/29 18:19:32 INFO NettyBlockTransferService: Server created on 192.168.1.20:44983 17/05/29 18:19:32 INFO BlockManager: Using org.apache.spark.storage.RandomBlockReplicationPolicy for block replication policy 17/05/29 18:19:32 INFO BlockManagerMaster: Registering BlockManager BlockManagerId(driver, 192.168.1.20, 44983, None) 17/05/29 18:19:32 INFO BlockManagerMasterEndpoint: Registering block manager 192.168.1.20:44983 with 366.3 MB RAM, BlockManagerId(driver, 192.168.1.20, 44983, None) 17/05/29 18:19:32 INFO StandaloneAppClient$ClientEndpoint: Executor updated: app-20170529181932-0016/0 is now RUNNING 17/05/29 18:19:32 INFO BlockManagerMaster: Registered BlockManager BlockManagerId(driver, 192.168.1.20, 44983, None) 17/05/29 18:19:32 INFO BlockManager: Initialized BlockManager: BlockManagerId(driver, 192.168.1.20, 44983, None) 17/05/29 18:19:32 INFO StandaloneSchedulerBackend: SchedulerBackend is ready for scheduling beginning after reached minRegisteredResourcesRatio: 0.0 <pyspark.streaming.dstream.TransformedDStream object at 0x2ba4910> 17/05/29 18:19:32 INFO SparkContext: Starting job: start at NativeMethodAccessorImpl.java:0 17/05/29 18:19:33 INFO DAGScheduler: Registering RDD 1 (start at NativeMethodAccessorImpl.java:0) 17/05/29 18:19:33 INFO DAGScheduler: Got job 0 (start at NativeMethodAccessorImpl.java:0) with 20 output partitions 17/05/29 18:19:33 INFO DAGScheduler: Final stage: ResultStage 1 (start at NativeMethodAccessorImpl.java:0) 17/05/29 18:19:33 INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 0) 17/05/29 18:19:33 INFO DAGScheduler: Missing parents: List(ShuffleMapStage 0) 17/05/29 18:19:33 INFO DAGScheduler: Submitting ShuffleMapStage 0 (MapPartitionsRDD[1] at start at NativeMethodAccessorImpl.java:0), which has no missing parents 17/05/29 18:19:33 INFO MemoryStore: Block broadcast_0 stored as values in memory (estimated size 3.1 KB, free 366.3 MB) 17/05/29 18:19:33 INFO MemoryStore: Block broadcast_0_piece0 stored as bytes in memory (estimated size 2011.0 B, free 366.3 MB) 17/05/29 18:19:33 INFO BlockManagerInfo: Added broadcast_0_piece0 in memory on 192.168.1.20:44983 (size: 2011.0 B, free: 366.3 MB) 17/05/29 18:19:33 INFO SparkContext: Created broadcast 0 from broadcast at DAGScheduler.scala:996 17/05/29 18:19:33 INFO DAGScheduler: Submitting 50 missing tasks from ShuffleMapStage 0 (MapPartitionsRDD[1] at start at NativeMethodAccessorImpl.java:0) 17/05/29 18:19:33 INFO TaskSchedulerImpl: Adding task set 0.0 with 50 tasks 17/05/29 18:19:34 INFO CoarseGrainedSchedulerBackend$DriverEndpoint: Registered executor NettyRpcEndpointRef(null) (192.168.1.20:38450) with ID 0 17/05/29 18:19:34 INFO TaskSetManager: Starting task 0.0 in stage 0.0 (TID 0, 192.168.1.20, executor 0, partition 0, PROCESS_LOCAL, 9337 bytes) 17/05/29 18:19:34 INFO TaskSetManager: Starting task 1.0 in stage 0.0 (TID 1, 192.168.1.20, executor 0, partition 1, PROCESS_LOCAL, 9337 bytes) 17/05/29 18:19:34 INFO TaskSetManager: Starting task 2.0 in stage 0.0 (TID 2, 192.168.1.20, executor 0, partition 2, PROCESS_LOCAL, 9337 bytes) 17/05/29 18:19:34 INFO TaskSetManager: Starting task 3.0 in stage 0.0 (TID 3, 192.168.1.20, executor 0, partition 3, PROCESS_LOCAL, 9337 bytes) 17/05/29 18:19:34 INFO TaskSetManager: Starting task 4.0 in stage 0.0 (TID 4, 192.168.1.20, executor 0, partition 4, PROCESS_LOCAL, 9337 bytes) 17/05/29 18:19:34 INFO TaskSetManager: Starting task 5.0 in stage 0.0 (TID 5, 192.168.1.20, executor 0, partition 5, PROCESS_LOCAL, 9337 bytes) 17/05/29 18:19:34 INFO TaskSetManager: Starting task 6.0 in stage 0.0 (TID 6, 192.168.1.20, executor 0, partition 6, PROCESS_LOCAL, 9337 bytes) 17/05/29 18:19:34 INFO TaskSetManager: Starting task 7.0 in stage 0.0 (TID 7, 192.168.1.20, executor 0, partition 7, PROCESS_LOCAL, 9337 bytes) 17/05/29 18:19:34 INFO TaskSetManager: Starting task 8.0 in stage 0.0 (TID 8, 192.168.1.20, executor 0, partition 8, PROCESS_LOCAL, 9337 bytes) 17/05/29 18:19:34 INFO TaskSetManager: Starting task 9.0 in stage 0.0 (TID 9, 192.168.1.20, executor 0, partition 9, PROCESS_LOCAL, 9337 bytes) 17/05/29 18:19:34 INFO TaskSetManager: Starting task 10.0 in stage 0.0 (TID 10, 192.168.1.20, executor 0, partition 10, PROCESS_LOCAL, 9337 bytes) 17/05/29 18:19:34 INFO TaskSetManager: Starting task 11.0 in stage 0.0 (TID 11, 192.168.1.20, executor 0, partition 11, PROCESS_LOCAL, 9337 bytes) 17/05/29 18:19:34 INFO TaskSetManager: Starting task 12.0 in stage 0.0 (TID 12, 192.168.1.20, executor 0, partition 12, PROCESS_LOCAL, 9337 bytes) 17/05/29 18:19:34 INFO BlockManagerMasterEndpoint: Registering block manager 192.168.1.20:44197 with 366.3 MB RAM, BlockManagerId(0, 192.168.1.20, 44197, None) 17/05/29 18:19:34 INFO TaskSetManager: Starting task 13.0 in stage 0.0 (TID 13, 192.168.1.20, executor 0, partition 13, PROCESS_LOCAL, 9337 bytes) 17/05/29 18:19:34 INFO TaskSetManager: Starting task 14.0 in stage 0.0 (TID 14, 192.168.1.20, executor 0, partition 14, PROCESS_LOCAL, 9337 bytes) 17/05/29 18:19:34 INFO TaskSetManager: Starting task 15.0 in stage 0.0 (TID 15, 192.168.1.20, executor 0, partition 15, PROCESS_LOCAL, 9337 bytes) 17/05/29 18:19:35 INFO BlockManagerInfo: Added broadcast_0_piece0 in memory on 192.168.1.20:44197 (size: 2011.0 B, free: 366.3 MB) 17/05/29 18:19:35 INFO TaskSetManager: Starting task 16.0 in stage 0.0 (TID 16, 192.168.1.20, executor 0, partition 16, PROCESS_LOCAL, 9337 bytes) 17/05/29 18:19:35 INFO TaskSetManager: Starting task 17.0 in stage 0.0 (TID 17, 192.168.1.20, executor 0, partition 17, PROCESS_LOCAL, 9337 bytes) 17/05/29 18:19:35 INFO TaskSetManager: Starting task 18.0 in stage 0.0 (TID 18, 192.168.1.20, executor 0, partition 18, PROCESS_LOCAL, 9337 bytes) 17/05/29 18:19:35 INFO TaskSetManager: Starting task 19.0 in stage 0.0 (TID 19, 192.168.1.20, executor 0, partition 19, PROCESS_LOCAL, 9337 bytes) 17/05/29 18:19:35 INFO TaskSetManager: Finished task 0.0 in stage 0.0 (TID 0) in 1291 ms on 192.168.1.20 (executor 0) (1/50) 17/05/29 18:19:35 INFO TaskSetManager: Finished task 15.0 in stage 0.0 (TID 15) in 1219 ms on 192.168.1.20 (executor 0) (2/50) 17/05/29 18:19:35 INFO TaskSetManager: Starting task 20.0 in stage 0.0 (TID 20, 192.168.1.20, executor 0, partition 20, PROCESS_LOCAL, 9337 bytes) 17/05/29 18:19:35 INFO TaskSetManager: Finished task 13.0 in stage 0.0 (TID 13) in 1223 ms on 192.168.1.20 (executor 0) (3/50) 17/05/29 18:19:35 INFO TaskSetManager: Finished task 1.0 in stage 0.0 (TID 1) in 1248 ms on 192.168.1.20 (executor 0) (4/50) 17/05/29 18:19:35 INFO TaskSetManager: Starting task 21.0 in stage 0.0 (TID 21, 192.168.1.20, executor 0, partition 21, PROCESS_LOCAL, 9337 bytes) 17/05/29 18:19:35 INFO TaskSetManager: Starting task 22.0 in stage 0.0 (TID 22, 192.168.1.20, executor 0, partition 22, PROCESS_LOCAL, 9337 bytes) 17/05/29 18:19:35 INFO TaskSetManager: Finished task 7.0 in stage 0.0 (TID 7) in 1243 ms on 192.168.1.20 (executor 0) (5/50) 17/05/29 18:19:35 INFO TaskSetManager: Starting task 23.0 in stage 0.0 (TID 23, 192.168.1.20, executor 0, partition 23, PROCESS_LOCAL, 9337 bytes) 17/05/29 18:19:35 INFO TaskSetManager: Finished task 2.0 in stage 0.0 (TID 2) in 1251 ms on 192.168.1.20 (executor 0) (6/50) 17/05/29 18:19:35 INFO TaskSetManager: Starting task 24.0 in stage 0.0 (TID 24, 192.168.1.20, executor 0, partition 24, PROCESS_LOCAL, 9337 bytes) 17/05/29 18:19:35 INFO TaskSetManager: Starting task 25.0 in stage 0.0 (TID 25, 192.168.1.20, executor 0, partition 25, PROCESS_LOCAL, 9337 bytes) 17/05/29 18:19:35 INFO TaskSetManager: Starting task 26.0 in stage 0.0 (TID 26, 192.168.1.20, executor 0, partition 26, PROCESS_LOCAL, 9337 bytes) 17/05/29 18:19:35 INFO TaskSetManager: Starting task 27.0 in stage 0.0 (TID 27, 192.168.1.20, executor 0, partition 27, PROCESS_LOCAL, 9337 bytes) 17/05/29 18:19:35 INFO TaskSetManager: Starting task 28.0 in stage 0.0 (TID 28, 192.168.1.20, executor 0, partition 28, PROCESS_LOCAL, 9337 bytes) 17/05/29 18:19:35 INFO TaskSetManager: Finished task 12.0 in stage 0.0 (TID 12) in 1243 ms on 192.168.1.20 (executor 0) (7/50) 17/05/29 18:19:35 INFO TaskSetManager: Finished task 3.0 in stage 0.0 (TID 3) in 1260 ms on 192.168.1.20 (executor 0) (8/50) 17/05/29 18:19:35 INFO TaskSetManager: Finished task 8.0 in stage 0.0 (TID 8) in 1253 ms on 192.168.1.20 (executor 0) (9/50) 17/05/29 18:19:35 INFO TaskSetManager: Finished task 10.0 in stage 0.0 (TID 10) in 1252 ms on 192.168.1.20 (executor 0) (10/50) 17/05/29 18:19:35 INFO TaskSetManager: Finished task 4.0 in stage 0.0 (TID 4) in 1260 ms on 192.168.1.20 (executor 0) (11/50) 17/05/29 18:19:35 INFO TaskSetManager: Finished task 9.0 in stage 0.0 (TID 9) in 1253 ms on 192.168.1.20 (executor 0) (12/50) 17/05/29 18:19:35 INFO TaskSetManager: Starting task 29.0 in stage 0.0 (TID 29, 192.168.1.20, executor 0, partition 29, PROCESS_LOCAL, 9337 bytes) 17/05/29 18:19:35 INFO TaskSetManager: Starting task 30.0 in stage 0.0 (TID 30, 192.168.1.20, executor 0, partition 30, PROCESS_LOCAL, 9337 bytes) 17/05/29 18:19:35 INFO TaskSetManager: Starting task 31.0 in stage 0.0 (TID 31, 192.168.1.20, executor 0, partition 31, PROCESS_LOCAL, 9337 bytes) 17/05/29 18:19:35 INFO TaskSetManager: Starting task 32.0 in stage 0.0 (TID 32, 192.168.1.20, executor 0, partition 32, PROCESS_LOCAL, 9337 bytes) 17/05/29 18:19:35 INFO TaskSetManager: Starting task 33.0 in stage 0.0 (TID 33, 192.168.1.20, executor 0, partition 33, PROCESS_LOCAL, 9337 bytes) 17/05/29 18:19:35 INFO TaskSetManager: Starting task 34.0 in stage 0.0 (TID 34, 192.168.1.20, executor 0, partition 34, PROCESS_LOCAL, 9337 bytes) 17/05/29 18:19:35 INFO TaskSetManager: Starting task 35.0 in stage 0.0 (TID 35, 192.168.1.20, executor 0, partition 35, PROCESS_LOCAL, 9337 bytes) 17/05/29 18:19:35 INFO TaskSetManager: Finished task 11.0 in stage 0.0 (TID 11) in 1277 ms on 192.168.1.20 (executor 0) (13/50) 17/05/29 18:19:35 INFO TaskSetManager: Starting task 36.0 in stage 0.0 (TID 36, 192.168.1.20, executor 0, partition 36, PROCESS_LOCAL, 9337 bytes) 17/05/29 18:19:35 INFO TaskSetManager: Starting task 37.0 in stage 0.0 (TID 37, 192.168.1.20, executor 0, partition 37, PROCESS_LOCAL, 9337 bytes) 17/05/29 18:19:35 INFO TaskSetManager: Finished task 5.0 in stage 0.0 (TID 5) in 1296 ms on 192.168.1.20 (executor 0) (14/50) 17/05/29 18:19:35 INFO TaskSetManager: Finished task 14.0 in stage 0.0 (TID 14) in 1276 ms on 192.168.1.20 (executor 0) (15/50) 17/05/29 18:19:35 INFO TaskSetManager: Starting task 38.0 in stage 0.0 (TID 38, 192.168.1.20, executor 0, partition 38, PROCESS_LOCAL, 9337 bytes) 17/05/29 18:19:35 INFO TaskSetManager: Starting task 39.0 in stage 0.0 (TID 39, 192.168.1.20, executor 0, partition 39, PROCESS_LOCAL, 9337 bytes) 17/05/29 18:19:35 INFO TaskSetManager: Starting task 40.0 in stage 0.0 (TID 40, 192.168.1.20, executor 0, partition 40, PROCESS_LOCAL, 9337 bytes) 17/05/29 18:19:35 INFO TaskSetManager: Starting task 41.0 in stage 0.0 (TID 41, 192.168.1.20, executor 0, partition 41, PROCESS_LOCAL, 9337 bytes) 17/05/29 18:19:35 INFO TaskSetManager: Starting task 42.0 in stage 0.0 (TID 42, 192.168.1.20, executor 0, partition 42, PROCESS_LOCAL, 9337 bytes) 17/05/29 18:19:35 INFO TaskSetManager: Starting task 43.0 in stage 0.0 (TID 43, 192.168.1.20, executor 0, partition 43, PROCESS_LOCAL, 9337 bytes) 17/05/29 18:19:35 INFO TaskSetManager: Starting task 44.0 in stage 0.0 (TID 44, 192.168.1.20, executor 0, partition 44, PROCESS_LOCAL, 9337 bytes) 17/05/29 18:19:35 INFO TaskSetManager: Starting task 45.0 in stage 0.0 (TID 45, 192.168.1.20, executor 0, partition 45, PROCESS_LOCAL, 9337 bytes) 17/05/29 18:19:35 INFO TaskSetManager: Starting task 46.0 in stage 0.0 (TID 46, 192.168.1.20, executor 0, partition 46, PROCESS_LOCAL, 9337 bytes) 17/05/29 18:19:35 INFO TaskSetManager: Starting task 47.0 in stage 0.0 (TID 47, 192.168.1.20, executor 0, partition 47, PROCESS_LOCAL, 9337 bytes) 17/05/29 18:19:35 INFO TaskSetManager: Starting task 48.0 in stage 0.0 (TID 48, 192.168.1.20, executor 0, partition 48, PROCESS_LOCAL, 9337 bytes) 17/05/29 18:19:35 INFO TaskSetManager: Starting task 49.0 in stage 0.0 (TID 49, 192.168.1.20, executor 0, partition 49, PROCESS_LOCAL, 9394 bytes) 17/05/29 18:19:35 INFO TaskSetManager: Finished task 19.0 in stage 0.0 (TID 19) in 132 ms on 192.168.1.20 (executor 0) (16/50) 17/05/29 18:19:35 INFO TaskSetManager: Finished task 17.0 in stage 0.0 (TID 17) in 141 ms on 192.168.1.20 (executor 0) (17/50) 17/05/29 18:19:35 INFO TaskSetManager: Finished task 6.0 in stage 0.0 (TID 6) in 1366 ms on 192.168.1.20 (executor 0) (18/50) 17/05/29 18:19:35 INFO TaskSetManager: Finished task 23.0 in stage 0.0 (TID 23) in 123 ms on 192.168.1.20 (executor 0) (19/50) 17/05/29 18:19:35 INFO TaskSetManager: Finished task 25.0 in stage 0.0 (TID 25) in 120 ms on 192.168.1.20 (executor 0) (20/50) 17/05/29 18:19:35 INFO TaskSetManager: Finished task 16.0 in stage 0.0 (TID 16) in 146 ms on 192.168.1.20 (executor 0) (21/50) 17/05/29 18:19:35 INFO TaskSetManager: Finished task 29.0 in stage 0.0 (TID 29) in 110 ms on 192.168.1.20 (executor 0) (22/50) 17/05/29 18:19:35 INFO TaskSetManager: Finished task 22.0 in stage 0.0 (TID 22) in 125 ms on 192.168.1.20 (executor 0) (23/50) 17/05/29 18:19:35 INFO TaskSetManager: Finished task 21.0 in stage 0.0 (TID 21) in 128 ms on 192.168.1.20 (executor 0) (24/50) 17/05/29 18:19:35 INFO TaskSetManager: Finished task 20.0 in stage 0.0 (TID 20) in 130 ms on 192.168.1.20 (executor 0) (25/50) 17/05/29 18:19:35 INFO TaskSetManager: Finished task 30.0 in stage 0.0 (TID 30) in 110 ms on 192.168.1.20 (executor 0) (26/50) 17/05/29 18:19:35 INFO TaskSetManager: Finished task 18.0 in stage 0.0 (TID 18) in 138 ms on 192.168.1.20 (executor 0) (27/50) 17/05/29 18:19:35 INFO TaskSetManager: Finished task 32.0 in stage 0.0 (TID 32) in 108 ms on 192.168.1.20 (executor 0) (28/50) 17/05/29 18:19:35 INFO TaskSetManager: Finished task 33.0 in stage 0.0 (TID 33) in 108 ms on 192.168.1.20 (executor 0) (29/50) 17/05/29 18:19:35 INFO TaskSetManager: Finished task 26.0 in stage 0.0 (TID 26) in 120 ms on 192.168.1.20 (executor 0) (30/50) 17/05/29 18:19:35 INFO TaskSetManager: Finished task 31.0 in stage 0.0 (TID 31) in 111 ms on 192.168.1.20 (executor 0) (31/50) 17/05/29 18:19:35 INFO TaskSetManager: Finished task 34.0 in stage 0.0 (TID 34) in 103 ms on 192.168.1.20 (executor 0) (32/50) 17/05/29 18:19:35 INFO TaskSetManager: Finished task 24.0 in stage 0.0 (TID 24) in 128 ms on 192.168.1.20 (executor 0) (33/50) 17/05/29 18:19:35 INFO TaskSetManager: Finished task 28.0 in stage 0.0 (TID 28) in 120 ms on 192.168.1.20 (executor 0) (34/50) 17/05/29 18:19:35 INFO TaskSetManager: Finished task 37.0 in stage 0.0 (TID 37) in 83 ms on 192.168.1.20 (executor 0) (35/50) 17/05/29 18:19:35 INFO TaskSetManager: Finished task 27.0 in stage 0.0 (TID 27) in 122 ms on 192.168.1.20 (executor 0) (36/50) 17/05/29 18:19:35 INFO TaskSetManager: Finished task 35.0 in stage 0.0 (TID 35) in 91 ms on 192.168.1.20 (executor 0) (37/50) 17/05/29 18:19:35 INFO TaskSetManager: Finished task 36.0 in stage 0.0 (TID 36) in 110 ms on 192.168.1.20 (executor 0) (38/50) 17/05/29 18:19:35 INFO TaskSetManager: Finished task 39.0 in stage 0.0 (TID 39) in 104 ms on 192.168.1.20 (executor 0) (39/50) 17/05/29 18:19:35 INFO TaskSetManager: Finished task 40.0 in stage 0.0 (TID 40) in 97 ms on 192.168.1.20 (executor 0) (40/50) 17/05/29 18:19:35 INFO TaskSetManager: Finished task 38.0 in stage 0.0 (TID 38) in 109 ms on 192.168.1.20 (executor 0) (41/50) 17/05/29 18:19:35 INFO TaskSetManager: Finished task 41.0 in stage 0.0 (TID 41) in 98 ms on 192.168.1.20 (executor 0) (42/50) 17/05/29 18:19:35 INFO TaskSetManager: Finished task 45.0 in stage 0.0 (TID 45) in 97 ms on 192.168.1.20 (executor 0) (43/50) 17/05/29 18:19:35 INFO TaskSetManager: Finished task 47.0 in stage 0.0 (TID 47) in 98 ms on 192.168.1.20 (executor 0) (44/50) 17/05/29 18:19:35 INFO TaskSetManager: Finished task 42.0 in stage 0.0 (TID 42) in 105 ms on 192.168.1.20 (executor 0) (45/50) 17/05/29 18:19:35 INFO TaskSetManager: Finished task 46.0 in stage 0.0 (TID 46) in 101 ms on 192.168.1.20 (executor 0) (46/50) 17/05/29 18:19:35 INFO TaskSetManager: Finished task 44.0 in stage 0.0 (TID 44) in 105 ms on 192.168.1.20 (executor 0) (47/50) 17/05/29 18:19:35 INFO TaskSetManager: Finished task 43.0 in stage 0.0 (TID 43) in 105 ms on 192.168.1.20 (executor 0) (48/50) 17/05/29 18:19:35 INFO TaskSetManager: Finished task 49.0 in stage 0.0 (TID 49) in 84 ms on 192.168.1.20 (executor 0) (49/50) 17/05/29 18:19:35 INFO TaskSetManager: Finished task 48.0 in stage 0.0 (TID 48) in 117 ms on 192.168.1.20 (executor 0) (50/50) 17/05/29 18:19:35 INFO TaskSchedulerImpl: Removed TaskSet 0.0, whose tasks have all completed, from pool 17/05/29 18:19:36 INFO DAGScheduler: ShuffleMapStage 0 (start at NativeMethodAccessorImpl.java:0) finished in 2.766 s 17/05/29 18:19:36 INFO DAGScheduler: looking for newly runnable stages 17/05/29 18:19:36 INFO DAGScheduler: running: Set() 17/05/29 18:19:36 INFO DAGScheduler: waiting: Set(ResultStage 1) 17/05/29 18:19:36 INFO DAGScheduler: failed: Set()