21/01/25 10:31:20 WARN scheduler.TaskSetManager: Lost task 29.0 in stage 2.0 (TID 7, 192.168.100.72, executor 0): java.lang.NullPointerException
at com.vesoft.nebula.client.meta.MetaManager.fillMetaInfo(MetaManager.java:93)
at com.vesoft.nebula.client.meta.MetaManager.getSpace(MetaManager.java:162)
at com.vesoft.nebula.encoder.NebulaCodecImpl.getSpaceVidLen(NebulaCodecImpl.java:54)
at com.vesoft.nebula.encoder.NebulaCodecImpl.vertexKey(NebulaCodecImpl.java:75)
at com.vesoft.nebula.exchange.processor.VerticesProcessor$$anonfun$process$1$$anonfun$apply$1.apply(VerticesProcessor.scala:142)
at com.vesoft.nebula.exchange.processor.VerticesProcessor$$anonfun$process$1$$anonfun$apply$1.apply(VerticesProcessor.scala:118)
at scala.collection.Iterator$$anon$11.next(Iterator.scala:410)
at scala.collection.Iterator$$anon$11.next(Iterator.scala:410)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage2.sort_addToSorter_0$(Unknown Source)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage2.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenExec$$anonfun$13$$anon$1.hasNext(WholeStageCodegenExec.scala:636)
at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:409)
at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:409)
at scala.collection.Iterator$class.foreach(Iterator.scala:891)
at scala.collection.AbstractIterator.foreach(Iterator.scala:1334)
at com.vesoft.nebula.exchange.processor.VerticesProcessor$$anonfun$process$2.apply(VerticesProcessor.scala:161)
at com.vesoft.nebula.exchange.processor.VerticesProcessor$$anonfun$process$2.apply(VerticesProcessor.scala:154)
at org.apache.spark.rdd.RDD$$anonfun$foreachPartition$1$$anonfun$apply$28.apply(RDD.scala:980)
at org.apache.spark.rdd.RDD$$anonfun$foreachPartition$1$$anonfun$apply$28.apply(RDD.scala:980)
at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:2101)
at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:2101)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)
at org.apache.spark.scheduler.Task.run(Task.scala:123)
at org.apache.spark.executor.Executor$TaskRunner$$anonfun$10.apply(Executor.scala:408)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1360)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:414)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
(base) [consumer@localhost nebula-exchange]$ git log
commit f00e94c5389eeff68529ce19027f659216c65c15
Merge: 413bfef 87ebcc5
Author: jude-zhu <51590253+jude-zhu@users.noreply.github.com>
Date: Fri Jan 22 16:32:05 2021 +0800
Merge pull request #19 from Nicole00/rc1
support data's field both as VID and property with different dataType
commit 87ebcc58588d9e83d014379cffded727bfd507a8
Author: Nicole00 <16240361+Nicole00@users.noreply.github.com>
Date: Fri Jan 22 16:25:27 2021 +0800
revert remote path
commit 85d726545d6e00c609794075f84323391aae072f
Author: Nicole00 <16240361+Nicole00@users.noreply.github.com>
Date: Fri Jan 15 16:36:07 2021 +0800
support data both as VID and property with different dataType
21/01/25 15:14:55 WARN scheduler.TaskSetManager: Lost task 10.0 in stage 9.0 (TID 112, 192.168.100.72, executor 0): java.io.IOException: Failed on local exception: java.io.IOException; Host Details : local host is: "localhost/127.0.0.1"; destination host is: "localhost":9000;
at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:776)
at org.apache.hadoop.ipc.Client.call(Client.java:1479)
at org.apache.hadoop.ipc.Client.call(Client.java:1412)
at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:229)
at com.sun.proxy.$Proxy20.complete(Unknown Source)
at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:462)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:191)
at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:102)
at com.sun.proxy.$Proxy21.complete(Unknown Source)
at org.apache.hadoop.hdfs.DFSOutputStream.completeFile(DFSOutputStream.java:2291)
at org.apache.hadoop.hdfs.DFSOutputStream.closeImpl(DFSOutputStream.java:2267)
at org.apache.hadoop.hdfs.DFSOutputStream.close(DFSOutputStream.java:2232)
at org.apache.hadoop.fs.FSDataOutputStream$PositionCache.close(FSDataOutputStream.java:72)
at org.apache.hadoop.fs.FSDataOutputStream.close(FSDataOutputStream.java:106)
at com.vesoft.nebula.exchange.ErrorHandler$.save(ErrorHandler.scala:24)
at com.vesoft.nebula.exchange.processor.ReloadProcessor.com$vesoft$nebula$exchange$processor$ReloadProcessor$$processEachPartition(ReloadProcessor.scala:47)
at com.vesoft.nebula.exchange.processor.ReloadProcessor$$anonfun$process$1.apply(ReloadProcessor.scala:24)
at com.vesoft.nebula.exchange.processor.ReloadProcessor$$anonfun$process$1.apply(ReloadProcessor.scala:24)
at org.apache.spark.rdd.RDD$$anonfun$foreachPartition$1$$anonfun$apply$28.apply(RDD.scala:980)
at org.apache.spark.rdd.RDD$$anonfun$foreachPartition$1$$anonfun$apply$28.apply(RDD.scala:980)
at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:2101)
at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:2101)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)
at org.apache.spark.scheduler.Task.run(Task.scala:123)
at org.apache.spark.executor.Executor$TaskRunner$$anonfun$10.apply(Executor.scala:408)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1360)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:414)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.io.IOException
at org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:946)
at org.apache.hadoop.ipc.Client$Connection.run(Client.java:978)
Caused by: java.lang.InterruptedException
... 2 more
21/01/25 15:14:55 INFO scheduler.TaskSetManager: Starting task 10.1 in stage 9.0 (TID 114, 192.168.100.72, executor 0, partition 10, PROCESS_LOCAL, 8334 bytes)
21/01/25 15:14:55 WARN scheduler.TaskSetManager: Lost task 0.0 in stage 9.0 (TID 102, 192.168.100.72, executor 0): org.apache.spark.util.TaskCompletionListenerException: Filesystem closed
Previous exception in task: Filesystem closed
org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:808)
org.apache.hadoop.hdfs.DFSInputStream.readWithStrategy(DFSInputStream.java:868)
org.apache.hadoop.hdfs.DFSInputStream.read(DFSInputStream.java:934)
java.io.DataInputStream.read(DataInputStream.java:149)
org.apache.hadoop.mapreduce.lib.input.UncompressedSplitLineReader.fillBuffer(UncompressedSplitLineReader.java:62)
org.apache.hadoop.util.LineReader.readDefaultLine(LineReader.java:216)
org.apache.hadoop.util.LineReader.readLine(LineReader.java:174)
org.apache.hadoop.mapreduce.lib.input.UncompressedSplitLineReader.readLine(UncompressedSplitLineReader.java:94)
org.apache.hadoop.mapreduce.lib.input.LineRecordReader.nextKeyValue(LineRecordReader.java:186)
org.apache.spark.sql.execution.datasources.RecordReaderIterator.hasNext(RecordReaderIterator.scala:39)
org.apache.spark.sql.execution.datasources.HadoopFileLinesReader.hasNext(HadoopFileLinesReader.scala:69)
scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:409)
scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:409)
org.apache.spark.sql.execution.datasources.FileScanRDD$$anon$1.hasNext(FileScanRDD.scala:101)
org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
org.apache.spark.sql.execution.WholeStageCodegenExec$$anonfun$13$$anon$1.hasNext(WholeStageCodegenExec.scala:636)
scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:409)
scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:409)
scala.collection.Iterator$class.foreach(Iterator.scala:891)
scala.collection.AbstractIterator.foreach(Iterator.scala:1334)
com.vesoft.nebula.exchange.processor.ReloadProcessor.com$vesoft$nebula$exchange$processor$ReloadProcessor$$processEachPartition(ReloadProcessor.scala:36)
com.vesoft.nebula.exchange.processor.ReloadProcessor$$anonfun$process$1.apply(ReloadProcessor.scala:24)
com.vesoft.nebula.exchange.processor.ReloadProcessor$$anonfun$process$1.apply(ReloadProcessor.scala:24)
org.apache.spark.rdd.RDD$$anonfun$foreachPartition$1$$anonfun$apply$28.apply(RDD.scala:980)
org.apache.spark.rdd.RDD$$anonfun$foreachPartition$1$$anonfun$apply$28.apply(RDD.scala:980)
org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:2101)
org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:2101)
org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)
org.apache.spark.scheduler.Task.run(Task.scala:123)
org.apache.spark.executor.Executor$TaskRunner$$anonfun$10.apply(Executor.scala:408)
org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1360)
org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:414)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
java.lang.Thread.run(Thread.java:748)
at org.apache.spark.TaskContextImpl.invokeListeners(TaskContextImpl.scala:138)
at org.apache.spark.TaskContextImpl.markTaskCompleted(TaskContextImpl.scala:116)
at org.apache.spark.scheduler.Task.run(Task.scala:133)
at org.apache.spark.executor.Executor$TaskRunner$$anonfun$10.apply(Executor.scala:408)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1360)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:414)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
21/01/25 15:14:55 INFO scheduler.TaskSetManager: Lost task 7.0 in stage 9.0 (TID 109) on 192.168.100.72, executor 0: java.io.IOException (Failed on local exception: java.io.IOException; Host Details : local host is: "localhost/127.0.0.1"; destination host is: "localhost":9000; ) [duplicate 1]
21/01/25 15:14:55 INFO scheduler.TaskSetManager: Starting task 7.1 in stage 9.0 (TID 115, 192.168.100.72, executor 0, partition 7, PROCESS_LOCAL, 8334 bytes)
21/01/25 15:14:55 INFO scheduler.TaskSetManager: Starting task 0.1 in stage 9.0 (TID 116, 192.168.100.72, executor 0, partition 0, PROCESS_LOCAL, 8332 bytes)
21/01/25 15:14:55 INFO scheduler.TaskSetManager: Finished task 8.0 in stage 9.0 (TID 110) in 352 ms on 192.168.100.72 (executor 0) (1/12)
21/01/25 15:14:55 INFO scheduler.TaskSetManager: Finished task 11.0 in stage 9.0 (TID 113) in 352 ms on 192.168.100.72 (executor 0) (2/12)
21/01/25 15:14:55 INFO scheduler.TaskSetManager: Finished task 5.0 in stage 9.0 (TID 107) in 380 ms on 192.168.100.72 (executor 0) (3/12)
21/01/25 15:14:55 INFO scheduler.TaskSetManager: Finished task 1.0 in stage 9.0 (TID 103) in 382 ms on 192.168.100.72 (executor 0) (4/12)
21/01/25 15:14:55 INFO scheduler.TaskSetManager: Finished task 9.0 in stage 9.0 (TID 111) in 381 ms on 192.168.100.72 (executor 0) (5/12)
21/01/25 15:14:55 INFO scheduler.TaskSetManager: Finished task 4.0 in stage 9.0 (TID 106) in 382 ms on 192.168.100.72 (executor 0) (6/12)
21/01/25 15:14:55 INFO scheduler.TaskSetManager: Finished task 3.0 in stage 9.0 (TID 105) in 383 ms on 192.168.100.72 (executor 0) (7/12)
21/01/25 15:14:55 INFO scheduler.TaskSetManager: Finished task 6.0 in stage 9.0 (TID 108) in 383 ms on 192.168.100.72 (executor 0) (8/12)
21/01/25 15:14:55 INFO scheduler.TaskSetManager: Finished task 2.0 in stage 9.0 (TID 104) in 384 ms on 192.168.100.72 (executor 0) (9/12)
21/01/25 15:14:55 INFO scheduler.TaskSetManager: Lost task 0.1 in stage 9.0 (TID 116) on 192.168.100.72, executor 0: org.apache.spark.util.TaskCompletionListenerException (Filesystem closed
Previous exception in task: Filesystem closed
org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:808)
org.apache.hadoop.hdfs.DFSInputStream.readWithStrategy(DFSInputStream.java:868)
org.apache.hadoop.hdfs.DFSInputStream.read(DFSInputStream.java:934)
java.io.DataInputStream.read(DataInputStream.java:149)
org.apache.hadoop.mapreduce.lib.input.UncompressedSplitLineReader.fillBuffer(UncompressedSplitLineReader.java:62)
org.apache.hadoop.util.LineReader.readDefaultLine(LineReader.java:216)
org.apache.hadoop.util.LineReader.readLine(LineReader.java:174)
org.apache.hadoop.mapreduce.lib.input.UncompressedSplitLineReader.readLine(UncompressedSplitLineReader.java:94)
org.apache.hadoop.mapreduce.lib.input.LineRecordReader.nextKeyValue(LineRecordReader.java:186)
org.apache.spark.sql.execution.datasources.RecordReaderIterator.hasNext(RecordReaderIterator.scala:39)
org.apache.spark.sql.execution.datasources.HadoopFileLinesReader.hasNext(HadoopFileLinesReader.scala:69)
scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:409)
scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:409)
org.apache.spark.sql.execution.datasources.FileScanRDD$$anon$1.hasNext(FileScanRDD.scala:101)
org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
org.apache.spark.sql.execution.WholeStageCodegenExec$$anonfun$13$$anon$1.hasNext(WholeStageCodegenExec.scala:636)
scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:409)
scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:409)
scala.collection.Iterator$class.foreach(Iterator.scala:891)
scala.collection.AbstractIterator.foreach(Iterator.scala:1334)
com.vesoft.nebula.exchange.processor.ReloadProcessor.com$vesoft$nebula$exchange$processor$ReloadProcessor$$processEachPartition(ReloadProcessor.scala:36)
com.vesoft.nebula.exchange.processor.ReloadProcessor$$anonfun$process$1.apply(ReloadProcessor.scala:24)
com.vesoft.nebula.exchange.processor.ReloadProcessor$$anonfun$process$1.apply(ReloadProcessor.scala:24)
org.apache.spark.rdd.RDD$$anonfun$foreachPartition$1$$anonfun$apply$28.apply(RDD.scala:980)
org.apache.spark.rdd.RDD$$anonfun$foreachPartition$1$$anonfun$apply$28.apply(RDD.scala:980)
org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:2101)
org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:2101)
org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)
org.apache.spark.scheduler.Task.run(Task.scala:123)
org.apache.spark.executor.Executor$TaskRunner$$anonfun$10.apply(Executor.scala:408)
org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1360)
org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:414)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
java.lang.Thread.run(Thread.java:748)) [duplicate 1]
21/01/25 15:14:55 INFO scheduler.TaskSetManager: Starting task 0.2 in stage 9.0 (TID 117, 192.168.100.72, executor 0, partition 0, PROCESS_LOCAL, 8332 bytes)