- nebula 版本：2.6.1
- 部署方式：单机
- 安装方式： RPM
- 是否为线上版本：N
- 硬件信息
- 磁盘（ 推荐使用 SSD）
- CPU、内存信息
spark version 2.5
主要出错位置在这一句：
Exception in thread “main” org.apache.commons.math3.ode.UnknownParameterException: unknown parameter {0}
应该是参数的问题，但是不知道哪里有问题
编译nebula-algorithm后运行出现如下报错：
21/11/24 11:20:03 INFO algorithm.Main$: configs = Configs(Map(spark.app.name - LPA, spark.app.partitionNum - 100, spark.master - local),DataSourceEntry: {source:nebula, sink:nebula, hasWeight:false},NebulaConfigEntry:{NebulaReadConfigEntry: {address: 10.154.109.68:9559, space: basketball, labels: serve, weightCols: start_year}, NebulaWriteConfigEntry: {graphAddress: 10.154.109.68:9669, user: root, password: password, space: basketball, tag: pagerank},LocalConfigEntry: {filePath: file:///tmp/algo_edge.csv, srcId: _c0, dstId: _c1, weight:null, resultPath:/tmp/count, delimiter:,},Map(algorithm.pagerank.resetProb - 0.15, algorithm.louvain.tol - 0.5, algorithm.labelpropagation.maxIter - 20, algorithm.louvain.internalIter - 10, algorithm.kcore.maxIter - 10, algorithm.betweenness.maxIter - 5, algorithm.louvain.maxIter - 20, algorithm.shortestpaths.landmarks - 1, algorithm.executeAlgo - graphtrianglecount, algorithm.kcore.degree - 1, algorithm.pagerank.maxIter - 10, algorithm.connectedcomponent.maxIter - 20))
21/11/24 11:20:03 INFO algorithm.Main$: algoName= graphtrianglecount
21/11/24 11:20:03 INFO spark.SparkContext: Running Spark version 2.4.0-cdh6.2.0
21/11/24 11:20:03 INFO logging.DriverLogger: Added a local log appender at: /tmp/spark-4f2b9897-ec78-4740-9572-55f23b97bf6d/__driver_logs__/driver.log
21/11/24 11:20:03 INFO spark.SparkContext: Submitted application: LPA
21/11/24 11:20:03 INFO spark.SecurityManager: Changing view acls to: useradmin
21/11/24 11:20:03 INFO spark.SecurityManager: Changing modify acls to: useradmin
21/11/24 11:20:03 INFO spark.SecurityManager: Changing view acls groups to:
21/11/24 11:20:03 INFO spark.SecurityManager: Changing modify acls groups to:
21/11/24 11:20:03 INFO spark.SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(useradmin); groups with view permissions: Set(); users with modify permissions: Set(useradmin); groups with modify permissions: Set()
21/11/24 11:20:04 INFO util.Utils: Successfully started service 'sparkDriver' on port 35954.
21/11/24 11:20:04 INFO spark.SparkEnv: Registering MapOutputTracker
21/11/24 11:20:04 INFO spark.SparkEnv: Registering BlockManagerMaster
21/11/24 11:20:04 INFO storage.BlockManagerMasterEndpoint: Using org.apache.spark.storage.DefaultTopologyMapper for getting topology information
21/11/24 11:20:04 INFO storage.BlockManagerMasterEndpoint: BlockManagerMasterEndpoint up
21/11/24 11:20:04 INFO storage.DiskBlockManager: Created local directory at /tmp/blockmgr-4bf8fdc0-f3f1-4def-8ac8-b72324d888a8
21/11/24 11:20:04 INFO memory.MemoryStore: MemoryStore started with capacity 366.3 MB
21/11/24 11:20:04 INFO spark.SparkEnv: Registering OutputCommitCoordinator
21/11/24 11:20:04 INFO util.log: Logging initialized @2010ms
21/11/24 11:20:04 INFO server.Server: jetty-9.3.z-SNAPSHOT, build timestamp: 2018-09-05T05:11:46+08:00, git hash: 3ce520221d0240229c862b122d2b06c12a625732
21/11/24 11:20:04 INFO server.Server: Started @2113ms
21/11/24 11:20:04 INFO server.AbstractConnector: Started ServerConnector@4d4d48a6{HTTP/1.1,[http/1.1]}{0.0.0.0:4040}
21/11/24 11:20:04 INFO util.Utils: Successfully started service 'SparkUI' on port 4040.
21/11/24 11:20:04 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@492fc69e{/jobs,null,AVAILABLE, }
21/11/24 11:20:04 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@3b718392{/jobs/json,null,AVAILABLE, }
21/11/24 11:20:04 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@6d2260db{/jobs/job,null,AVAILABLE, }
21/11/24 11:20:04 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@7ee55e70{/jobs/job/json,null,AVAILABLE, }
21/11/24 11:20:04 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@3fcdcf{/stages,null,AVAILABLE, }
21/11/24 11:20:04 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@7668d560{/stages/json,null,AVAILABLE, }
21/11/24 11:20:04 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@46292372{/stages/stage,null,AVAILABLE, }
21/11/24 11:20:04 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@5c371e13{/stages/stage/json,null,AVAILABLE, }
21/11/24 11:20:04 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@530a8454{/stages/pool,null,AVAILABLE, }
21/11/24 11:20:04 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@1e34c607{/stages/pool/json,null,AVAILABLE, }
21/11/24 11:20:04 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@5215cd9a{/storage,null,AVAILABLE, }
21/11/24 11:20:04 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@36b6964d{/storage/json,null,AVAILABLE, }
21/11/24 11:20:04 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@31198ceb{/storage/rdd,null,AVAILABLE, }
21/11/24 11:20:04 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@9257031{/storage/rdd/json,null,AVAILABLE, }
21/11/24 11:20:04 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@75201592{/environment,null,AVAILABLE, }
21/11/24 11:20:04 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@7726e185{/environment/json,null,AVAILABLE, }
21/11/24 11:20:04 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@aa5455e{/executors,null,AVAILABLE, }
21/11/24 11:20:04 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@282308c3{/executors/json,null,AVAILABLE, }
21/11/24 11:20:04 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@5dda14d0{/executors/threadDump,null,AVAILABLE, }
21/11/24 11:20:04 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@1db0ec27{/executors/threadDump/json,null,AVAILABLE, }
21/11/24 11:20:04 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@3d9fc57a{/static,null,AVAILABLE, }
21/11/24 11:20:04 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@38d5b107{/,null,AVAILABLE, }
21/11/24 11:20:04 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@6650813a{/api,null,AVAILABLE, }
21/11/24 11:20:04 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@273c947f{/jobs/job/kill,null,AVAILABLE, }
21/11/24 11:20:04 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@30457e14{/stages/stage/kill,null,AVAILABLE, }
21/11/24 11:20:04 INFO ui.SparkUI: Bound SparkUI to 0.0.0.0, and started
21/11/24 11:20:04 INFO spark.SparkContext: Added JAR file:/home/useradmin/nebula-spark-utils/nebula-spark-utils-2.5/nebula-algorithm/./target/nebula-algorithm-2.5.0.jar at spark://t3-nlprdp-idss-web05:35954/jars/nebula-algorithm-2.5.0.jar with timestamp 1637724004615
21/11/24 11:20:04 INFO executor.Executor: Starting executor ID driver on host localhost
21/11/24 11:20:04 INFO util.Utils: Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 42026.
21/11/24 11:20:04 INFO netty.NettyBlockTransferService: Server created on t3-nlprdp-idss-web05:42026
21/11/24 11:20:04 INFO storage.BlockManager: Using org.apache.spark.storage.RandomBlockReplicationPolicy for block replication policy
21/11/24 11:20:04 INFO storage.BlockManagerMaster: Registering BlockManager BlockManagerId(driver, t3-nlprdp-idss-web05, 42026, None)
21/11/24 11:20:04 INFO storage.BlockManagerMasterEndpoint: Registering block manager t3-nlprdp-idss-web05:42026 with 366.3 MB RAM, BlockManagerId(driver, t3-nlprdp-idss-web05, 42026, None)
21/11/24 11:20:04 INFO storage.BlockManagerMaster: Registered BlockManager BlockManagerId(driver, t3-nlprdp-idss-web05, 42026, None)
21/11/24 11:20:04 INFO storage.BlockManager: external shuffle service port = 7337
21/11/24 11:20:04 INFO storage.BlockManager: Initialized BlockManager: BlockManagerId(driver, t3-nlprdp-idss-web05, 42026, None)
21/11/24 11:20:04 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@16a9a4f1{/metrics/json,null,AVAILABLE, }
21/11/24 11:20:05 INFO scheduler.EventLoggingListener: Logging events to hdfs://t3-nlprdp-idss-web01:8020/user/spark/applicationHistory/local-1637724004659
21/11/24 11:20:05 WARN lineage.LineageWriter: Lineage directory /var/log/spark/lineage doesn't exist or is not writable. Lineage for this application will be disabled.
21/11/24 11:20:05 INFO util.Utils: Extension com.cloudera.spark.lineage.NavigatorAppListener not being initialized.
21/11/24 11:20:05 INFO logging.DriverLogger$DfsAsyncWriter: Started driver log file sync to: /user/spark/driverLogs/local-1637724004659_driver.log
21/11/24 11:20:06 WARN connector.ReadNebulaConfig$: returnCols is empty and your result will contain all properties for serve
21/11/24 11:20:06 INFO connector.ReadNebulaConfig$: NebulaReadConfig={space=basketball,label=serve,returnCols=List(),noColumn=false,partitionNum=100}
21/11/24 11:20:06 INFO internal.SharedState: loading hive config file: file:/etc/hive/conf.cloudera.hive/hive-site.xml
21/11/24 11:20:06 INFO internal.SharedState: spark.sql.warehouse.dir is not set, but hive.metastore.warehouse.dir is set. Setting spark.sql.warehouse.dir to the value of hive.metastore.warehouse.dir ('/user/hive/warehouse').
21/11/24 11:20:06 INFO internal.SharedState: Warehouse path is '/user/hive/warehouse'.
21/11/24 11:20:06 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@619f2afc{/SQL,null,AVAILABLE, }
21/11/24 11:20:06 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@df921b1{/SQL/json,null,AVAILABLE, }
21/11/24 11:20:06 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@7d32e714{/SQL/execution,null,AVAILABLE, }
21/11/24 11:20:06 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@eb6ec6{/SQL/execution/json,null,AVAILABLE, }
21/11/24 11:20:06 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@3f018494{/static/sql,null,AVAILABLE, }
21/11/24 11:20:06 INFO state.StateStoreCoordinatorRef: Registered StateStoreCoordinator endpoint
21/11/24 11:20:06 WARN lineage.LineageWriter: Lineage directory /var/log/spark/lineage doesn't exist or is not writable. Lineage for this application will be disabled.
21/11/24 11:20:06 INFO util.Utils: Extension com.cloudera.spark.lineage.NavigatorQueryListener not being initialized.
21/11/24 11:20:06 INFO connector.NebulaDataSource: create reader
21/11/24 11:20:06 INFO connector.NebulaDataSource: options {spacename=basketball, nocolumn=false, metaaddress=10.154.109.68:9559, label=serve, type=edge, connectionretry=2, timeout=6000, executionretry=1, paths=[], limit=1000, returncols=, partitionnumber=100}
21/11/24 11:20:06 INFO reader.NebulaDataSourceEdgeReader: prop name start_year, type INT64
21/11/24 11:20:06 INFO reader.NebulaDataSourceEdgeReader: prop name end_year, type INT64
21/11/24 11:20:06 INFO reader.NebulaDataSourceEdgeReader: dataset's schema: StructType(StructField(_srcId,StringType,false), StructField(_dstId,StringType,false), StructField(_rank,LongType,false), StructField(start_year,LongType,true), StructField(end_year,LongType,true))
Exception in thread "main" org.apache.commons.math3.ode.UnknownParameterException: unknown parameter {0}
at com.vesoft.nebula.algorithm.Main$.executeAlgorithm(Main.scala:163)
at com.vesoft.nebula.algorithm.Main$.main(Main.scala:75)
at com.vesoft.nebula.algorithm.Main.main(Main.scala)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
at org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:851)
at org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:167)
at org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:195)
at org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:86)
at org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:926)
at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:935)
at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
21/11/24 11:20:07 INFO spark.SparkContext: Invoking stop() from shutdown hook
21/11/24 11:20:07 INFO server.AbstractConnector: Stopped Spark@4d4d48a6{HTTP/1.1,[http/1.1]}{0.0.0.0:4040}
21/11/24 11:20:07 INFO ui.SparkUI: Stopped Spark web UI at
21/11/24 11:20:07 INFO spark.MapOutputTrackerMasterEndpoint: MapOutputTrackerMasterEndpoint stopped!
21/11/24 11:20:07 INFO memory.MemoryStore: MemoryStore cleared
21/11/24 11:20:07 INFO storage.BlockManager: BlockManager stopped
21/11/24 11:20:07 INFO storage.BlockManagerMaster: BlockManagerMaster stopped
21/11/24 11:20:07 INFO scheduler.OutputCommitCoordinator$OutputCommitCoordinatorEndpoint: OutputCommitCoordinator stopped!
21/11/24 11:20:07 INFO spark.SparkContext: Successfully stopped SparkContext
21/11/24 11:20:07 INFO util.ShutdownHookManager: Shutdown hook called
21/11/24 11:20:07 INFO util.ShutdownHookManager: Deleting directory /tmp/spark-4f2b9897-ec78-4740-9572-55f23b97bf6d
21/11/24 11:20:07 INFO util.ShutdownHookManager: Deleting directory /tmp/spark-03cbaa3f-bc60-4bd5-9af3-673c79000991
配置文件：
{
# Spark relation config
spark: {
app: {
name: LPA
# spark.app.partitionNum
partitionNum:100
}
master:local
}
data: {
# data source. optional of nebula,csv,json
source: nebula
# data sink, means the algorithm result will be write into this sink. optional of nebula,csv,text
sink: nebula
# if your algorithm needs weight
hasWeight: false
}
# Nebula Graph relation config
nebula: {
# algo's data source from Nebula. If data.source is nebula, then this nebula.read config can be valid.
read: {
# Nebula metad server address, multiple addresses are split by English comma
metaAddress: "10.154.109.68:9559"
# Nebula space
space: basketball
# Nebula edge types, multiple labels means that data from multiple edges will union together
labels: ["serve"]
# Nebula edge property name for each edge type, this property will be as weight col for algorithm.
# Make sure the weightCols are corresponding to labels.
weightCols: ["start_year"]
}
# algo result sink into Nebula. If data.sink is nebula, then this nebula.write config can be valid.
write:{
# Nebula graphd server address， multiple addresses are split by English comma
graphAddress: "10.154.109.68:9669"
# Nebula metad server address, multiple addresses are split by English comma
metaAddress: "10.154.109.68:9559"
user:root
pswd:password
# Nebula space name
space:basketball
# Nebula tag name, the algorithm result will be write into this tag
tag:pagerank
}
}
local: {
# algo's data source from Nebula. If data.source is csv or json, then this local.read can be valid.
read:{
filePath: "file:///tmp/algo_edge.csv"
# srcId column
srcId:"_c0"
# dstId column
dstId:"_c1"
# weight column
#weight: "col3"
# if csv file has header
header: false
# csv file's delimiter
delimiter:","
}
# algo result sink into local file. If data.sink is csv or text, then this local.write can be valid.
write:{
resultPath:/tmp/count
}
}