nebula graph版本:3.3.0
nebula studio版本:3.5.0
部署方式:分布式
是否为线上版本:N
数据量如下:
共5.76M
用impoter导入数据的时候,报错如下信息:
请问大佬应该怎么解决呢?
nebula graph版本:3.3.0
nebula studio版本:3.5.0
部署方式:分布式
是否为线上版本:N
数据量如下:
请问大佬应该怎么解决呢?
可以补充下 storage 的日志吗?
导入太快,并发调小点。也可以把服务器配置和importer配置发出来看看有没有配置可以调优的地方
我的数据是9W多条,这个节点schema如下:
这个是impoter的配置文件:
version: v3
description: ldbc
removeTempFiles: false
clientSettings:
retry: 3
concurrency: 5 # number of graph clients
channelBufferSize: 128
space: ldbc
connection:
user: root
password: nebula
address: 192.168.200.100:9669,192.168.200.101:9669,192.168.200.111:9669,192.168.200.112:9669,192.168.200.114:9669
postStart:
commands: |
CREATE SPACE IF NOT EXISTS ldbc(PARTITION_NUM = 20, REPLICA_FACTOR = 2, vid_type = FIXED_STRING(32));
USE ldbc;
CREATE TAG IF NOT EXISTS `Comment`(`creationDate` datetime,`locationIP` string,`browserUsed` string,`content` string,`length` int);
CREATE TAG IF NOT EXISTS `Place`(`name` string,`url` string,`type` string);
CREATE TAG IF NOT EXISTS `Forum`(`title` string,`creationDate` datetime);
CREATE TAG IF NOT EXISTS `Person`(`firstName` string,`lastName` string,`gender` string,`birthday` string,`creationDate` datetime,`locationIP` string,`browserUsed` string);
CREATE TAG IF NOT EXISTS `Tagclass`(`name` string,`url` string);
CREATE TAG IF NOT EXISTS `Organisation`(`type` string,`name` string,`url` string);
CREATE TAG IF NOT EXISTS `Post`(`imageFile` string,`creationDate` datetime,`locationIP` string,`browserUsed` string,`language` string,`content` string,`length` int);
CREATE TAG IF NOT EXISTS `Tag`(`name` string,`url` string);
CREATE EDGE IF NOT EXISTS `IS_LOCATED_IN`();
CREATE EDGE IF NOT EXISTS `STUDY_AT`(`classYear` int);
CREATE EDGE IF NOT EXISTS `HAS_MEMBER`(`joinDate` datetime);
CREATE EDGE IF NOT EXISTS `HAS_TYPE`();
CREATE EDGE IF NOT EXISTS `REPLY_OF`();
CREATE EDGE IF NOT EXISTS `LIKES`(`creationDate` datetime);
CREATE EDGE IF NOT EXISTS `HAS_TAG`();
CREATE EDGE IF NOT EXISTS `KNOWS`(`creationDate` datetime);
CREATE EDGE IF NOT EXISTS `HAS_MODERATOR`();
CREATE EDGE IF NOT EXISTS `HAS_CREATOR`();
CREATE EDGE IF NOT EXISTS `CONTAINER_OF`();
CREATE EDGE IF NOT EXISTS `IS_PART_OF`();
CREATE EDGE IF NOT EXISTS `IS_SUBCLASS_OF`();
CREATE EDGE IF NOT EXISTS `HAS_INTEREST`();
CREATE EDGE IF NOT EXISTS `WORK_AT`(`workFrom` int);
CREATE TAG INDEX IF NOT EXISTS Tagclass_index on `Tagclass`();
CREATE TAG INDEX IF NOT EXISTS Tag_index on `Tag`();
CREATE TAG INDEX IF NOT EXISTS Post_index on `Post`();
CREATE TAG INDEX IF NOT EXISTS Place_index on `Place`();
CREATE TAG INDEX IF NOT EXISTS Person_index on `Person`();
CREATE TAG INDEX IF NOT EXISTS Organisation_index on `Organisation`();
CREATE TAG INDEX IF NOT EXISTS Forum_index on `Forum`();
CREATE TAG INDEX IF NOT EXISTS Comment_index on `Comment`();
CREATE EDGE INDEX IF NOT EXISTS CONTAINER_OF_index on IS_LOCATED_IN();
CREATE EDGE INDEX IF NOT EXISTS HAS_CREATOR_index on STUDY_AT(classYear);
CREATE EDGE INDEX IF NOT EXISTS HAS_INTEREST_index on HAS_MEMBER(joinDate);
CREATE EDGE INDEX IF NOT EXISTS HAS_MEMBER_index on HAS_TYPE();
CREATE EDGE INDEX IF NOT EXISTS HAS_MODERATOR_index on REPLY_OF();
CREATE EDGE INDEX IF NOT EXISTS HAS_TAG_index on LIKES(creationDate);
CREATE EDGE INDEX IF NOT EXISTS HAS_TYPE_index on HAS_TAG();
CREATE EDGE INDEX IF NOT EXISTS IS_LOCATED_IN_index on KNOWS(creationDate);
CREATE EDGE INDEX IF NOT EXISTS IS_PART_OF_index on HAS_MODERATOR();
CREATE EDGE INDEX IF NOT EXISTS IS_SUBCLASS_OF_index on HAS_CREATOR();
CREATE EDGE INDEX IF NOT EXISTS KNOWS_index on CONTAINER_OF();
CREATE EDGE INDEX IF NOT EXISTS LIKES_index on IS_PART_OF();
CREATE EDGE INDEX IF NOT EXISTS REPLY_OF_index on IS_SUBCLASS_OF();
CREATE EDGE INDEX IF NOT EXISTS STUDY_AT_index on HAS_INTEREST();
CREATE EDGE INDEX IF NOT EXISTS WORK_AT_index on WORK_AT(workFrom);
afterPeriod: 20s
logPath: ./err/test.log
files:
- path: /data/nebula/import/ldbc_forum/forum_leiden.csv
failDataPath: ./err/data/forum_leiden.csv
batchSize: 100
type: csv
csv:
withHeader: false
withLabel: false
delimiter: "|"
schema:
type: vertex
vertex:
vid:
index: 0
type: string
tags:
- name: Forum
props:
- name: title
type: string
index: 1
- name: creationDate
type: datetime
index: 2
- name: leiden
type: float
index: 3
另外,再上传一下storage日志:
nebula-graphd (2).INFO (2.6 MB)
nebula-storaged.INFO (1.9 MB)
咋这么多索引,几个storage节点,磁盘是HDD吗?
不需要的索引去掉吧,然后把storaged.conf配置文件里面的–rocksdb_db_options={“max_subcompactions”:“48”,“max_background_jobs”:“48”},这个值调大点,可以调整到cpu核数的一半。
硬盘性能太差
把写入压力减少试试
我们总共是32核的,将max_subcompactions和max_background_jobs都设置为了16,问题解决了! 谢谢!
好的!我们目前是机械硬盘,所以性能有点差
问题解决的话,可以勾选自己的回复为解决方案哈~