nebula 版本:2.1.0
部署方式 :分布式
是否为线上版本:Y
硬件信息
磁盘 SSD
CPU、内存信息 16core 128G
问题的具体描述:如题
问题描述:每天早上这边七点半通过nebula-importer 导入数据到nebula, 在十月四号前平稳运行至少有一周时间了,十月四号、五号早上七点四十左右的时候graphd节点全部退出,查看coredump文件,有如下信息
gdb) bt
#0 0x00007fa55e182337 in raise () from /lib64/libc.so.6
#1 0x00007fa55e183a28 in abort () from /lib64/libc.so.6
#2 0x0000000001081376 in __gnu_cxx::__verbose_terminate_handler() [clone .cold] ()
#3 0x000000000221fb86 in __cxxabiv1::__terminate(void (*)()) ()
#4 0x000000000221fbd1 in std::terminate() ()
#5 0x000000000222022f in __cxa_pure_virtual ()
#6 0x0000000001327bcf in nebula::graph::Scheduler::execute(nebula::graph::Executor*) ()
#7 0x000000000132a96c in void folly::detail::function::FunctionTraits<void (folly::Try<nebula::Status>&&)>::callSmall<std::enable_if<folly::futures::detail::callableResult<nebula::Status, nebula::graph::Scheduler::doSchedule(nebula::graph::Executor*)::{lambda(nebula::Status)#5}>::ReturnsFuture::value, std::enable_if::Return>::type folly::futures::detail::FutureBase<nebula::Status>::thenImplementation<nebula::graph::Scheduler::ExecTask<nebula::graph::Scheduler::doSchedule(nebula::graph::Executor*)::{lambda(nebula::Status)#5}>, folly::futures::detail::callableResult<nebula::Status, nebula::graph::Scheduler::doSchedule(nebula::graph::Executor*)::{lambda(nebula::Status)#5}>, false, nebula::Status&&>(nebula::graph::Scheduler::ExecTask<nebula::graph::Scheduler::doSchedule(nebula::graph::Executor*)::{lambda(nebula::Status)#5}>&&, folly::futures::detail::argResult<false, std::enable_if<folly::futures::detail::callableResult<nebula::Status, nebula::graph::Scheduler::doSchedule(nebula::graph::Executor*)::{lambda(nebula::Status)#5}>::ReturnsFuture::value, std::enable_if::Return>::type, nebula::Status&&>)::{lambda(folly::Try<nebula::Status>&&)#1}>(folly::detail::function::Data&, folly::Try<nebula::Status>&&) ()
#8 0x00000000010d396a in void folly::detail::function::FunctionTraits<void ()>::callSmall<folly::futures::detail::Core<nebula::Status>::doCallback()::{lambda()#1}>(folly::detail::function::Data&) ()
#9 0x00000000010d3f0c in folly::futures::detail::Core<nebula::Status>::doCallback() ()
#10 0x000000000132784c in void folly::detail::function::FunctionTraits<void (folly::Try<std::vector<nebula::Status, std::allocator<nebula::Status> > >&&)>::callSmall<std::enable_if<!folly::futures::detail::callableResult<std::vector<nebula::Status, std::allocator<nebula::Status> >, {lambda(std::vector<nebula::Status, std::allocator<nebula::Status> >)#1}>::ReturnsFuture::value, std::enable_if::Return>::type folly::futures::detail::FutureBase<std::vector<nebula::Status, std::allocator<nebula::Status> > >::thenImplementation<nebula::graph::Scheduler::doScheduleParallel(std::set<nebula::graph::Executor*, std::less<nebula::graph::Executor*>, std::allocator<nebula::graph::Executor*> > const&)::{lambda(std::vector<nebula::Status, std::allocator<nebula::Status> >)#1}, folly::futures::detail::callableResult<std::vector<nebula::Status, std::allocator<nebula::Status> >, {lambda(std::vector<nebula::Status, std::allocator<nebula::Status> >)#1}>, false, std::vector<nebula::Status, std::allocator<nebula::Status> >&&>(nebula::graph::Scheduler::doScheduleParallel(std::set<nebula::graph::Executor*, std::less<nebula::graph::Executor*>, std::allocator<nebula::graph::Executor*> > const&)::{lambda(std::vector<nebula::Status, std::allocator<nebula::Status> >)#1}&&, folly::futures::detail::argResult<false, std::enable_if<!folly::futures::detail::callableResult<std::vector<nebula::Status, std::allocator<nebula::Status> >, {lambda(std::vector<nebula::Status, std::allocator<nebula::Status> >)#1}>::ReturnsFuture::value, std::enable_if::Return>::type, std::vector<nebula::Status, std::allocator<nebula::Status> >&&>)::{lambda(folly::Try<std::vector<nebula::Status, std::allocator<nebula::Status> > >&&)#1}>(folly::detail::function::Data&, folly::Try<std::vector<nebula::Status, std::allocator<nebula::Status> > >&&) ()
#11 0x000000000132c8e2 in void folly::detail::function::FunctionTraits<void ()>::callSmall<folly::futures::detail::Core<std::vector<nebula::Status, std::allocator<nebula::Status> > >::doCallback()::{lambda()#1}>(folly::detail::function::Data&) ()
#12 0x000000000132d9bf in folly::futures::detail::Core<std::vector<nebula::Status, std::allocator<nebula::Status> > >::doCallback() ()
#13 0x000000000132efe4 in std::_Sp_counted_ptr_inplace<folly::Future<std::vector<std::iterator_traits<__gnu_cxx::__normal_iterator<folly::Future<nebula::Status>*, std::vector<folly::Future<nebula::Status>, std::allocator<folly::Future<nebula::Status> > > > >::value_type::value_type, std::allocator<std::iterator_traits<__gnu_cxx::__normal_iterator<folly::Future<nebula::Status>*, std::vector<folly::Future<nebula::Status>, std::allocator<folly::Future<nebula::Status> > > > >::value_type::value_type> > > folly::collect<__gnu_cxx::__normal_iterator<folly::Future<nebula::Status>*, std::vector<folly::Future<nebula::Status>, std::allocator<folly::Future<nebula::Status> > > > >(__gnu_cxx::__normal_iterator<folly::Future<nebula::Status>*, std::vector<folly::Future<nebula::Status>, std::allocator<folly::Future<nebula::Status> > > >, __gnu_cxx::__normal_iterator<folly::Future<nebula::Status>*, std::vector<folly::Future<nebula::Status>, std::allocator<folly::Future<nebula::Status> > > >)::Context, std::allocator<folly::Future<std::vector<std::iterator_traits<__gnu_cxx::__normal_iterator<folly::Future<nebula::Status>*, std::vector<folly::Future<nebula::Status>, std::allocator<folly::Future<nebula::Status> > > > >::value_type::value_type, std::allocator<std::iterator_traits<__gnu_cxx::__normal_iterator<folly::Future<nebula::Status>*, std::vector<folly::Future<nebula::Status>, std::allocator<folly::Future<nebula::Status> > > > >::value_type::value_type> > > folly::collect<__gnu_cxx::__normal_iterator<folly::Future<nebula::Status>*, std::vector<folly::Future<nebula::Status>, std::allocator<folly::Future<nebula::Status> > > > >(__gnu_cxx::__normal_iterator<folly::Future<nebula::Status>*, std::vector<folly::Future<nebula::Status>, std::allocator<folly::Future<nebula::Status> > > >, __gnu_cxx::__normal_iterator<folly::Future<nebula::Status>*, std::vector<folly::Future<nebula::Status>, std::allocator<folly::Future<nebula::Status> > > >)::Context>, (__gnu_cxx::_Lock_policy)2>::_M_dispose() ()
#14 0x000000000109c3b8 in std::_Sp_counted_base<(__gnu_cxx::_Lock_policy)2>::_M_release() ()
#15 0x000000000132c0b8 in bool folly::detail::function::execSmall<folly::Future<std::vector<std::iterator_traits<__gnu_cxx::__normal_iterator<folly::Future<nebula::Status>*, std::vector<folly::Future<nebula::Status>, std::allocator<folly::Future<nebula::Status> > > > >::value_type::value_type, std::allocator<std::iterator_traits<__gnu_cxx::__normal_iterator<folly::Future<nebula::Status>*, std::vector<folly::Future<nebula::Status>, std::allocator<folly::Future<nebula::Status> > > > >::value_type::value_type> > > folly::collect<__gnu_cxx::__normal_iterator<folly::Future<nebula::Status>*, std::vector<folly::Future<nebula::Status>, std::allocator<folly::Future<nebula::Status> > > > >(__gnu_cxx::__normal_iterator<folly::Future<nebula::Status>*, std::vector<folly::Future<nebula::Status>, std::allocator<folly::Future<nebula::Status> > > >, __gnu_cxx::__normal_iterator<folly::Future<nebula::Status>*, std::vector<folly::Future<nebula::Status>, std::allocator<folly::Future<nebula::Status> > > >)::{lambda(folly::Try<nebula::Status>&&)#1}>(folly::detail::function::Op, folly::detail::function::Data*, folly::detail::function::Data) ()
#16 0x00000000010ceb2b in folly::futures::detail::Core<nebula::Status>::CoreAndCallbackReference::detach() [clone .isra.0] ()
#17 0x00000000010d4022 in folly::futures::detail::Core<nebula::Status>::doCallback() ()
#18 0x00000000010d3afa in void folly::detail::function::FunctionTraits<void ()>::callSmall<folly::futures::detail::Core<nebula::Status>::doCallback()::{lambda()#2}>(folly::detail::function::Data&) ()
#19 0x000000000109bdb7 in apache::thrift::concurrency::FunctionRunner::run() ()
#20 0x00000000010aad2a in apache::thrift::concurrency::ThreadManager::ImplT<folly::LifoSemImpl<std::atomic, folly::SaturatingSemaphore<true, std::atomic> > >::Worker<folly::LifoSemImpl<std::atomic, folly::SaturatingSemaphore<true, std::atomic> > >::run() ()
#21 0x0000000001c8535c in apache::thrift::concurrency::PthreadThread::threadMain(void*) ()
#22 0x00007fa55e521e65 in start_thread () from /lib64/libpthread.so.0
#23 0x00007fa55e24a88d in clone () from /lib64/libc.so.6
出现这个问题后,执行 balance leader
, 也会导致graphd 节点退出,core 信息如下
(gdb) bt
#0 0x00007f85fd0dc337 in raise () from /lib64/libc.so.6
#1 0x00007f85fd0dda28 in abort () from /lib64/libc.so.6
#2 0x0000000001081376 in __gnu_cxx::__verbose_terminate_handler() [clone .cold] ()
#3 0x000000000221fb86 in __cxxabiv1::__terminate(void (*)()) ()
#4 0x000000000221fbd1 in std::terminate() ()
#5 0x000000000222022f in __cxa_pure_virtual ()
#6 0x0000000001327bcf in nebula::graph::Scheduler::execute(nebula::graph::Executor*) ()
#7 0x000000000132a96c in void folly::detail::function::FunctionTraits<void (folly::Try<nebula::Status>&&)>::callSmall<std::enable_if<folly::futures::detail::callableResult<nebula::Status, nebula::graph::Scheduler::doSchedule(nebula::graph::Executor*)::{lambda(nebula::Status)#5}>::ReturnsFuture::value, std::enable_if::Return>::type folly::futures::detail::FutureBase<nebula::Status>::thenImplementation<nebula::graph::Scheduler::ExecTask<nebula::graph::Scheduler::doSchedule(nebula::graph::Executor*)::{lambda(nebula::Status)#5}>, folly::futures::detail::callableResult<nebula::Status, nebula::graph::Scheduler::doSchedule(nebula::graph::Executor*)::{lambda(nebula::Status)#5}>, false, nebula::Status&&>(nebula::graph::Scheduler::ExecTask<nebula::graph::Scheduler::doSchedule(nebula::graph::Executor*)::{lambda(nebula::Status)#5}>&&, folly::futures::detail::argResult<false, std::enable_if<folly::futures::detail::callableResult<nebula::Status, nebula::graph::Scheduler::doSchedule(nebula::graph::Executor*)::{lambda(nebula::Status)#5}>::ReturnsFuture::value, std::enable_if::Return>::type, nebula::Status&&>)::{lambda(folly::Try<nebula::Status>&&)#1}>(folly::detail::function::Data&, folly::Try<nebula::Status>&&) ()
#8 0x00000000010d396a in void folly::detail::function::FunctionTraits<void ()>::callSmall<folly::futures::detail::Core<nebula::Status>::doCallback()::{lambda()#1}>(folly::detail::function::Data&) ()
#9 0x00000000010d3f0c in folly::futures::detail::Core<nebula::Status>::doCallback() ()
#10 0x000000000132784c in void folly::detail::function::FunctionTraits<void (folly::Try<std::vector<nebula::Status, std::allocator<nebula::Status> > >&&)>::callSmall<std::enable_if<!folly::futures::detail::callableResult<std::vector<nebula::Status, std::allocator<nebula::Status> >, {lambda(std::vector<nebula::Status, std::allocator<nebula::Status> >)#1}>::ReturnsFuture::value, std::enable_if::Return>::type folly::futures::detail::FutureBase<std::vector<nebula::Status, std::allocator<nebula::Status> > >::thenImplementation<nebula::graph::Scheduler::doScheduleParallel(std::set<nebula::graph::Executor*, std::less<nebula::graph::Executor*>, std::allocator<nebula::graph::Executor*> > const&)::{lambda(std::vector<nebula::Status, std::allocator<nebula::Status> >)#1}, folly::futures::detail::callableResult<std::vector<nebula::Status, std::allocator<nebula::Status> >, {lambda(std::vector<nebula::Status, std::allocator<nebula::Status> >)#1}>, false, std::vector<nebula::Status, std::allocator<nebula::Status> >&&>(nebula::graph::Scheduler::doScheduleParallel(std::set<nebula::graph::Executor*, std::less<nebula::graph::Executor*>, std::allocator<nebula::graph::Executor*> > const&)::{lambda(std::vector<nebula::Status, std::allocator<nebula::Status> >)#1}&&, folly::futures::detail::argResult<false, std::enable_if<!folly::futures::detail::callableResult<std::vector<nebula::Status, std::allocator<nebula::Status> >, {lambda(std::vector<nebula::Status, std::allocator<nebula::Status> >)#1}>::ReturnsFuture::value, std::enable_if::Return>::type, std::vector<nebula::Status, std::allocator<nebula::Status> >&&>)::{lambda(folly::Try<std::vector<nebula::Status, std::allocator<nebula::Status> > >&&)#1}>(folly::detail::function::Data&, folly::Try<std::vector<nebula::Status, std::allocator<nebula::Status> > >&&) ()
#11 0x000000000132c8e2 in void folly::detail::function::FunctionTraits<void ()>::callSmall<folly::futures::detail::Core<std::vector<nebula::Status, std::allocator<nebula::Status> > >::doCallback()::{lambda()#1}>(folly::detail::function::Data&) ()
#12 0x000000000132d9bf in folly::futures::detail::Core<std::vector<nebula::Status, std::allocator<nebula::Status> > >::doCallback() ()
#13 0x000000000132efe4 in std::_Sp_counted_ptr_inplace<folly::Future<std::vector<std::iterator_traits<__gnu_cxx::__normal_iterator<folly::Future<nebula::Status>*, std::vector<folly::Future<nebula::Status>, std::allocator<folly::Future<nebula::Status> > > > >::value_type::value_type, std::allocator<std::iterator_traits<__gnu_cxx::__normal_iterator<folly::Future<nebula::Status>*, std::vector<folly::Future<nebula::Status>, std::allocator<folly::Future<nebula::Status> > > > >::value_type::value_type> > > folly::collect<__gnu_cxx::__normal_iterator<folly::Future<nebula::Status>*, std::vector<folly::Future<nebula::Status>, std::allocator<folly::Future<nebula::Status> > > > >(__gnu_cxx::__normal_iterator<folly::Future<nebula::Status>*, std::vector<folly::Future<nebula::Status>, std::allocator<folly::Future<nebula::Status> > > >, __gnu_cxx::__normal_iterator<folly::Future<nebula::Status>*, std::vector<folly::Future<nebula::Status>, std::allocator<folly::Future<nebula::Status> > > >)::Context, std::allocator<folly::Future<std::vector<std::iterator_traits<__gnu_cxx::__normal_iterator<folly::Future<nebula::Status>*, std::vector<folly::Future<nebula::Status>, std::allocator<folly::Future<nebula::Status> > > > >::value_type::value_type, std::allocator<std::iterator_traits<__gnu_cxx::__normal_iterator<folly::Future<nebula::Status>*, std::vector<folly::Future<nebula::Status>, std::allocator<folly::Future<nebula::Status> > > > >::value_type::value_type> > > folly::collect<__gnu_cxx::__normal_iterator<folly::Future<nebula::Status>*, std::vector<folly::Future<nebula::Status>, std::allocator<folly::Future<nebula::Status> > > > >(__gnu_cxx::__normal_iterator<folly::Future<nebula::Status>*, std::vector<folly::Future<nebula::Status>, std::allocator<folly::Future<nebula::Status> > > >, __gnu_cxx::__normal_iterator<folly::Future<nebula::Status>*, std::vector<folly::Future<nebula::Status>, std::allocator<folly::Future<nebula::Status> > > >)::Context>, (__gnu_cxx::_Lock_policy)2>::_M_dispose() ()
#14 0x000000000109c3b8 in std::_Sp_counted_base<(__gnu_cxx::_Lock_policy)2>::_M_release() ()
#15 0x000000000132c0b8 in bool folly::detail::function::execSmall<folly::Future<std::vector<std::iterator_traits<__gnu_cxx::__normal_iterator<folly::Future<nebula::Status>*, std::vector<folly::Future<nebula::Status>, std::allocator<folly::Future<nebula::Status> > > > >::value_type::value_type, std::allocator<std::iterator_traits<__gnu_cxx::__normal_iterator<folly::Future<nebula::Status>*, std::vector<folly::Future<nebula::Status>, std::allocator<folly::Future<nebula::Status> > > > >::value_type::value_type> > > folly::collect<__gnu_cxx::__normal_iterator<folly::Future<nebula::Status>*, std::vector<folly::Future<nebula::Status>, std::allocator<folly::Future<nebula::Status> > > > >(__gnu_cxx::__normal_iterator<folly::Future<nebula::Status>*, std::vector<folly::Future<nebula::Status>, std::allocator<folly::Future<nebula::Status> > > >, __gnu_cxx::__normal_iterator<folly::Future<nebula::Status>*, std::vector<folly::Future<nebula::Status>, std::allocator<folly::Future<nebula::Status> > > >)::{lambda(folly::Try<nebula::Status>&&)#1}>(folly::detail::function::Op, folly::detail::function::Data*, folly::detail::function::Data) ()
#16 0x00000000010ceb2b in folly::futures::detail::Core<nebula::Status>::CoreAndCallbackReference::detach() [clone .isra.0] ()
#17 0x00000000010d4022 in folly::futures::detail::Core<nebula::Status>::doCallback() ()
#18 0x00000000010d3afa in void folly::detail::function::FunctionTraits<void ()>::callSmall<folly::futures::detail::Core<nebula::Status>::doCallback()::{lambda()#2}>(folly::detail::function::Data&) ()
#19 0x000000000109bdb7 in apache::thrift::concurrency::FunctionRunner::run() ()
#20 0x00000000010aad2a in apache::thrift::concurrency::ThreadManager::ImplT<folly::LifoSemImpl<std::atomic, folly::SaturatingSemaphore<true, std::atomic> > >::Worker<folly::LifoSemImpl<std::atomic, folly::SaturatingSemaphore<true, std::atomic> > >::run() ()
#21 0x0000000001c8535c in apache::thrift::concurrency::PthreadThread::threadMain(void*) ()
#22 0x00007f85fd47be65 in start_thread () from /lib64/libpthread.so.0
#23 0x00007f85fd1a488d in clone () from /lib64/libc.so.6
麻烦开发人员看一下这个错误是啥导致的?谢谢