[root@10-130-166-241.okg-db-graphd.cp01 storaged]# dmesg -Tgrep nebula [Wed Jun 16 051910 2021] [16660] 0 16660 1482045 34120 575 0 830 nebula-metad [Wed Jun 16 051910 2021] [24288] 0 24288 18989677 6761462 28147 0 830 nebula-storaged [Wed Jun 16 051910 2021] [185224] 0 185224 4873295 22514 2175 0 830 nebula-graphd [Wed Jun 16 051910 2021] Killed process 24288 (nebula-storaged) total-vm75958708kB, anon-rss27045848kB, file-rss0kB, shmem-rss0kB [Wed Jun 16 051910 2021] [16660] 0 16660 1482045 34120 575 0 830 nebula-metad [Wed Jun 16 051910 2021] [185224] 0 185224 4873295 22514 2175 0 830 nebula-graphd [Thu Jun 17 003538 2021] [16660] 0 16660 1482045 35350 576 0 830 nebula-metad [Thu Jun 17 003538 2021] [185224] 0 185224 5135439 22406 2178 0 830 nebula-graphd [Thu Jun 17 003538 2021] [378291] 0 378291 10605791 7848856 17993 0 830 nebula-storaged [Thu Jun 17 003538 2021] Killed process 378291 (nebula-storaged) total-vm42423164kB, anon-rss31395424kB, file-rss0kB, shmem-rss0kB [Thu Jun 17 003538 2021] [16660] 0 16660 1482045 35352 576 0 830 nebula-metad [Thu Jun 17 003538 2021] [185224] 0 185224 5135439 22406 2178 0 830 nebula-graphd [Thu Jun 24 044135 2021] [16660] 0 16660 2026045 34656 593 0 830 nebula-metad [Thu Jun 24 044135 2021] [185224] 0 185224 5926735 22583 2193 0 830 nebula-graphd [Thu Jun 24 044135 2021] [290965] 0 290965 13961462 7598532 19211 0 830 nebula-storaged [Thu Jun 24 044135 2021] Killed process 290965 (nebula-storaged) total-vm55845848kB, anon-rss30394128kB, file-rss0kB, shmem-rss0kB [Thu Jun 24 044135 2021] [16660] 0 16660 2026045 34657 593 0 830 nebula-metad [Thu Jun 24 044135 2021] [185224] 0 185224 5926735 22583 2193 0 830 nebula-graphd [Thu Jun 24 044136 2021] [16660] 0 16660 2026045 34657 593 0 830 nebula-metad [Thu Jun 24 044136 2021] [185224] 0 185224 5926735 22583 2193 0 830 nebula-graphd [Thu Jun 24 044136 2021] [16660] 0 16660 2026045 34657 593 0 830 nebula-metad [Thu Jun 24 044136 2021] [185224] 0 185224 5926735 22583 2193 0 830 nebula-graphd [Thu Oct 7 191648 2021] [230966] 0 230966 637989 42640 1017 0 830 nebula-metad [Thu Oct 7 191648 2021] [237748] 0 237748 18874628 7046105 30777 0 830 nebula-storaged [Thu Oct 7 191648 2021] [244527] 0 244527 1436806 447462 2361 0 830 nebula-graphd [Thu Oct 7 191648 2021] Killed process 237748 (nebula-storaged) total-vm75498512kB, anon-rss28184420kB, file-rss0kB, shmem-rss0kB [Thu Oct 7 191649 2021] [230966] 0 230966 637989 42743 1017 0 830 nebula-metad [Thu Oct 7 191649 2021] [244527] 0 244527 1436806 447462 2361 0 830 nebula-graphd [Thu Oct 7 191649 2021] [230966] 0 230966 637989 42776 1017 0 830 nebula-metad [Thu Oct 7 191649 2021] [244527] 0 244527 1436806 447462 2361 0 830 nebula-graphd [Thu Oct 7 191649 2021] Killed process 244527 (nebula-graphd) total-vm5747224kB, anon-rss1789848kB, file-rss0kB, shmem-rss0kB [Wed Jan 5 174552 2022] nebula-metad invoked oom-killer gfp_mask=0xd0, order=0, oom_score_adj=830 [Wed Jan 5 174552 2022] nebula-metad cpuset=85a4e217caf1cb046caf1cd421a48e377b2cf29bf10b09a564215e577d3aa015 mems_allowed=0-1 [Wed Jan 5 174552 2022] CPU 59 PID 273956 Comm nebula-metad Kdump loaded Tainted G OE ------------ 3.10.0-957.27.2.el7.x86_64 #1 [Wed Jan 5 174552 2022] [338184] 0 338184 2997126 128264 5125 0 830 nebula-graphd [Wed Jan 5 174552 2022] [65652] 0 65652 17592830 5267744 31248 0 830 nebula-storaged [Wed Jan 5 174552 2022] [273758] 0 273758 1799461 396551 3059 0 830 nebula-metad [Wed Jan 5 174552 2022] Killed process 65652 (nebula-storaged) total-vm70371320kB, anon-rss21070976kB, file-rss0kB, shmem-rss0kB [Wed Jan 5 174552 2022] [338184] 0 338184 2997126 128264 5125 0 830 nebula-graphd [Wed Jan 5 174552 2022] [273758] 0 273758 1799461 396609 3059 0 830 nebula-metad [Wed Jan 5 174552 2022] [338184] 0 338184 2997126 128264 5125 0 830 nebula-graphd [Wed Jan 5 174552 2022] [273758] 0 273758 1799461 396669 3059 0 830 nebula-metad [Wed Jan 5 174552 2022] [338184] 0 338184 2997126 128264 5125 0 830 nebula-graphd [Wed Jan 5 174552 2022] [273758] 0 273758 1799461 396669 3059 0 830 nebula-metad [Wed Jan 5 174552 2022] nebula-metad invoked oom-killer gfp_mask=0xd0, order=0, oom_score_adj=830 [Wed Jan 5 174552 2022] nebula-metad cpuset=85a4e217caf1cb046caf1cd421a48e377b2cf29bf10b09a564215e577d3aa015 mems_allowed=0-1 [Wed Jan 5 174552 2022] CPU 59 PID 273956 Comm nebula-metad Kdump loaded Tainted G OE ------------ 3.10.0-957.27.2.el7.x86_64 #1 [Wed Jan 5 174552 2022] [338184] 0 338184 2997126 128264 5125 0 830 nebula-graphd [Wed Jan 5 174552 2022] [273758] 0 273758 1799461 396669 3059 0 830 nebula-metad [Wed Jan 5 174552 2022] Killed process 273758 (nebula-metad) total-vm7197844kB, anon-rss1586676kB, file-rss0kB, shmem-rss0kB [root@10-130-166-241.okg-db-graphd.cp01 storaged]# root@10-130-166-242.okg-db-graphd.cp01 etc]# dmesg -T|grep nebula [Wed Jun 16 05:17:35 2021] [436536] 0 436536 1938099 17046 466 0 830 nebula-metad [Wed Jun 16 05:17:35 2021] [438309] 0 438309 18851693 6796157 27993 0 830 nebula-storaged [Wed Jun 16 05:17:35 2021] [490332] 0 490332 4945487 24082 2419 0 830 nebula-graphd [Wed Jun 16 05:17:35 2021] Killed process 438309 (nebula-storaged) total-vm:75406772kB, anon-rss:27184628kB, file-rss:0kB, shmem-rss:0kB [Thu Jun 17 00:33:43 2021] [436536] 0 436536 1938099 19688 466 0 830 nebula-metad [Thu Jun 17 00:33:43 2021] [490332] 0 490332 5273167 23901 2423 0 830 nebula-graphd [Thu Jun 17 00:33:43 2021] [377836] 0 377836 8316256 6793720 15505 0 830 nebula-storaged [Thu Jun 17 00:33:43 2021] Killed process 377836 (nebula-storaged) total-vm:33265024kB, anon-rss:27174880kB, file-rss:0kB, shmem-rss:0kB [Thu Jun 24 04:40:03 2021] nebula-metad invoked oom-killer: gfp_mask=0xd0, order=0, oom_score_adj=830 [Thu Jun 24 04:40:03 2021] nebula-metad cpuset=a79ba9e63fe24b94a08d53bed4dbb922a4e3d6b3cc6f0d26249d83ae4024bd5b mems_allowed=0-1 [Thu Jun 24 04:40:03 2021] CPU: 63 PID: 436752 Comm: nebula-metad Kdump: loaded Tainted: G OE ------------ 3.10.0-957.27.2.el7.x86_64 #1 [Thu Jun 24 04:40:03 2021] [436536] 0 436536 2744627 24478 481 0 830 nebula-metad [Thu Jun 24 04:40:03 2021] [490332] 0 490332 5802319 23937 2436 0 830 nebula-graphd [Thu Jun 24 04:40:03 2021] [137404] 0 137404 14829941 7656278 19305 0 830 nebula-storaged [Thu Jun 24 04:40:03 2021] Killed process 137404 (nebula-storaged) total-vm:59319764kB, anon-rss:30625112kB, file-rss:0kB, shmem-rss:0kB [Thu Jun 24 04:40:04 2021] [436536] 0 436536 2744627 24478 481 0 830 nebula-metad [Thu Jun 24 04:40:04 2021] [490332] 0 490332 5802319 23937 2436 0 830 nebula-graphd [Thu Jun 24 04:40:04 2021] [436536] 0 436536 2744627 24478 481 0 830 nebula-metad [Thu Jun 24 04:40:04 2021] [490332] 0 490332 5802319 23937 2436 0 830 nebula-graphd [Thu Oct 7 19:14:20 2021] [180031] 0 180031 593701 35811 957 0 830 nebula-metad [Thu Oct 7 19:14:20 2021] [187977] 0 187977 18193820 7188639 30103 0 830 nebula-storaged [Thu Oct 7 19:14:20 2021] [193637] 0 193637 999814 315070 1517 0 830 nebula-graphd [Thu Oct 7 19:14:20 2021] Killed process 187977 (nebula-storaged) total-vm:72775280kB, anon-rss:28754556kB, file-rss:0kB, shmem-rss:0kB [Thu Oct 7 19:14:20 2021] [180031] 0 180031 593701 35811 957 0 830 nebula-metad [Thu Oct 7 19:14:20 2021] [193637] 0 193637 999814 315080 1517 0 830 nebula-graphd [Wed Oct 27 17:37:45 2021] [376480] 0 376480 573830 46473 976 0 830 nebula-graphd [Wed Oct 27 17:37:45 2021] [390892] 0 390892 9605408 7075336 16538 0 830 nebula-storaged [Wed Oct 27 17:37:45 2021] [520688] 0 520688 1592879 355875 2676 0 830 nebula-metad [Wed Oct 27 17:37:45 2021] Killed process 390892 (nebula-storaged) total-vm:38421632kB, anon-rss:28301344kB, file-rss:0kB, shmem-rss:0kB [Wed Oct 27 17:37:45 2021] nebula-metad invoked oom-killer: gfp_mask=0xd0, order=0, oom_score_adj=830 [Wed Oct 27 17:37:45 2021] nebula-metad cpuset=a79ba9e63fe24b94a08d53bed4dbb922a4e3d6b3cc6f0d26249d83ae4024bd5b mems_allowed=0-1 [Wed Oct 27 17:37:45 2021] CPU: 23 PID: 520974 Comm: nebula-metad Kdump: loaded Tainted: G OE ------------ 3.10.0-957.27.2.el7.x86_64 #1 [Wed Oct 27 17:37:45 2021] [376480] 0 376480 573830 49416 976 0 830 nebula-graphd [Wed Oct 27 17:37:45 2021] [520688] 0 520688 1592879 354269 2676 0 830 nebula-metad [Wed Oct 27 17:37:46 2021] [376480] 0 376480 573830 49417 976 0 830 nebula-graphd [Wed Oct 27 17:37:46 2021] [520688] 0 520688 1592879 354432 2676 0 830 nebula-metad [Wed Oct 27 17:37:46 2021] Killed process 520688 (nebula-metad) total-vm:6371516kB, anon-rss:1417728kB, file-rss:0kB, shmem-rss:0kB [Mon Jan 10 15:59:29 2022] [376480] 0 376480 695430 196411 1224 0 830 nebula-graphd [Mon Jan 10 15:59:29 2022] [604849] 0 604849 19507247 7168135 34469 0 830 nebula-storaged [Mon Jan 10 15:59:29 2022] [587639] 0 587639 876837 136497 1396 0 830 nebula-metad [Mon Jan 10 15:59:29 2022] Killed process 604849 (nebula-storaged) total-vm:78028988kB, anon-rss:28672440kB, file-rss:100kB, shmem-rss:0kB [Mon Jan 10 15:59:29 2022] [376480] 0 376480 695430 196411 1224 0 830 nebula-graphd [Mon Jan 10 15:59:29 2022] [607972] 0 604849 19507247 7171970 34469 0 830 nebula-storaged [Mon Jan 10 15:59:29 2022] [587639] 0 587639 876837 136497 1396 0 830 nebula-metad [Mon Jan 10 15:59:29 2022] Memory cgroup out of memory: Kill process 607972 (nebula-storaged) score 1663 or sacrifice child [Mon Jan 10 15:59:29 2022] Killed process 607972 (nebula-storaged) total-vm:78028988kB, anon-rss:28687104kB, file-rss:776kB, shmem-rss:0kB [root@10-130-166-242.okg-db-graphd.cp01 etc]# root@10-130-166-243.okg-db-graphd.cp01 ~]# dmesg -T|grep nebula [Wed Jun 16 05:18:07 2021] [357704] 0 357704 1414067 18369 496 0 830 nebula-metad [Wed Jun 16 05:18:07 2021] [370239] 0 370239 17241325 6047612 26084 0 830 nebula-storaged [Wed Jun 16 05:18:07 2021] [171551] 0 171551 12274383 725617 15698 0 830 nebula-graphd [Wed Jun 16 05:18:07 2021] Killed process 370239 (nebula-storaged) total-vm:68965300kB, anon-rss:24190448kB, file-rss:0kB, shmem-rss:0kB [Thu Jun 17 00:34:18 2021] [357704] 0 357704 1414067 21013 496 0 830 nebula-metad [Thu Jun 17 00:34:18 2021] [171551] 0 171551 12274383 537571 15748 0 830 nebula-graphd [Thu Jun 17 00:34:18 2021] [580967] 0 580967 8313695 6233132 14209 0 830 nebula-storaged [Thu Jun 17 00:34:18 2021] Killed process 580967 (nebula-storaged) total-vm:33254780kB, anon-rss:24932528kB, file-rss:0kB, shmem-rss:0kB [Thu Jun 24 04:40:45 2021] [357704] 0 357704 2499251 25956 513 0 830 nebula-metad [Thu Jun 24 04:40:45 2021] [171551] 0 171551 12675535 426454 15914 0 830 nebula-graphd [Thu Jun 24 04:40:45 2021] [349171] 0 349171 13208949 7212380 18188 0 830 nebula-storaged [Thu Jun 24 04:40:45 2021] Killed process 349171 (nebula-storaged) total-vm:52835796kB, anon-rss:28849520kB, file-rss:0kB, shmem-rss:0kB [Thu Jun 24 04:40:45 2021] [357704] 0 357704 2499251 25957 513 0 830 nebula-metad [Thu Jun 24 04:40:45 2021] [171551] 0 171551 12675535 426454 15914 0 830 nebula-graphd [Thu Jun 24 04:40:45 2021] [357704] 0 357704 2499251 25957 513 0 830 nebula-metad [Thu Jun 24 04:40:45 2021] [171551] 0 171551 12675535 426454 15914 0 830 nebula-graphd [Thu Jun 24 04:40:45 2021] Killed process 171551 (nebula-graphd) total-vm:50702140kB, anon-rss:1705816kB, file-rss:0kB, shmem-rss:0kB [Tue Jun 29 00:54:40 2021] nebula-storaged invoked oom-killer: gfp_mask=0xd0, order=0, oom_score_adj=830 [Tue Jun 29 00:54:40 2021] nebula-storaged cpuset=39c0f68f05b88ed41e90e9fdc6efba4e5d6c7c03dd4694329c2b5d904bc6fdd2 mems_allowed=0-1 [Tue Jun 29 00:54:40 2021] CPU: 61 PID: 148839 Comm: nebula-storaged Kdump: loaded Tainted: G OE ------------ 3.10.0-957.27.2.el7.x86_64 #1 [Tue Jun 29 00:54:40 2021] [357704] 0 357704 3026611 21656 523 0 830 nebula-metad [Tue Jun 29 00:54:40 2021] [147144] 0 147144 10278141 3324934 12185 0 830 nebula-storaged [Tue Jun 29 00:54:40 2021] [147773] 0 147773 5629391 4227429 8667 0 830 nebula-graphd [Tue Jun 29 00:54:40 2021] Killed process 147773 (nebula-graphd) total-vm:22517564kB, anon-rss:16909716kB, file-rss:0kB, shmem-rss:0kB [Tue Jun 29 00:54:41 2021] [357704] 0 357704 3026611 21662 523 0 830 nebula-metad [Tue Jun 29 00:54:41 2021] [147144] 0 147144 10278141 3324934 12185 0 830 nebula-storaged [Tue Jun 29 00:54:41 2021] Killed process 147144 (nebula-storaged) total-vm:41112564kB, anon-rss:13299736kB, file-rss:0kB, shmem-rss:0kB [Thu Oct 7 19:15:21 2021] nebula-metad invoked oom-killer: gfp_mask=0xd0, order=0, oom_score_adj=830 [Thu Oct 7 19:15:21 2021] nebula-metad cpuset=39c0f68f05b88ed41e90e9fdc6efba4e5d6c7c03dd4694329c2b5d904bc6fdd2 mems_allowed=0-1 [Thu Oct 7 19:15:21 2021] CPU: 59 PID: 638109 Comm: nebula-metad Kdump: loaded Tainted: G OE ------------ 3.10.0-957.27.2.el7.x86_64 #1 [Thu Oct 7 19:15:21 2021] [637449] 0 637449 1906490 356451 3263 0 830 nebula-metad [Thu Oct 7 19:15:21 2021] [645637] 0 645637 18847389 7167083 31029 0 830 nebula-storaged [Thu Oct 7 19:15:21 2021] [652632] 0 652632 461318 20686 778 0 830 nebula-graphd [Thu Oct 7 19:15:21 2021] Killed process 645637 (nebula-storaged) total-vm:75389556kB, anon-rss:28668332kB, file-rss:0kB, shmem-rss:0kB [Thu Oct 7 19:15:23 2021] nebula-metad invoked oom-killer: gfp_mask=0xd0, order=0, oom_score_adj=830 [Thu Oct 7 19:15:23 2021] nebula-metad cpuset=39c0f68f05b88ed41e90e9fdc6efba4e5d6c7c03dd4694329c2b5d904bc6fdd2 mems_allowed=0-1 [Thu Oct 7 19:15:23 2021] CPU: 59 PID: 638106 Comm: nebula-metad Kdump: loaded Tainted: G OE ------------ 3.10.0-957.27.2.el7.x86_64 #1 [Thu Oct 7 19:15:23 2021] [637449] 0 637449 1906490 356480 3263 0 830 nebula-metad [Thu Oct 7 19:15:23 2021] [652632] 0 652632 461318 20686 778 0 830 nebula-graphd [Wed Nov 24 01:41:49 2021] nebula-metad invoked oom-killer: gfp_mask=0xd0, order=0, oom_score_adj=830 [Wed Nov 24 01:41:49 2021] nebula-metad cpuset=39c0f68f05b88ed41e90e9fdc6efba4e5d6c7c03dd4694329c2b5d904bc6fdd2 mems_allowed=0-1 [Wed Nov 24 01:41:49 2021] CPU: 1 PID: 480416 Comm: nebula-metad Kdump: loaded Tainted: G OE ------------ 3.10.0-957.27.2.el7.x86_64 #1 [Wed Nov 24 01:41:49 2021] [247748] 0 247748 1174918 23872 1825 0 830 nebula-graphd [Wed Nov 24 01:41:49 2021] [340075] 0 340075 17547938 5491457 30743 0 830 nebula-storaged [Wed Nov 24 01:41:49 2021] [480118] 0 480118 4151354 1909913 6924 0 830 nebula-metad [Wed Nov 24 01:41:49 2021] Killed process 340075 (nebula-storaged) total-vm:70191752kB, anon-rss:21965828kB, file-rss:0kB, shmem-rss:0kB [Wed Nov 24 01:41:49 2021] [247748] 0 247748 1174918 23872 1825 0 830 nebula-graphd [Wed Nov 24 01:41:49 2021] [341541] 0 340075 17547938 5493285 30743 0 830 nebula-storaged [Wed Nov 24 01:41:49 2021] [480118] 0 480118 4151354 1909913 6924 0 830 nebula-metad [Wed Nov 24 01:41:49 2021] Memory cgroup out of memory: Kill process 341541 (nebula-storaged) score 1468 or sacrifice child [Wed Nov 24 01:41:49 2021] Killed process 341541 (nebula-storaged) total-vm:70191752kB, anon-rss:21973140kB, file-rss:0kB, shmem-rss:0kB [Mon Jan 10 15:06:44 2022] [499283] 0 499283 1036038 124689 1846 0 830 nebula-graphd [Mon Jan 10 15:06:44 2022] [499310] 0 499310 17993096 6531791 31986 0 830 nebula-storaged [Mon Jan 10 15:06:44 2022] [192448] 0 192448 1967665 600488 3370 0 830 nebula-metad [Mon Jan 10 15:06:44 2022] Killed process 499310 (nebula-storaged) total-vm:71972384kB, anon-rss:26127164kB, file-rss:0kB, shmem-rss:0kB [Mon Jan 10 15:06:44 2022] [499283] 0 499283 1036038 124689 1846 0 830 nebula-graphd [Mon Jan 10 15:06:44 2022] [192448] 0 192448 1967665 600597 3370 0 830 nebula-metad [root@10-130-166-243.okg-db-graphd.cp01 ~]# [root@10-131-55-170.okg-db-graphd.cp01 ~]# dmesg -T|grep nebula [Wed Jun 16 05:35:18 2021] [452339] 0 452339 18837358 6778864 28257 0 830 nebula-storaged [Wed Jun 16 05:35:18 2021] [452920] 0 452920 6730959 32383 4031 0 830 nebula-graphd [Wed Jun 16 05:35:18 2021] Killed process 452339 (nebula-storaged) total-vm:75349432kB, anon-rss:27115456kB, file-rss:0kB, shmem-rss:0kB [Thu Jun 17 00:51:27 2021] [452920] 0 452920 6730959 32217 4033 0 830 nebula-graphd [Thu Jun 17 00:51:27 2021] [228376] 0 228376 9230431 6778894 15442 0 830 nebula-storaged [Thu Jun 17 00:51:27 2021] Killed process 228376 (nebula-storaged) total-vm:36921724kB, anon-rss:27115576kB, file-rss:0kB, shmem-rss:0kB [Thu Jun 17 00:51:27 2021] [452920] 0 452920 6730959 32217 4033 0 830 nebula-graphd [Thu Jun 24 04:57:49 2021] [452920] 0 452920 7785423 32813 4045 0 830 nebula-graphd [Thu Jun 24 04:57:49 2021] [383663] 0 383663 14081270 7544270 18965 0 830 nebula-storaged [Thu Jun 24 04:57:49 2021] Killed process 383663 (nebula-storaged) total-vm:56325080kB, anon-rss:30177080kB, file-rss:0kB, shmem-rss:0kB [Thu Oct 7 19:32:34 2021] [629800] 0 629800 2670982 1517629 4103 0 830 nebula-graphd [Thu Oct 7 19:32:34 2021] [639219] 0 639219 15806724 6028647 25945 0 830 nebula-storaged [Thu Oct 7 19:32:34 2021] Killed process 639219 (nebula-storaged) total-vm:63226896kB, anon-rss:24114588kB, file-rss:0kB, shmem-rss:0kB [Thu Oct 7 19:32:34 2021] [629800] 0 629800 2670982 1517704 4103 0 830 nebula-graphd [Thu Oct 7 19:32:34 2021] Killed process 629800 (nebula-graphd) total-vm:10683928kB, anon-rss:6070816kB, file-rss:0kB, shmem-rss:0kB [Mon Jan 10 16:18:35 2022] [330891] 0 330891 650630 165485 1111 0 830 nebula-graphd [Mon Jan 10 16:18:35 2022] [585300] 0 585300 19489075 7140184 34861 0 830 nebula-storaged [Mon Jan 10 16:18:35 2022] Killed process 585300 (nebula-storaged) total-vm:77956300kB, anon-rss:28560736kB, file-rss:0kB, shmem-rss:0kB [Mon Jan 10 16:18:35 2022] [330891] 0 330891 650630 165485 1111 0 830 nebula-graphd [root@10-131-55-170.okg-db-graphd.cp01 ~]# [root@10-131-55-171.okg-db-graphd.cp01 ~]# dmesg -T|grep nebula [Thu Oct 7 19:32:05 2021] [ 7583] 0 7583 18748038 7032490 30551 0 830 nebula-storaged [Thu Oct 7 19:32:05 2021] [ 9590] 0 9590 1280646 518571 2015 0 830 nebula-graphd [Thu Oct 7 19:32:05 2021] Killed process 7583 (nebula-storaged) total-vm:74992152kB, anon-rss:28129960kB, file-rss:0kB, shmem-rss:0kB [Thu Oct 7 19:32:05 2021] [ 9590] 0 9590 1280646 518646 2015 0 830 nebula-graphd [Thu Oct 7 19:32:05 2021] [ 9590] 0 9590 1280646 518706 2015 0 830 nebula-graphd [Thu Oct 7 19:32:05 2021] Killed process 9590 (nebula-graphd) total-vm:5122584kB, anon-rss:2074824kB, file-rss:0kB, shmem-rss:0kB [root@10-131-55-171.okg-db-graphd.cp01 ~]# ******************************************************************************** [root@10-128-31-23.okg-db-graphd.cp01 ~]# dmesg -T|grep nebula [Wed Jun 9 17:35:52 2021] [139744] 0 139744 489045 739 67 0 660 nebula-importer [Wed Jun 9 17:35:52 2021] [139745] 0 139745 3604601 2886792 5746 0 660 nebula-http-gat [Wed Jun 9 17:35:52 2021] [328639] 0 328639 13764715 3701562 20712 0 660 nebula-storaged [Wed Jun 9 17:35:52 2021] [328948] 0 328948 5732303 32531 3757 0 660 nebula-graphd [Wed Jun 9 17:35:52 2021] Killed process 328639 (nebula-storaged) total-vm:55058860kB, anon-rss:14806248kB, file-rss:0kB, shmem-rss:0kB [Wed Jun 9 17:37:54 2021] [139744] 0 139744 491094 809 68 0 660 nebula-importer [Wed Jun 9 17:37:54 2021] [139745] 0 139745 8477835 6588938 14006 0 660 nebula-http-gat [Wed Jun 9 17:37:54 2021] [328948] 0 328948 5732303 32531 3757 0 660 nebula-graphd [Wed Jun 9 17:37:54 2021] Memory cgroup out of memory: Kill process 357554 (nebula-http-gat) score 1423 or sacrifice child [Wed Jun 9 17:37:54 2021] Killed process 139745 (nebula-http-gat) total-vm:33911340kB, anon-rss:26355752kB, file-rss:0kB, shmem-rss:0kB [Wed Jun 9 17:37:54 2021] [139744] 0 139744 491094 809 68 0 660 nebula-importer [Wed Jun 9 17:37:54 2021] [328948] 0 328948 5732303 32531 3757 0 660 nebula-graphd [Tue Jun 15 20:15:07 2021] nebula-bgworker invoked oom-killer: gfp_mask=0xd0, order=0, oom_score_adj=660 [Tue Jun 15 20:15:07 2021] nebula-bgworker cpuset=2a3465bea197121fb6b6570c064393aad50b9a1351e1067ee5f6ac790050a040 mems_allowed=0-1 [Tue Jun 15 20:15:07 2021] CPU: 18 PID: 406150 Comm: nebula-bgworker Kdump: loaded Tainted: G OE ------------ 3.10.0-957.27.2.el7.x86_64 #1 [Tue Jun 15 20:15:07 2021] [328948] 0 328948 6780879 32319 3769 0 660 nebula-graphd [Tue Jun 15 20:15:07 2021] [406018] 0 406018 13590125 7292572 19700 0 660 nebula-storaged [Tue Jun 15 20:15:07 2021] [428695] 0 428695 431681 1154 60 0 660 nebula-importer [Tue Jun 15 20:15:07 2021] [428696] 0 428696 2075403 67356 2620 0 660 nebula-http-gat [Tue Jun 15 20:15:07 2021] Killed process 406018 (nebula-storaged) total-vm:54360500kB, anon-rss:29170288kB, file-rss:0kB, shmem-rss:0kB [Tue Jun 15 20:15:07 2021] [328948] 0 328948 6780879 32319 3769 0 660 nebula-graphd [Tue Jun 15 20:15:07 2021] [428695] 0 428695 431681 1154 60 0 660 nebula-importer [Tue Jun 15 20:15:07 2021] [428696] 0 428696 2075403 67356 2620 0 660 nebula-http-gat [Tue Jun 15 20:15:07 2021] [328948] 0 328948 6780879 32319 3769 0 660 nebula-graphd [Tue Jun 15 20:15:07 2021] [428695] 0 428695 431681 1154 60 0 660 nebula-importer [Tue Jun 15 20:15:07 2021] [428696] 0 428696 2075403 67356 2620 0 660 nebula-http-gat [Tue Jun 15 20:15:07 2021] Memory cgroup out of memory: Kill process 440889 (nebula-http-gat) score 668 or sacrifice child [Tue Jun 15 20:15:07 2021] Killed process 428696 (nebula-http-gat) total-vm:8301612kB, anon-rss:269424kB, file-rss:0kB, shmem-rss:0kB [Wed Jun 16 15:31:12 2021] [328948] 0 328948 6780879 32208 3769 0 660 nebula-graphd [Wed Jun 16 15:31:12 2021] [414642] 0 414642 10610528 7731050 17642 0 660 nebula-storaged [Wed Jun 16 15:31:12 2021] [444863] 0 444863 205171 348 24 0 660 nebula-importer [Wed Jun 16 15:31:12 2021] [444864] 0 444864 982470 11060 481 0 660 nebula-http-gat [Wed Jun 16 15:31:12 2021] Killed process 414642 (nebula-storaged) total-vm:42442112kB, anon-rss:30924200kB, file-rss:0kB, shmem-rss:0kB [Wed Jun 16 15:31:12 2021] [328948] 0 328948 6780879 32208 3769 0 660 nebula-graphd [Wed Jun 16 15:31:12 2021] [444863] 0 444863 205171 348 24 0 660 nebula-importer [Wed Jun 16 15:31:12 2021] [444864] 0 444864 982470 11060 481 0 660 nebula-http-gat [Wed Jun 16 15:31:12 2021] [328948] 0 328948 6780879 32208 3769 0 660 nebula-graphd [Wed Jun 16 15:31:12 2021] [444863] 0 444863 205171 348 24 0 660 nebula-importer [Wed Jun 16 15:31:12 2021] [444864] 0 444864 982470 11060 481 0 660 nebula-http-gat [Wed Jun 23 19:37:18 2021] [328948] 0 328948 7835343 31978 3785 0 660 nebula-graphd [Wed Jun 23 19:37:18 2021] [444863] 0 444863 403075 1105 60 0 660 nebula-importer [Wed Jun 23 19:37:18 2021] [444864] 0 444864 982854 12961 483 0 660 nebula-http-gat [Wed Jun 23 19:37:18 2021] [227318] 0 227318 14094964 7341691 18469 0 660 nebula-storaged [Wed Jun 23 19:37:18 2021] Killed process 227318 (nebula-storaged) total-vm:56379856kB, anon-rss:29366764kB, file-rss:0kB, shmem-rss:0kB [Wed Jun 23 19:37:19 2021] [328948] 0 328948 7835343 31978 3785 0 660 nebula-graphd [Wed Jun 23 19:37:19 2021] [444863] 0 444863 403075 1105 60 0 660 nebula-importer [Wed Jun 23 19:37:19 2021] [444864] 0 444864 982854 12961 483 0 660 nebula-http-gat [Wed Jun 23 19:37:19 2021] [328948] 0 328948 7835343 31978 3785 0 660 nebula-graphd [Wed Jun 23 19:37:19 2021] [444863] 0 444863 403075 1105 60 0 660 nebula-importer [Wed Jun 23 19:37:19 2021] [444864] 0 444864 982854 12961 483 0 660 nebula-http-gat [Wed Jun 23 19:37:19 2021] [328948] 0 328948 7835343 31978 3785 0 660 nebula-graphd [Wed Jun 23 19:37:19 2021] [444863] 0 444863 403075 1105 60 0 660 nebula-importer [Wed Jun 23 19:37:19 2021] [444864] 0 444864 982854 12961 483 0 660 nebula-http-gat [Thu Oct 7 10:15:50 2021] [20366] 0 20366 629150 1005 100 0 660 nebula-importer [Thu Oct 7 10:15:50 2021] [20367] 0 20367 953057 19086 594 0 660 nebula-httpd [Thu Oct 7 10:15:50 2021] [158172] 0 158172 18060166 7010110 29585 0 660 nebula-storaged [Thu Oct 7 10:15:50 2021] [158946] 0 158946 1280646 466414 2209 0 660 nebula-graphd [Thu Oct 7 10:15:50 2021] Killed process 158172 (nebula-storaged) total-vm:72240664kB, anon-rss:28040440kB, file-rss:0kB, shmem-rss:0kB [Thu Oct 7 10:15:50 2021] [20366] 0 20366 629150 1005 100 0 660 nebula-importer [Thu Oct 7 10:15:50 2021] [20367] 0 20367 953057 19086 594 0 660 nebula-httpd [Thu Oct 7 10:15:50 2021] [158946] 0 158946 1280646 466543 2209 0 660 nebula-graphd [Thu Oct 7 10:15:50 2021] [20366] 0 20366 629150 1005 100 0 660 nebula-importer [Thu Oct 7 10:15:50 2021] [20367] 0 20367 953057 19086 594 0 660 nebula-httpd [Thu Oct 7 10:15:50 2021] [158946] 0 158946 1280646 466543 2209 0 660 nebula-graphd [Thu Oct 7 10:15:50 2021] [20366] 0 20366 629150 1005 100 0 660 nebula-importer [Thu Oct 7 10:15:50 2021] [20367] 0 20367 953057 19086 594 0 660 nebula-httpd [Thu Oct 7 10:15:50 2021] [158946] 0 158946 1280646 466664 2209 0 660 nebula-graphd [Thu Oct 7 10:15:50 2021] Killed process 158946 (nebula-graphd) total-vm:5122584kB, anon-rss:1866656kB, file-rss:0kB, shmem-rss:0kB [root@10-128-31-23.okg-db-graphd.cp01 ~]#