Comments (8)
martinsumner
commented on August 20, 2024
([email protected])4> ProcFun8 = fun(P) -> process_info(P, [binary, dictionary]) end.
#Fun<erl_eval.6.80484245>
([email protected])5> BM8 = lists:map(ProcFun8, PL1).
* 1: variable 'PL1' is unbound
([email protected])6> PL1 = processes() -- [self()].
[<0.0.0>,<0.0.2649>,<0.1.2753>,<0.2.2098>,<0.3.0>,<0.6.0>,
<0.7.0>,<0.7.91>,<0.9.0>,<0.10.0>,<0.11.0>,<0.11.2905>,
<0.11.3797>,<0.12.0>,<0.13.0>,<0.14.0>,<0.15.0>,<0.16.0>,
<0.17.2763>,<0.18.0>,<0.19.0>,<0.19.875>,<0.20.0>,
<0.20.4194>,<0.21.0>,<0.22.0>,<0.22.2533>,<0.23.0>,<0.24.0>|...]
([email protected])7> BM8 = lists:map(ProcFun8, PL1).
[[{binary,[{139823559737576,87,1},{139823559737448,84,1}]},
{dictionary,[]}],
[{binary,[{139808459837480,280,1},
{139801014952912,280,1},
{139801014952592,280,1},
{139801014952272,280,1},
{139809177094184,280,1},
{139809177093864,280,1},
{139809177093544,280,1},
{139809177093224,280,1},
{139802293643112,280,1},
{139802293642792,280,1},
{139802293642472,280,1},
{139802293642152,280,1},
{139800721671432,280,1},
{139800721671112,280,1},
{139800721670792,280,1},
{139811778210928,280,1},
{139811778210608,280,1},
{139811778210288,280,1},
{139800048539920,280,1},
{139800048539600,280,1},
{139800048539280,280,1},
{139800048820792,280,...},
{139800048820472,...},
{...}|...]},
{dictionary,[{random_seed,{29488,2847,19820}},
{'$ancestors',[<0.785.0>,<0.772.0>,<0.706.0>,<0.663.0>,
riak_core_vnode_sup,riak_core_sup,<0.166.0>]},
{'$initial_call',{leveled_sst,init,1}}]}],
[{binary,[{139801014428000,280,1},
{139810511749128,280,1},
{139805871272824,280,1},
{139798396295600,280,1},
{139800101623352,280,1},
{139798670896040,280,1},
{139800510711112,280,1},
{139800443017648,280,1},
{139800443017328,280,1},
{139800443017008,280,1},
{139800443016688,280,1},
{139800443016368,280,1},
{139799186325536,280,1},
{139799186325216,280,1},
{139799186324896,280,1},
{139799186324576,280,1},
{139799186324256,280,1},
{139799415345424,280,1},
{139799415345104,280,1},
{139799415344784,280,1},
{139799415344464,280,...},
{139799415344144,...},
{...}|...]},
{dictionary,[{random_seed,{28175,13302,16361}},
{'$ancestors',[<0.1504.0>,<0.1503.0>,<0.1438.0>,<0.1300.0>,
riak_core_vnode_sup,riak_core_sup,<0.166.0>]},
{'$initial_call',{leveled_sst,init,1}}]}],
[{binary,[{139802061964704,280,1},
{139801495629696,280,1},
{139801495629376,280,1},
{139816202822328,280,1},
{139816202822008,280,1},
{139803106021976,280,1},
{139803106021656,280,1},
{139802124006416,280,1},
{139802124006096,280,1},
{139806063706696,280,1},
{139802126249544,280,1},
{139807608154680,280,1},
{139817563750616,280,1},
{139817563750296,280,1},
{139807608154360,280,1},
{139802607793776,280,1},
{139802617285888,280,1},
{139801978470904,280,1},
{139801877093432,280,1},
{139805643337304,280,...},
{139803372959496,...},
{...}|...]},
{dictionary,[{random_seed,{19551,15211,16472}},
{'$ancestors',[<0.1158.0>,<0.1157.0>,<0.1076.0>,<0.1026.0>,
riak_core_vnode_sup,riak_core_sup,<0.166.0>]},
{'$initial_call',{leveled_sst,init,1}}]}],
[{binary,[{139815315578120,65,1},
{139789590270048,665,1},
{139798945437152,70,1},
{139794848596104,724,1}]},
{dictionary,[]}],
[{binary,[]},
{dictionary,[{'$ancestors',[<0.2.0>]},
{'$initial_call',{gen_event,init_it,6}}]}],
[{binary,[]},
{dictionary,[{'$ancestors',[<0.2.0>]},
{'$initial_call',{application_controller,start,1}}]}],
[{binary,[]},
{dictionary,[{'$ancestors',[exometer_admin,
exometer_core_sup,<0.157.0>]},
{{exometer_proc,info},
{info,<0.159.0>,[],
{sys,undefined,exometer_probe,
[riak,riak_kv,node,gets,read_repairs|...]}}},
{'$initial_call',{exometer_proc,'-spawn_process/2-fun-0-',
0}}]}],
[{binary,[]},
{dictionary,[{'$ancestors',[<0.8.0>]},
{'$initial_call',{application_master,init,4}}]}],
[{binary,[]},{dictionary,[]}],
[{binary,[]},
{dictionary,[{'$ancestors',[<0.10.0>]},
{'$initial_call',{supervisor,kernel,1}}]}],
[{binary,[{139810815085640,280,1},
{139808500896456,280,1},
{139799925096504,280,1},
{139797033947440,280,1},
{139798281388472,280,1},
{139797033947120,280,1},
{139799925096184,280,1},
{139797031271600,280,1},
{139797031271280,280,1},
{139797031270960,280,1},
{139797031270640,280,1},
{139796812932632,280,...},
{139796812932312,...},
{...}|...]},
{dictionary,[{random_seed,{916,9853,10475}},
{'$ancestors',[<0.791.0>,<0.771.0>,<0.699.0>,<0.654.0>,
riak_core_vnode_sup,riak_core_sup,<0.166.0>]},
{'$initial_call',{leveled_sst,init,1}}]}],
[{binary,[{139793185077232,280,1},
{139795392964400,280,1},
{139797106399928,280,1},
{139795390562856,280,1},
{139794488113432,280,1},
{139793610657328,280,1},
{139794956078360,280,1},
{139811497520616,280,1},
{139791499450552,280,1},
{139794718649520,280,1},
{139793571578760,280,...},
{139803054430880,...},
{...}|...]},
{dictionary,[{random_seed,{16873,9914,28119}},
{'$ancestors',[<0.1158.0>,<0.1157.0>,<0.1076.0>,<0.1026.0>,
riak_core_vnode_sup,riak_core_sup,<0.166.0>]},
{'$initial_call',{leveled_sst,init,1}}]}],
[{binary,[]},
{dictionary,[{'$ancestors',[kernel_sup,<0.10.0>]},
{'$initial_call',{rpc,init,1}}]}],
[{binary,[]},
{dictionary,[{'$ancestors',[kernel_sup,<0.10.0>]},
{'$initial_call',{global,init,1}}]}],
[{binary,[]},{dictionary,[]}],
[{binary,[]},{dictionary,[]}],
[{binary,[]},
{dictionary,[{'$ancestors',[kernel_sup,<0.10.0>]},
{'$initial_call',{inet_db,init,1}}]}],
[{binary,[{139799560590040,280,1},
{139802486803248,280,1},
{139810861316360,280,1},
{139800872136824,280,1},
{139806291244792,280,...},
{139803043005992,...},
{...}|...]},
{dictionary,[{random_seed,{699,21190,19162}},
{'$ancestors',[<0.1139.0>,<0.1137.0>|...]},
{'$initial_call',{leveled_sst,...}}]}],
[{binary,[]},
{dictionary,[{'$ancestors',[kernel_sup,<0.10.0>]},
{'$initial_call',{supervisor,...}}]}],
[{binary,[]},
{dictionary,[{'$ancestors',[net_sup|...]},
{'$initial_call',{...}}]}],
[{binary,[{139816926648648,280,1},
{139813170963232,280,...},
{139814096561912,...},
{...}|...]},
{dictionary,[{random_seed,{...}},{'$ancestors',...},{...}]}],
[{binary,[]},{dictionary,[{'$ancestors',...},{...}]}],
[{binary,[{139789413458440,...},{...}|...]},
{dictionary,[{...}|...]}],
[{binary,[]},{dictionary,[...]}],
[{binary,[]},{dictionary,...}],
[{binary,...},{...}],
[{...}|...],
[...]|...]
([email protected])8> BinTotFun8 = fun([{binary, BL}, {dictionary, DL}]) -> {DL, lists:foldl(fun({_, Mem, _}, Acc) -> Acc + Mem end, 0, BL)} end.
#Fun<erl_eval.6.80484245>
([email protected])9> Out8 = lists:map(BinTotFun8, BM8).
[{[],171},
{[{random_seed,{29488,2847,19820}},
{'$ancestors',[<0.785.0>,<0.772.0>,<0.706.0>,<0.663.0>,
riak_core_vnode_sup,riak_core_sup,<0.166.0>]},
{'$initial_call',{leveled_sst,init,1}}],
104448},
{[{random_seed,{28175,13302,16361}},
{'$ancestors',[<0.1504.0>,<0.1503.0>,<0.1438.0>,<0.1300.0>,
riak_core_vnode_sup,riak_core_sup,<0.166.0>]},
{'$initial_call',{leveled_sst,init,1}}],
104448},
{[{random_seed,{19551,15211,16472}},
{'$ancestors',[<0.1158.0>,<0.1157.0>,<0.1076.0>,<0.1026.0>,
riak_core_vnode_sup,riak_core_sup,<0.166.0>]},
{'$initial_call',{leveled_sst,init,1}}],
104448},
{[],1524},
{[{'$ancestors',[<0.2.0>]},
{'$initial_call',{gen_event,init_it,6}}],
0},
{[{'$ancestors',[<0.2.0>]},
{'$initial_call',{application_controller,start,1}}],
0},
{[{'$ancestors',[exometer_admin,exometer_core_sup,
<0.157.0>]},
{{exometer_proc,info},
{info,<0.159.0>,[],
{sys,undefined,exometer_probe,
[riak,riak_kv,node,gets,read_repairs,
'[email protected]',primary,
outofdate]}}},
{'$initial_call',{exometer_proc,'-spawn_process/2-fun-0-',
0}}],
0},
{[{'$ancestors',[<0.8.0>]},
{'$initial_call',{application_master,init,4}}],
0},
{[],0},
{[{'$ancestors',[<0.10.0>]},
{'$initial_call',{supervisor,kernel,1}}],
0},
{[{random_seed,{916,9853,10475}},
{'$ancestors',[<0.791.0>,<0.771.0>,<0.699.0>,<0.654.0>,
riak_core_vnode_sup,riak_core_sup,<0.166.0>]},
{'$initial_call',{leveled_sst,init,1}}],
104448},
{[{random_seed,{16873,9914,28119}},
{'$ancestors',[<0.1158.0>,<0.1157.0>,<0.1076.0>,<0.1026.0>,
riak_core_vnode_sup,riak_core_sup,<0.166.0>]},
{'$initial_call',{leveled_sst,init,1}}],
117023},
{[{'$ancestors',[kernel_sup,<0.10.0>]},
{'$initial_call',{rpc,init,1}}],
0},
{[{'$ancestors',[kernel_sup,<0.10.0>]},
{'$initial_call',{global,init,1}}],
0},
{[],0},
{[],0},
{[{'$ancestors',[kernel_sup,<0.10.0>]},
{'$initial_call',{inet_db,init,1}}],
0},
{[{random_seed,{699,21190,19162}},
{'$ancestors',[<0.1139.0>,<0.1137.0>,<0.1069.0>,<0.1035.0>,
riak_core_vnode_sup|...]},
{'$initial_call',{leveled_sst,init,1}}],
104448},
{[{'$ancestors',[kernel_sup,<0.10.0>]},
{'$initial_call',{supervisor,erl_distribution,1}}],
0},
{[{'$ancestors',[net_sup,kernel_sup,<0.10.0>]},
{'$initial_call',{erl_epmd,init,1}}],
0},
{[{random_seed,{29888,29165,7235}},
{'$ancestors',[<0.1520.0>,<0.1519.0>|...]},
{'$initial_call',{leveled_sst,...}}],
104448},
{[{'$ancestors',[net_sup,kernel_sup|...]},
{'$initial_call',{auth,...}}],
0},
{[{'$ancestors',[<0.1520.0>|...]},{'$initial_call',{...}}],
71769},
{[{'$ancestors',[...]},{longnames,...},{...}],0},
{[],0},
{[{...}|...],104448},
{[],...},
{...}|...]
([email protected])10>
Another 30 minutes:
===============================================================================================================================
'[email protected]' 14:16:45
Load: cpu 88 Memory: total 5667911 binary 4856336
procs 2002 processes 713611 code 15918
runq 2 atom 719 ets 50194
Pid Name or Initial Func Time Reds Memory MsgQ Current Function
-------------------------------------------------------------------------------------------------------------------------------
<6939.764.0> leveled_penciller:init/1 '-' 11762076 41263840 0 gen_server:loop/6
<6939.773.0> leveled_penciller:init/1 '-' 12210912 28655728 0 gen_server:loop/6
<6939.1843.0> leveled_penciller:init/1 '-' 11383114 23880072 0 gen:do_call/4
<6939.776.0> leveled_penciller:init/1 '-' 9663572 23879928 0 gen_server:loop/6
<6939.1159.0> leveled_penciller:init/1 '-' 10986471 23879928 0 gen_server:loop/6
<6939.1493.0> leveled_penciller:init/1 '-' 10599386 23879928 0 gen_server:loop/6
<6939.1513.0> leveled_penciller:init/1 '-' 13158039 23879928 0 gen_server:loop/6
<6939.1141.0> leveled_penciller:init/1 '-' 8415712 19900096 0 gen_server:loop/6
<6939.1186.704> leveled_penciller:init/1 '-' 0 19900096 0 gen_server:loop/6
<6939.1521.0> leveled_penciller:init/1 '-' 11993679 19900096 0 gen_server:loop/6
===============================================================================================================================
'[email protected]' 14:17:45
Load: cpu 88 Memory: total 5378872 binary 4583662
procs 2023 processes 694059 code 15918
runq 5 atom 719 ets 50215
Pid Name or Initial Func Time Reds Memory MsgQ Current Function
-------------------------------------------------------------------------------------------------------------------------------
<6939.1513.0> leveled_penciller:init/1 '-' 15206835 49516424 0 gen_server:loop/6
<6939.1843.0> leveled_penciller:init/1 '-' 13124327 34386832 0 gen:do_call/4
<6939.776.0> leveled_penciller:init/1 '-' 10752976 28655728 0 gen_server:loop/6
<6939.1141.0> leveled_penciller:init/1 '-' 9617627 28655728 0 gen_server:loop/6
<6939.1521.0> leveled_penciller:init/1 '-' 14395115 28655728 0 gen_server:loop/6
<6939.1186.704> leveled_penciller:init/1 '-' 0 19900096 0 gen_server:loop/6
<6939.1153.0> leveled_penciller:init/1 '-' 7532119 16583568 0 gen_server:loop/6
<6939.21632.622> leveled_penciller:init/1 '-' 0 16583568 0 gen_server:loop/6
<6939.1884.0> leveled_penciller:init/1 '-' 8314558 13819792 0 gen_server:loop/6
<6939.9434.838> leveled_penciller:init/1 '-' 0 13819792 0 gen_server:loop/6
===============================================================================================================================
'[email protected]' 14:18:44
Load: cpu 88 Memory: total 4599838 binary 3875696
procs 2052 processes 614380 code 15918
runq 13 atom 719 ets 58453
Pid Name or Initial Func Time Reds Memory MsgQ Current Function
-------------------------------------------------------------------------------------------------------------------------------
<6939.1141.0> leveled_penciller:init/1 '-' 11522281 34386688 0 gen_server:loop/6
<6939.1884.0> leveled_penciller:init/1 '-' 9907293 23879928 0 gen_server:loop/6
<6939.1153.0> leveled_penciller:init/1 '-' 9320885 19900096 0 gen_server:loop/6
<6939.1186.704> leveled_penciller:init/1 '-' 0 19900096 0 gen_server:loop/6
<6939.1889.0> leveled_penciller:init/1 '-' 7461791 16583568 0 gen_server:loop/6
<6939.21632.622> leveled_penciller:init/1 '-' 0 16583568 0 gen_server:loop/6
<6939.764.0> leveled_penciller:init/1 '-' 7480932 13819792 0 gen_server:loop/6
<6939.9434.838> leveled_penciller:init/1 '-' 0 13819792 0 gen_server:loop/6
<6939.1493.0> leveled_penciller:init/1 '-' 6463530 11516648 0 gen_server:loop/6
<6939.15575.195> leveled_penciller:init/1 '-' 0 11516648 0 gen_server:loop/6
from leveled.
martinsumner
commented on August 20, 2024
Re-running after change the Inker clerk so that it starts a temporary assistant to manage each compaction.
After 30 minutes running
===============================================================================================================================
'[email protected]' 13:18:54
Load: cpu 82 Memory: total 3526570 binary 3004856
procs 1230 processes 408972 code 15918
runq 0 atom 719 ets 64952
Pid Name or Initial Func Time Reds Memory MsgQ Current Function
-------------------------------------------------------------------------------------------------------------------------------
<6941.1493.0> leveled_penciller:init/1 '-' 1230077 23879928 0 gen_server:loop/6
<6941.1513.0> leveled_penciller:init/1 '-' 1037029 34386688 0 gen_server:loop/6
<6941.764.0> leveled_penciller:init/1 '-' 1011927 28655728 0 gen_server:loop/6
<6941.1521.0> leveled_penciller:init/1 '-' 887950 16583672 1 gen_server:loop/6
<6941.1884.0> leveled_penciller:init/1 '-' 859656 23879928 0 gen_server:loop/6
<6941.8714.308> leveled_cdb:init/1 '-' 820659 13728 0 gen_fsm:loop/7
<6941.1045.0> riak_core_vnode:init/1 '-' 793482 34880 5 gen:do_call/4
<6941.1018.0> riak_core_vnode:init/1 '-' 790688 21856 0 gen_fsm:loop/7
<6941.658.0> riak_core_vnode:init/1 '-' 789815 21992 2 gen:do_call/4
<6941.1211.0> riak_core_vnode:init/1 '-' 787896 21992 2 gen:do_call/4
===============================================================================================================================
'[email protected]' 13:18:58
Load: cpu 84 Memory: total 3621255 binary 3087907
procs 1231 processes 422494 code 15918
runq 1 atom 719 ets 65457
Pid Name or Initial Func Time Reds Memory MsgQ Current Function
-------------------------------------------------------------------------------------------------------------------------------
<6941.1493.0> leveled_penciller:init/1 '-' 1389726 23879928 0 gen_server:loop/6
<6941.1513.0> leveled_penciller:init/1 '-' 1180412 34386688 0 gen_server:loop/6
<6941.764.0> leveled_penciller:init/1 '-' 1071182 28655800 0 gen_server:loop/6
<6941.1884.0> leveled_penciller:init/1 '-' 887051 23879928 0 gen_server:loop/6
<6941.1521.0> leveled_penciller:init/1 '-' 843496 16583568 0 gen_server:loop/6
<6941.8714.308> leveled_cdb:init/1 '-' 826813 13728 0 gen_fsm:loop/7
<6941.658.0> riak_core_vnode:init/1 '-' 825352 21856 0 gen_fsm:loop/7
<6941.17895.309> leveled_cdb:init/1 '-' 820165 13728 0 gen_fsm:loop/7
<6941.1309.0> riak_core_vnode:init/1 '-' 804719 34880 5 gen:do_call/4
<6941.22078.329> leveled_cdb:init/1 '-' 803517 13728 0 gen_fsm:loop/7
This is already at 7.4% memory for beam, early on in the test.
Then after another 30 minutes (now 1 hour into the test):
===============================================================================================================================
'[email protected]' 13:45:09
Load: cpu 87 Memory: total 4362415 binary 3687495
procs 1583 processes 569867 code 15918
runq 10 atom 719 ets 52875
Pid Name or Initial Func Time Reds Memory MsgQ Current Function
-------------------------------------------------------------------------------------------------------------------------------
<6939.773.0> leveled_penciller:init/1 '-' 12114869 41263984 0 gen:do_call/4
<6939.1141.0> leveled_penciller:init/1 '-' 12429483 34386688 0 gen_server:loop/6
<6939.1521.0> leveled_penciller:init/1 '-' 12828866 34386688 0 gen_server:loop/6
<6939.1153.0> leveled_penciller:init/1 '-' 13258810 28655872 0 gen:do_call/4
<6939.1493.0> leveled_penciller:init/1 '-' 11314641 28655728 0 gen_server:loop/6
<6939.1843.0> leveled_penciller:init/1 '-' 11273293 28655728 0 gen_server:loop/6
<6939.1884.0> leveled_penciller:init/1 '-' 12205498 28655728 0 gen_server:loop/6
<6939.1513.0> leveled_penciller:init/1 '-' 8423506 16583568 0 gen_server:loop/6
<6939.1889.0> leveled_penciller:init/1 '-' 8288297 13819792 0 gen_server:loop/6
<6939.15575.195> leveled_penciller:init/1 '-' 0 11516648 0 gen_server:loop/6
===============================================================================================================================
'[email protected]' 13:46:09
Load: cpu 87 Memory: total 4950114 binary 4238023
procs 1587 processes 606753 code 15918
runq 11 atom 719 ets 58704
Pid Name or Initial Func Time Reds Memory MsgQ Current Function
-------------------------------------------------------------------------------------------------------------------------------
<6939.1521.0> leveled_penciller:init/1 '-' 15218531 49516424 0 gen_server:loop/6
<6939.1141.0> leveled_penciller:init/1 '-' 14326448 34386832 0 gen:do_call/4
<6939.1493.0> leveled_penciller:init/1 '-' 13479516 34386688 0 gen_server:loop/6
<6939.1843.0> leveled_penciller:init/1 '-' 13463341 34386688 0 gen_server:loop/6
<6939.1884.0> leveled_penciller:init/1 '-' 14421737 34386688 0 gen_server:loop/6
<6939.773.0> leveled_penciller:init/1 '-' 13953614 28655728 0 gen_server:loop/6
<6939.1889.0> leveled_penciller:init/1 '-' 9993834 28655728 0 gen_server:loop/6
<6939.1513.0> leveled_penciller:init/1 '-' 10452796 23879928 0 gen_server:loop/6
<6939.764.0> leveled_penciller:init/1 '-' 7890123 11516648 0 gen_server:loop/6
<6939.15575.195> leveled_penciller:init/1 '-' 0 11516648 0 gen_server:loop/6
===============================================================================================================================
'[email protected]' 13:47:08
Load: cpu 89 Memory: total 3129762 binary 2592897
procs 1613 processes 438487 code 15918
runq 6 atom 719 ets 48982
Pid Name or Initial Func Time Reds Memory MsgQ Current Function
-------------------------------------------------------------------------------------------------------------------------------
<6939.1513.0> leveled_penciller:init/1 '-' 12255606 28655800 0 gen_server:loop/6
<6939.764.0> leveled_penciller:init/1 '-' 9795208 23880072 0 gen:do_call/4
<6939.1889.0> leveled_penciller:init/1 '-' 12207526 23880032 1 gen_server:loop/6
<6939.1159.0> leveled_penciller:init/1 '-' 8551101 19900096 0 gen_server:loop/6
<6939.776.0> leveled_penciller:init/1 '-' 8686412 11516648 0 gen_server:loop/6
<6939.15575.195> leveled_penciller:init/1 '-' 0 11516648 0 gen_server:loop/6
<6939.1153.0> leveled_penciller:init/1 '-' 6769969 9597360 0 gen_server:loop/6
<6939.11262.295> leveled_penciller:init/1 '-' 0 9597360 0 gen_server:loop/6
<6939.21840.495> leveled_penciller:init/1 '-' 0 9597360 0 gen_server:loop/6
<6939.1147.138> leveled_penciller:init/1 '-' 0 6665112 0 gen_server:loop/6
Now beam is using 9% of RAM - but the internal recording of use is up/down in etop.
Now at 2.5 hours into the test. Note that compactions are still not being initiated (as they're just below the threshold score).
===============================================================================================================================
'[email protected]' 15:14:48
Load: cpu 88 Memory: total 6718914 binary 5647827
procs 3130 processes 969023 code 15918
runq 6 atom 719 ets 48411
Pid Name or Initial Func Time Reds Memory MsgQ Current Function
-------------------------------------------------------------------------------------------------------------------------------
<6939.764.0> leveled_penciller:init/1 '-' 10058090 28655872 0 gen:do_call/4
<6939.1159.0> leveled_penciller:init/1 '-' 13661918 28655728 0 gen_server:loop/6
<6939.776.0> leveled_penciller:init/1 '-' 12787045 23880104 1 io:wait_io_mon_reply/2
<6939.1493.0> leveled_penciller:init/1 '-' 10190506 23879928 0 gen_server:loop/6
<6939.1889.0> leveled_penciller:init/1 '-' 9770154 23879928 0 gen_server:loop/6
<6939.7751.1261> leveled_sst:init/1 '-' 0 19900136 0 gen_fsm:loop/7
<6939.1141.0> leveled_penciller:init/1 '-' 11688693 19900096 0 gen_server:loop/6
<6939.1186.704> leveled_penciller:init/1 '-' 0 19900096 0 gen_server:loop/6
<6939.1513.0> leveled_penciller:init/1 '-' 10854254 16583712 0 gen:do_call/4
<6939.1884.0> leveled_penciller:init/1 '-' 8356204 16583672 1 gen_server:loop/6
===============================================================================================================================
'[email protected]' 15:15:48
Load: cpu 88 Memory: total 6939698 binary 5811216
procs 3144 processes 1033438 code 15918
runq 3 atom 719 ets 52212
Pid Name or Initial Func Time Reds Memory MsgQ Current Function
-------------------------------------------------------------------------------------------------------------------------------
<6939.1159.0> leveled_penciller:init/1 '-' 15304157 49516528 1 gen_server:loop/6
<6939.1141.0> leveled_penciller:init/1 '-' 12783810 41263840 0 gen_server:loop/6
<6939.1493.0> leveled_penciller:init/1 '-' 11354398 34386832 0 gen:do_call/4
<6939.764.0> leveled_penciller:init/1 '-' 11349346 34386688 0 gen_server:loop/6
<6939.1513.0> leveled_penciller:init/1 '-' 11554348 23880104 1 gen:do_call/4
<6939.1889.0> leveled_penciller:init/1 '-' 11792318 23879928 0 gen_server:loop/6
<6939.773.0> leveled_penciller:init/1 '-' 8729132 19900240 0 gen:do_call/4
<6939.7751.1261> leveled_sst:init/1 '-' 16 19900136 0 gen_fsm:loop/7
<6939.1186.704> leveled_penciller:init/1 '-' 0 19900096 0 gen_server:loop/6
<6939.1153.0> leveled_penciller:init/1 '-' 6750081 16583712 0 gen:do_call/4
===============================================================================================================================
'[email protected]' 15:16:47
Load: cpu 89 Memory: total 6824149 binary 5706816
procs 3161 processes 1022111 code 15918
runq 10 atom 719 ets 47545
Pid Name or Initial Func Time Reds Memory MsgQ Current Function
-------------------------------------------------------------------------------------------------------------------------------
<6939.1141.0> leveled_penciller:init/1 '-' 13340484 49516424 0 gen_server:loop/6
<6939.1889.0> leveled_penciller:init/1 '-' 12624280 41263840 0 gen_server:loop/6
<6939.764.0> leveled_penciller:init/1 '-' 12435187 28655872 0 gen:do_call/4
<6939.773.0> leveled_penciller:init/1 '-' 9949071 28655872 0 gen:do_call/4
<6939.1513.0> leveled_penciller:init/1 '-' 13123416 28655728 0 gen_server:loop/6
<6939.1153.0> leveled_penciller:init/1 '-' 8014141 23880072 0 gen:do_call/4
<6939.7751.1261> leveled_sst:init/1 '-' 16 19900136 0 gen_fsm:loop/7
<6939.1186.704> leveled_penciller:init/1 '-' 0 19900096 0 gen_server:loop/6
<6939.1843.0> leveled_penciller:init/1 '-' 7758890 16583568 0 gen_server:loop/6
<6939.21632.622> leveled_penciller:init/1 '-' 0 16583568 0 gen_server:loop/6
Stop load, then restart:
===============================================================================================================================
'[email protected]' 15:36:06
Load: cpu 0 Memory: total 1246263 binary 897933
procs 1541 processes 262963 code 14286
runq 0 atom 639 ets 39955
Pid Name or Initial Func Time Reds Memory MsgQ Current Function
-------------------------------------------------------------------------------------------------------------------------------
<6939.166.0> application_master:start_ '-' 1544842 4119584 0 application_master:loop_it/4
<6939.1070.0> leveled_pclerk:init/1 '-' 8218409 2546400 0 gen_server:loop/6
<6939.1108.0> leveled_penciller:init/1 '-' 134761 2546400 0 gen_server:loop/6
<6939.1178.0> leveled_penciller:init/1 '-' 136246 2546400 0 gen_server:loop/6
<6939.1189.0> leveled_pclerk:init/1 '-' 13298943 2546400 0 gen_server:loop/6
<6939.1259.0> leveled_penciller:init/1 '-' 144138 2546400 0 gen_server:loop/6
<6939.1260.0> leveled_pclerk:init/1 '-' 8352840 2546400 0 gen_server:loop/6
<6939.1378.0> leveled_penciller:init/1 '-' 148666 2546400 0 gen_server:loop/6
<6939.1160.0> leveled_sst:init/1 '-' 146218 1574144 0 gen_fsm:loop/7
<6939.1250.0> leveled_sst:init/1 '-' 138905 1574144 0 gen_fsm:loop/7
from leveled.
martinsumner
commented on August 20, 2024
after the restart, start the load again:
===============================================================================================================================
'[email protected]' 15:36:06
Load: cpu 0 Memory: total 1246263 binary 897933
procs 1541 processes 262963 code 14286
runq 0 atom 639 ets 39955
Pid Name or Initial Func Time Reds Memory MsgQ Current Function
-------------------------------------------------------------------------------------------------------------------------------
<6939.166.0> application_master:start_ '-' 1544842 4119584 0 application_master:loop_it/4
<6939.1070.0> leveled_pclerk:init/1 '-' 8218409 2546400 0 gen_server:loop/6
<6939.1108.0> leveled_penciller:init/1 '-' 134761 2546400 0 gen_server:loop/6
<6939.1178.0> leveled_penciller:init/1 '-' 136246 2546400 0 gen_server:loop/6
<6939.1189.0> leveled_pclerk:init/1 '-' 13298943 2546400 0 gen_server:loop/6
<6939.1259.0> leveled_penciller:init/1 '-' 144138 2546400 0 gen_server:loop/6
<6939.1260.0> leveled_pclerk:init/1 '-' 8352840 2546400 0 gen_server:loop/6
<6939.1378.0> leveled_penciller:init/1 '-' 148666 2546400 0 gen_server:loop/6
<6939.1160.0> leveled_sst:init/1 '-' 146218 1574144 0 gen_fsm:loop/7
<6939.1250.0> leveled_sst:init/1 '-' 138905 1574144 0 gen_fsm:loop/7
===============================================================================================================================
'[email protected]' 15:37:05
Load: cpu 0 Memory: total 1322101 binary 968053
procs 2189 processes 267936 code 14655
runq 0 atom 679 ets 40555
Pid Name or Initial Func Time Reds Memory MsgQ Current Function
-------------------------------------------------------------------------------------------------------------------------------
<6939.1178.0> leveled_penciller:init/1 '-' 13600 4119608 0 gen_server:loop/6
<6939.1595.0> leveled_penciller:init/1 '-' 126001 4119608 0 gen_server:loop/6
<6939.1632.0> leveled_penciller:init/1 '-' 119850 4119608 0 gen_server:loop/6
<6939.166.0> application_master:start_ '-' 0 4119584 0 application_master:loop_it/4
<6939.1070.0> leveled_pclerk:init/1 '-' 390 2546400 0 gen_server:loop/6
<6939.1108.0> leveled_penciller:init/1 '-' 1050 2546400 0 gen_server:loop/6
<6939.1189.0> leveled_pclerk:init/1 '-' 377 2546400 0 gen_server:loop/6
<6939.1259.0> leveled_penciller:init/1 '-' 47210 2546400 0 gen_server:loop/6
<6939.1260.0> leveled_pclerk:init/1 '-' 390 2546400 0 gen_server:loop/6
<6939.1378.0> leveled_penciller:init/1 '-' 59980 2546400 0 gen_server:loop/6
===============================================================================================================================
'[email protected]' 15:38:05
Load: cpu 0 Memory: total 1326499 binary 968557
procs 2449 processes 271835 code 14680
runq 0 atom 679 ets 40468
Pid Name or Initial Func Time Reds Memory MsgQ Current Function
-------------------------------------------------------------------------------------------------------------------------------
<6939.1178.0> leveled_penciller:init/1 '-' 1050 4119608 0 gen_server:loop/6
<6939.1595.0> leveled_penciller:init/1 '-' 1015 4119608 0 gen_server:loop/6
<6939.1632.0> leveled_penciller:init/1 '-' 1050 4119608 0 gen_server:loop/6
<6939.166.0> application_master:start_ '-' 0 4119584 0 application_master:loop_it/4
<6939.1070.0> leveled_pclerk:init/1 '-' 390 2546400 0 gen_server:loop/6
<6939.1108.0> leveled_penciller:init/1 '-' 1015 2546400 0 gen_server:loop/6
<6939.1189.0> leveled_pclerk:init/1 '-' 390 2546400 0 gen_server:loop/6
<6939.1259.0> leveled_penciller:init/1 '-' 1050 2546400 0 gen_server:loop/6
<6939.1260.0> leveled_pclerk:init/1 '-' 390 2546400 0 gen_server:loop/6
<6939.1378.0> leveled_penciller:init/1 '-' 1050 2546400 0 gen_server:loop/6
===============================================================================================================================
'[email protected]' 15:39:05
Load: cpu 67 Memory: total 1272480 binary 916482
procs 1679 processes 265585 code 15692
runq 3 atom 687 ets 41867
Pid Name or Initial Func Time Reds Memory MsgQ Current Function
-------------------------------------------------------------------------------------------------------------------------------
<6939.1188.0> leveled_penciller:init/1 '-' 3579906 6665256 0 gen:do_call/4
<6939.1108.0> leveled_penciller:init/1 '-' 3864938 4119608 0 gen_server:loop/6
<6939.1178.0> leveled_penciller:init/1 '-' 3752768 4119608 0 gen_server:loop/6
<6939.1259.0> leveled_penciller:init/1 '-' 4170802 4119608 0 gen_server:loop/6
<6939.1307.0> leveled_penciller:init/1 '-' 3483376 4119608 0 gen_server:loop/6
<6939.1378.0> leveled_penciller:init/1 '-' 4006027 4119608 0 gen_server:loop/6
<6939.1553.0> leveled_penciller:init/1 '-' 3934522 4119608 0 gen_server:loop/6
<6939.1595.0> leveled_penciller:init/1 '-' 4093998 4119608 0 gen_server:loop/6
<6939.1632.0> leveled_penciller:init/1 '-' 4132536 4119608 0 gen_server:loop/6
<6939.166.0> application_master:start_ '-' 0 4119584 0 application_master:loop_it/4
===============================================================================================================================
'[email protected]' 15:40:04
Load: cpu 88 Memory: total 2109470 binary 1611083
procs 1619 processes 410850 code 15692
runq 9 atom 687 ets 42474
Pid Name or Initial Func Time Reds Memory MsgQ Current Function
-------------------------------------------------------------------------------------------------------------------------------
<6939.1595.0> leveled_penciller:init/1 '-' 6669082 16583640 0 gen_server:loop/6
<6939.1108.0> leveled_penciller:init/1 '-' 5846397 13819936 0 gen:do_call/4
<6939.1632.0> leveled_penciller:init/1 '-' 6487819 13819936 0 gen:do_call/4
<6939.1553.0> leveled_penciller:init/1 '-' 6312739 11516824 1 gen:do_call/4
<6939.1259.0> leveled_penciller:init/1 '-' 6630790 11516792 0 gen:do_call/4
<6939.1307.0> leveled_penciller:init/1 '-' 5698638 11516648 0 gen_server:loop/6
<6939.1378.0> leveled_penciller:init/1 '-' 6307640 11516648 0 gen_server:loop/6
<6939.1465.0> leveled_penciller:init/1 '-' 6230761 11516648 0 gen_server:loop/6
<6939.1178.0> leveled_penciller:init/1 '-' 6354310 9597536 1 gen:do_call/4
<6939.1069.0> leveled_penciller:init/1 '-' 6229597 9597504 0 gen:do_call/4
===============================================================================================================================
'[email protected]' 15:41:04
Load: cpu 88 Memory: total 2863781 binary 2297959
procs 1618 processes 459709 code 15692
runq 14 atom 687 ets 55221
Pid Name or Initial Func Time Reds Memory MsgQ Current Function
-------------------------------------------------------------------------------------------------------------------------------
<6939.1401.0> leveled_penciller:init/1 '-' 7579171 19900096 0 gen_server:loop/6
<6939.1465.0> leveled_penciller:init/1 '-' 7456935 19900096 0 gen_server:loop/6
<6939.1108.0> leveled_penciller:init/1 '-' 7412606 16583712 0 gen:do_call/4
<6939.1069.0> leveled_penciller:init/1 '-' 7308190 16583568 0 gen_server:loop/6
<6939.1178.0> leveled_penciller:init/1 '-' 7278689 16583568 0 gen_server:loop/6
<6939.1259.0> leveled_penciller:init/1 '-' 7963526 16583568 0 gen_server:loop/6
<6939.1378.0> leveled_penciller:init/1 '-' 7579770 16583568 0 gen_server:loop/6
<6939.1553.0> leveled_penciller:init/1 '-' 7458619 16583568 0 gen_server:loop/6
<6939.1595.0> leveled_penciller:init/1 '-' 7879662 16583568 0 gen_server:loop/6
<6939.1632.0> leveled_penciller:init/1 '-' 7711171 13819792 0 gen_server:loop/6
===============================================================================================================================
'[email protected]' 15:42:03
Load: cpu 89 Memory: total 3618586 binary 2987264
procs 1620 processes 533085 code 15692
runq 0 atom 687 ets 48546
Pid Name or Initial Func Time Reds Memory MsgQ Current Function
-------------------------------------------------------------------------------------------------------------------------------
<6939.1553.0> leveled_penciller:init/1 '-' 8705445 28655728 0 gen_server:loop/6
<6939.1178.0> leveled_penciller:init/1 '-' 8461192 23879928 0 gen_server:loop/6
<6939.1259.0> leveled_penciller:init/1 '-' 8820445 23879928 0 gen_server:loop/6
<6939.1378.0> leveled_penciller:init/1 '-' 8765942 23879928 0 gen_server:loop/6
<6939.1465.0> leveled_penciller:init/1 '-' 8523984 23879928 0 gen_server:loop/6
<6939.1595.0> leveled_penciller:init/1 '-' 8697428 23879928 0 gen_server:loop/6
<6939.1632.0> leveled_penciller:init/1 '-' 9126851 23879928 0 gen_server:loop/6
<6939.1188.0> leveled_penciller:init/1 '-' 8496312 19900200 1 gen_server:loop/6
<6939.1069.0> leveled_penciller:init/1 '-' 8414970 19900096 0 gen_server:loop/6
<6939.1307.0> leveled_penciller:init/1 '-' 7982036 19900096 0 gen_server:loop/6
===============================================================================================================================
'[email protected]' 15:43:03
Load: cpu 89 Memory: total 4375789 binary 3664650
procs 1631 processes 611795 code 15692
runq 6 atom 687 ets 50669
Pid Name or Initial Func Time Reds Memory MsgQ Current Function
-------------------------------------------------------------------------------------------------------------------------------
<6939.1595.0> leveled_penciller:init/1 '-' 9980089 41263840 0 gen_server:loop/6
<6939.1632.0> leveled_penciller:init/1 '-' 8136472 34386760 0 gen_server:loop/6
<6939.1259.0> leveled_penciller:init/1 '-' 9764615 34386688 0 gen_server:loop/6
<6939.1401.0> leveled_penciller:init/1 '-' 9741561 34386688 0 gen_server:loop/6
<6939.1378.0> leveled_penciller:init/1 '-' 9672702 28655872 0 gen:do_call/4
<6939.1465.0> leveled_penciller:init/1 '-' 9603231 28655872 0 gen:do_call/4
<6939.1553.0> leveled_penciller:init/1 '-' 9816173 28655872 0 gen:do_call/4
<6939.1178.0> leveled_penciller:init/1 '-' 9696122 23880072 0 gen:do_call/4
<6939.1069.0> leveled_penciller:init/1 '-' 9657140 23879928 0 gen_server:loop/6
<6939.1108.0> leveled_penciller:init/1 '-' 9936938 23879928 0 gen_server:loop/6
===============================================================================================================================
'[email protected]' 15:44:02
Load: cpu 89 Memory: total 4399240 binary 3711751
procs 1626 processes 581939 code 15692
runq 2 atom 687 ets 56615
Pid Name or Initial Func Time Reds Memory MsgQ Current Function
-------------------------------------------------------------------------------------------------------------------------------
<6939.1108.0> leveled_penciller:init/1 '-' 10623587 41263840 0 gen_server:loop/6
<6939.1378.0> leveled_penciller:init/1 '-' 10945328 34386832 0 gen:do_call/4
<6939.1069.0> leveled_penciller:init/1 '-' 10554834 34386688 0 gen_server:loop/6
<6939.1178.0> leveled_penciller:init/1 '-' 10512962 34386688 0 gen_server:loop/6
<6939.1188.0> leveled_penciller:init/1 '-' 10437297 34386688 0 gen_server:loop/6
<6939.1307.0> leveled_penciller:init/1 '-' 10378232 28655728 0 gen_server:loop/6
<6939.1465.0> leveled_penciller:init/1 '-' 10708127 28655728 0 gen_server:loop/6
<6939.1595.0> leveled_penciller:init/1 '-' 11015659 28655728 0 gen_server:loop/6
<6939.1401.0> leveled_penciller:init/1 '-' 10715454 23879928 0 gen_server:loop/6
<6939.1553.0> leveled_penciller:init/1 '-' 10706196 23879928 0 gen_server:loop/6
===============================================================================================================================
'[email protected]' 15:45:02
Load: cpu 90 Memory: total 4810859 binary 4072077
procs 1642 processes 655940 code 15692
runq 21 atom 687 ets 42782
Pid Name or Initial Func Time Reds Memory MsgQ Current Function
-------------------------------------------------------------------------------------------------------------------------------
<6939.1378.0> leveled_penciller:init/1 '-' 11067167 49516568 0 gen:do_call/4
<6939.1188.0> leveled_penciller:init/1 '-' 11402072 49516424 0 gen_server:loop/6
<6939.1595.0> leveled_penciller:init/1 '-' 12012806 41263984 0 gen:do_call/4
<6939.1178.0> leveled_penciller:init/1 '-' 11678635 41263840 0 gen_server:loop/6
<6939.1307.0> leveled_penciller:init/1 '-' 11174766 41263840 0 gen_server:loop/6
<6939.1401.0> leveled_penciller:init/1 '-' 11575632 41263840 0 gen_server:loop/6
<6939.1465.0> leveled_penciller:init/1 '-' 11842006 34386832 0 gen:do_call/4
<6939.1553.0> leveled_penciller:init/1 '-' 11708280 34386688 0 gen_server:loop/6
<6939.1069.0> leveled_penciller:init/1 '-' 11440593 28655832 1 gen_server:loop/6
<6939.1259.0> leveled_penciller:init/1 '-' 5030078 6665112 0 gen_server:loop/6
===============================================================================================================================
'[email protected]' 15:46:01
Load: cpu 92 Memory: total 1763149 binary 1296867
procs 1690 processes 378789 code 15692
runq 0 atom 687 ets 41402
Pid Name or Initial Func Time Reds Memory MsgQ Current Function
-------------------------------------------------------------------------------------------------------------------------------
<6939.1307.0> leveled_penciller:init/1 '-' 10174665 49516568 0 gen:do_call/4
<6939.1632.0> leveled_penciller:init/1 '-' 6365291 16583568 0 gen_server:loop/6
<6939.1259.0> leveled_penciller:init/1 '-' 5688364 11516648 0 gen_server:loop/6
<6939.1108.0> leveled_penciller:init/1 '-' 4776714 7998024 0 gen_server:loop/6
<6939.571.0> exometer_proc:'-spawn_pro '-' 3007446 6665184 0 exometer_probe:loop/1
<6939.3572.0> exometer_proc:'-spawn_pro '-' 2879775 6665184 0 exometer_probe:loop/1
<6939.1378.0> leveled_penciller:init/1 '-' 3937160 4119784 1 gen:do_call/4
<6939.1595.0> leveled_penciller:init/1 '-' 4922831 4119752 0 gen:do_call/4
<6939.1069.0> leveled_penciller:init/1 '-' 5095490 4119712 1 gen_server:loop/6
<6939.546.0> exometer_proc:'-spawn_pro '-' 1538401 4119680 0 exometer_probe:loop/1
===============================================================================================================================
'[email protected]' 15:47:01
Load: cpu 89 Memory: total 2022254 binary 1534867
procs 1636 processes 395109 code 15692
runq 17 atom 687 ets 42276
Pid Name or Initial Func Time Reds Memory MsgQ Current Function
-------------------------------------------------------------------------------------------------------------------------------
<6939.1632.0> leveled_penciller:init/1 '-' 8185205 23880032 1 gen_server:loop/6
<6939.1259.0> leveled_penciller:init/1 '-' 7448015 19900240 0 gen:do_call/4
<6939.1108.0> leveled_penciller:init/1 '-' 6341565 16583568 0 gen_server:loop/6
<6939.1553.0> leveled_penciller:init/1 '-' 5536025 9597504 0 gen:do_call/4
<6939.1178.0> leveled_penciller:init/1 '-' 5509690 9597464 1 gen_server:loop/6
<6939.1378.0> leveled_penciller:init/1 '-' 5540570 9597360 0 gen_server:loop/6
<6939.571.0> exometer_proc:'-spawn_pro '-' 3180845 7998056 1 exometer_probe:loop/1
<6939.1069.0> leveled_penciller:init/1 '-' 5566731 7997952 0 gen_server:loop/6
<6939.1595.0> leveled_penciller:init/1 '-' 5485061 7997952 0 gen_server:loop/6
<6939.1401.0> leveled_penciller:init/1 '-' 4911579 6665184 0 gen_server:loop/6
===============================================================================================================================
'[email protected]' 15:48:00
Load: cpu 89 Memory: total 2774364 binary 2216575
procs 1638 processes 469685 code 15692
runq 20 atom 687 ets 41649
Pid Name or Initial Func Time Reds Memory MsgQ Current Function
-------------------------------------------------------------------------------------------------------------------------------
<6939.1632.0> leveled_penciller:init/1 '-' 9228675 34386688 0 gen_server:loop/6
<6939.1259.0> leveled_penciller:init/1 '-' 8718346 28655872 0 gen:do_call/4
<6939.1108.0> leveled_penciller:init/1 '-' 7415775 19900096 0 gen_server:loop/6
<6939.1378.0> leveled_penciller:init/1 '-' 6577606 16583568 0 gen_server:loop/6
<6939.1069.0> leveled_penciller:init/1 '-' 6459144 13819792 0 gen_server:loop/6
<6939.1178.0> leveled_penciller:init/1 '-' 6443197 13819792 0 gen_server:loop/6
<6939.1465.0> leveled_penciller:init/1 '-' 6308713 13819792 0 gen_server:loop/6
<6939.1595.0> leveled_penciller:init/1 '-' 6377798 13819792 0 gen_server:loop/6
<6939.1553.0> leveled_penciller:init/1 '-' 6526398 11516648 0 gen_server:loop/6
<6939.1188.0> leveled_penciller:init/1 '-' 6004619 9597360 0 gen_server:loop/6
from leveled.
martinsumner
commented on August 20, 2024
30 minutes after restart of load:
===============================================================================================================================
'[email protected]' 16:07:51
Load: cpu 89 Memory: total 3227262 binary 2615665
procs 1766 processes 519283 code 15692
runq 4 atom 687 ets 44577
Pid Name or Initial Func Time Reds Memory MsgQ Current Function
-------------------------------------------------------------------------------------------------------------------------------
<6939.1553.0> leveled_penciller:init/1 '-' 11581132 41264016 1 gen:do_call/4
<6939.1259.0> leveled_penciller:init/1 '-' 8115225 28655728 0 gen_server:loop/6
<6939.1307.0> leveled_penciller:init/1 '-' 11193659 28655728 0 gen_server:loop/6
<6939.1632.0> leveled_penciller:init/1 '-' 9693028 28655728 0 gen_server:loop/6
<6939.25814.108> leveled_penciller:init/1 '-' 0 28655728 0 gen_server:loop/6
<6939.1108.0> leveled_penciller:init/1 '-' 7697013 16583568 0 gen_server:loop/6
<6939.1401.0> leveled_penciller:init/1 '-' 6041415 13819968 1 gen:do_call/4
<6939.5883.104> leveled_penciller:init/1 '-' 0 11516648 0 gen_server:loop/6
<6939.1378.0> leveled_penciller:init/1 '-' 5628977 9597360 0 gen_server:loop/6
<6939.571.0> exometer_proc:'-spawn_pro '-' 3070730 6665184 0 exometer_probe:loop/1
===============================================================================================================================
'[email protected]' 16:08:50
Load: cpu 89 Memory: total 2718965 binary 2148425
procs 1763 processes 471491 code 15692
runq 16 atom 687 ets 48432
Pid Name or Initial Func Time Reds Memory MsgQ Current Function
-------------------------------------------------------------------------------------------------------------------------------
<6939.1632.0> leveled_penciller:init/1 '-' 10801673 34386832 0 gen:do_call/4
<6939.25814.108> leveled_penciller:init/1 '-' 0 28655728 0 gen_server:loop/6
<6939.1108.0> leveled_penciller:init/1 '-' 9064055 23879928 0 gen_server:loop/6
<6939.1401.0> leveled_penciller:init/1 '-' 7282878 16583568 0 gen_server:loop/6
<6939.1378.0> leveled_penciller:init/1 '-' 6737633 13819792 0 gen_server:loop/6
<6939.5883.104> leveled_penciller:init/1 '-' 0 11516648 0 gen_server:loop/6
<6939.2865.188> leveled_penciller:init/1 '-' 371018 9597504 0 gen:do_call/4
<6939.1069.0> leveled_penciller:init/1 '-' 5273273 9597360 0 gen_server:loop/6
<6939.1188.0> leveled_penciller:init/1 '-' 5316513 9597360 0 gen_server:loop/6
<6939.1178.0> leveled_penciller:init/1 '-' 5719378 7997952 0 gen_server:loop/6
===============================================================================================================================
'[email protected]' 16:09:50
Load: cpu 91 Memory: total 3514414 binary 2822651
procs 1766 processes 578956 code 15692
runq 1 atom 687 ets 64714
Pid Name or Initial Func Time Reds Memory MsgQ Current Function
-------------------------------------------------------------------------------------------------------------------------------
<6939.1108.0> leveled_penciller:init/1 '-' 9732553 34386688 0 gen_server:loop/6
<6939.1632.0> leveled_penciller:init/1 '-' 11648210 34386688 0 gen_server:loop/6
<6939.1378.0> leveled_penciller:init/1 '-' 7982211 28655728 0 gen_server:loop/6
<6939.25814.108> leveled_penciller:init/1 '-' 0 28655728 0 gen_server:loop/6
<6939.1401.0> leveled_penciller:init/1 '-' 8080965 23879928 0 gen_server:loop/6
<6939.2865.188> leveled_penciller:init/1 '-' 9757215 23879928 0 gen_server:loop/6
<6939.1178.0> leveled_penciller:init/1 '-' 6665086 19900096 0 gen_server:loop/6
<6939.1188.0> leveled_penciller:init/1 '-' 6232072 16583568 0 gen_server:loop/6
<6939.1465.0> leveled_penciller:init/1 '-' 6957384 13819968 1 gen:do_call/4
<6939.1069.0> leveled_penciller:init/1 '-' 6296522 13819792 0 gen_server:loop/6
from leveled.
martinsumner
commented on August 20, 2024
Two hours on ...
===============================================================================================================================
'[email protected]' 17:57:24
Load: cpu 89 Memory: total 4241248 binary 3260079
procs 2941 processes 875105 code 15692
runq 3 atom 687 ets 52077
Pid Name or Initial Func Time Reds Memory MsgQ Current Function
-------------------------------------------------------------------------------------------------------------------------------
<6939.1188.0> leveled_penciller:init/1 '-' 8962497 34386832 0 gen:do_call/4
<6939.1378.0> leveled_penciller:init/1 '-' 9180728 28655728 0 gen_server:loop/6
<6939.25814.108> leveled_penciller:init/1 '-' 0 28655728 0 gen_server:loop/6
<6939.1553.0> leveled_penciller:init/1 '-' 7988133 23880072 0 gen:do_call/4
<6939.29138.730> leveled_penciller:init/1 '-' 0 23879928 0 gen_server:loop/6
<6939.761.0> leveled_iclerk:init/1 '-' 0 19900096 0 gen_server:loop/6
<6939.766.0> leveled_iclerk:init/1 '-' 0 19900096 0 gen_server:loop/6
<6939.1069.0> leveled_penciller:init/1 '-' 6776950 19900096 0 gen_server:loop/6
<6939.1108.0> leveled_penciller:init/1 '-' 8265675 19900096 0 gen_server:loop/6
<6939.1307.0> leveled_penciller:init/1 '-' 10034834 19900096 0 gen_server:loop/6
===============================================================================================================================
'[email protected]' 17:58:23
Load: cpu 88 Memory: total 4915227 binary 3889784
procs 2937 processes 929746 code 15692
runq 11 atom 687 ets 43310
Pid Name or Initial Func Time Reds Memory MsgQ Current Function
-------------------------------------------------------------------------------------------------------------------------------
<6939.1188.0> leveled_penciller:init/1 '-' 10374868 34386688 0 gen_server:loop/6
<6939.1378.0> leveled_penciller:init/1 '-' 10271046 28655904 1 gen:do_call/4
<6939.1069.0> leveled_penciller:init/1 '-' 8044958 28655800 0 gen_server:loop/6
<6939.1307.0> leveled_penciller:init/1 '-' 10844972 28655728 0 gen_server:loop/6
<6939.25814.108> leveled_penciller:init/1 '-' 0 28655728 0 gen_server:loop/6
<6939.1108.0> leveled_penciller:init/1 '-' 9409542 23879928 0 gen_server:loop/6
<6939.1553.0> leveled_penciller:init/1 '-' 9600560 23879928 0 gen_server:loop/6
<6939.29138.730> leveled_penciller:init/1 '-' 0 23879928 0 gen_server:loop/6
<6939.1178.0> leveled_penciller:init/1 '-' 8026100 19900272 1 gen:do_call/4
<6939.761.0> leveled_iclerk:init/1 '-' 0 19900096 0 gen_server:loop/6
from leveled.
martinsumner
commented on August 20, 2024
Changing the iclerk made no difference. However it was noticeable that each node had many more penciller processes than was expected, and also a lot of logs indicating that snapshots had been timed out after 1 hour.
The iclerk/inker code was not releasing/closing the Penciller snapshot. This would lead to penciller processes hanging around, as well as historic SST files.
from leveled.
martinsumner
commented on August 20, 2024
Running a new test with the inker now correct releasing snapshots:
1-hour in
===============================================================================================================================
'[email protected]' 11:10:03
Load: cpu 88 Memory: total 2822051 binary 2311309
procs 1284 processes 419432 code 15918
runq 19 atom 719 ets 43678
Pid Name or Initial Func Time Reds Memory MsgQ Current Function
-------------------------------------------------------------------------------------------------------------------------------
<6940.1847.0> leveled_penciller:init/1 '-' 13615667 41263840 0 gen_server:loop/6
<6940.766.0> leveled_penciller:init/1 '-' 15290795 34386688 0 gen_server:loop/6
<6940.2568.0> leveled_penciller:init/1 '-' 11942783 34386688 0 gen_server:loop/6
<6940.2566.0> leveled_penciller:init/1 '-' 10906979 28655728 0 gen_server:loop/6
<6940.11618.609> leveled_sst:init/1 '-' 1598310 23879936 0 lists:last/1
<6940.1143.0> leveled_penciller:init/1 '-' 9654682 19900096 0 gen_server:loop/6
<6940.1879.0> leveled_penciller:init/1 '-' 7444992 13819792 0 gen_server:loop/6
<6940.1527.0> leveled_penciller:init/1 '-' 6092460 9597360 0 gen_server:loop/6
<6940.1513.0> leveled_penciller:init/1 '-' 7112787 7997952 0 gen_server:loop/6
<6940.531.0> exometer_proc:'-spawn_pro '-' 4175181 4119680 0 exometer_probe:loop/1
<6940.3570.0> exometer_proc:'-spawn_pro '-' 4132860 4119680 0 exometer_probe:loop/1
<6940.764.0> leveled_penciller:init/1 '-' 8769377 4119608 0 gen_server:loop/6
<6940.1887.0> leveled_penciller:init/1 '-' 7454490 4119608 0 gen_server:loop/6
<6940.166.0> application_master:start_ '-' 0 4119584 0 application_master:loop_it/4
<6940.508.0> exometer_proc:'-spawn_pro '-' 2061642 2546504 1 exometer_probe:loop/1
<6940.3699.0> exometer_proc:'-spawn_pro '-' 220110 2546504 1 exometer_probe:loop/1
<6940.506.0> exometer_proc:'-spawn_pro '-' 1161223 2546472 0 exometer_probe:loop/1
<6940.527.0> exometer_proc:'-spawn_pro '-' 1103307 2546472 0 exometer_probe:loop/1
<6940.530.0> exometer_proc:'-spawn_pro '-' 1115237 2546472 0 exometer_probe:loop/1
<6940.554.0> exometer_proc:'-spawn_pro '-' 750690 2546472 0 exometer_probe:loop/1
===============================================================================================================================
'[email protected]' 11:11:02
Load: cpu 87 Memory: total 2837965 binary 2351458
procs 1259 processes 386449 code 15918
runq 10 atom 719 ets 51312
Pid Name or Initial Func Time Reds Memory MsgQ Current Function
-------------------------------------------------------------------------------------------------------------------------------
<6940.2568.0> leveled_penciller:init/1 '-' 14133178 41263984 0 gen:do_call/4
<6940.1143.0> leveled_penciller:init/1 '-' 11819457 34386688 0 gen_server:loop/6
<6940.2566.0> leveled_penciller:init/1 '-' 13423332 34386688 0 gen_server:loop/6
<6940.1879.0> leveled_penciller:init/1 '-' 9993779 16583568 0 gen_server:loop/6
<6940.1513.0> leveled_penciller:init/1 '-' 9143695 13819792 0 gen_server:loop/6
<6940.1527.0> leveled_penciller:init/1 '-' 8138719 13819792 0 gen_server:loop/6
<6940.1887.0> leveled_penciller:init/1 '-' 7035062 11516648 0 gen_server:loop/6
<6940.1486.0> leveled_penciller:init/1 '-' 6159866 9597360 0 gen_server:loop/6
<6940.1140.0> leveled_penciller:init/1 '-' 6169036 7997952 0 gen_server:loop/6
<6940.764.0> leveled_penciller:init/1 '-' 6904841 6665112 0 gen_server:loop/6
<6940.3570.0> exometer_proc:'-spawn_pro '-' 4293185 4119712 1 exometer_probe:loop/1
<6940.508.0> exometer_proc:'-spawn_pro '-' 2196992 4119680 0 exometer_probe:loop/1
<6940.531.0> exometer_proc:'-spawn_pro '-' 4322660 4119680 0 exometer_probe:loop/1
<6940.766.0> leveled_penciller:init/1 '-' 5648482 4119608 0 gen_server:loop/6
<6940.166.0> application_master:start_ '-' 0 4119584 0 application_master:loop_it/4
<6940.506.0> exometer_proc:'-spawn_pro '-' 1381152 2546472 0 exometer_probe:loop/1
<6940.527.0> exometer_proc:'-spawn_pro '-' 1144629 2546472 0 exometer_probe:loop/1
<6940.530.0> exometer_proc:'-spawn_pro '-' 1125183 2546472 0 exometer_probe:loop/1
<6940.554.0> exometer_proc:'-spawn_pro '-' 740244 2546472 0 exometer_probe:loop/1
<6940.3691.0> exometer_proc:'-spawn_pro '-' 236237 2546472 0 exometer_probe:loop/1
At 2.5 hours into the test - the memory footprint continues to be stable (and significantly better than before the change)
===============================================================================================================================
'[email protected]' 12:39:05
Load: cpu 88 Memory: total 2975311 binary 2445896
procs 1613 processes 446097 code 15918
runq 6 atom 719 ets 38004
Pid Name or Initial Func Time Reds Memory MsgQ Current Function
-------------------------------------------------------------------------------------------------------------------------------
<6940.2566.0> leveled_penciller:init/1 '-' 11361768 34386792 1 gen_server:loop/6
<6940.1140.0> leveled_penciller:init/1 '-' 9928924 34386688 0 gen_server:loop/6
<6940.1887.0> leveled_penciller:init/1 '-' 12059806 34386688 0 gen_server:loop/6
<6940.1527.0> leveled_penciller:init/1 '-' 12168039 28655832 1 gen_server:loop/6
<6940.1143.0> leveled_penciller:init/1 '-' 8064581 19900096 0 gen_server:loop/6
<6940.1486.0> leveled_penciller:init/1 '-' 7714559 16583744 1 gen:do_call/4
<6940.1513.0> leveled_penciller:init/1 '-' 8757016 16583568 0 gen_server:loop/6
<6940.1847.0> leveled_penciller:init/1 '-' 6062620 7998056 1 gen_server:loop/6
<6940.766.0> leveled_penciller:init/1 '-' 5542404 7997952 0 gen_server:loop/6
<6940.1879.0> leveled_penciller:init/1 '-' 5034953 4119712 1 gen_server:loop/6
<6940.531.0> exometer_proc:'-spawn_pro '-' 3454103 4119680 0 exometer_probe:loop/1
<6940.573.0> exometer_proc:'-spawn_pro '-' 573676 4119680 0 exometer_probe:loop/1
<6940.3570.0> exometer_proc:'-spawn_pro '-' 3492477 4119680 0 exometer_probe:loop/1
<6940.166.0> application_master:start_ '-' 0 4119584 0 application_master:loop_it/4
<6940.506.0> exometer_proc:'-spawn_pro '-' 1674209 2546472 0 exometer_probe:loop/1
<6940.508.0> exometer_proc:'-spawn_pro '-' 1782209 2546472 0 exometer_probe:loop/1
<6940.527.0> exometer_proc:'-spawn_pro '-' 1493985 2546472 0 exometer_probe:loop/1
<6940.530.0> exometer_proc:'-spawn_pro '-' 1475428 2546472 0 exometer_probe:loop/1
<6940.554.0> exometer_proc:'-spawn_pro '-' 638783 2546472 0 exometer_probe:loop/1
<6940.3691.0> exometer_proc:'-spawn_pro '-' 189807 2546472 0 exometer_probe:loop/1
===============================================================================================================================
'[email protected]' 12:40:05
Load: cpu 88 Memory: total 2944265 binary 2403827
procs 1609 processes 447289 code 15918
runq 6 atom 719 ets 45653
Pid Name or Initial Func Time Reds Memory MsgQ Current Function
-------------------------------------------------------------------------------------------------------------------------------
<6940.1887.0> leveled_penciller:init/1 '-' 11606129 49516424 0 gen_server:loop/6
<6940.1140.0> leveled_penciller:init/1 '-' 11497457 34386688 0 gen_server:loop/6
<6940.1513.0> leveled_penciller:init/1 '-' 10112276 34386688 0 gen_server:loop/6
<6940.1486.0> leveled_penciller:init/1 '-' 9090052 23879928 0 gen_server:loop/6
<6940.1143.0> leveled_penciller:init/1 '-' 9277196 19900096 0 gen_server:loop/6
<6940.1847.0> leveled_penciller:init/1 '-' 7263336 11516792 0 gen:do_call/4
<6940.766.0> leveled_penciller:init/1 '-' 6829559 11516648 0 gen_server:loop/6
<6940.1879.0> leveled_penciller:init/1 '-' 6433885 11516648 0 gen_server:loop/6
<6940.2568.0> leveled_penciller:init/1 '-' 5339053 4119752 0 gen:do_call/4
<6940.508.0> exometer_proc:'-spawn_pro '-' 1641733 4119712 1 exometer_probe:loop/1
<6940.531.0> exometer_proc:'-spawn_pro '-' 3554800 4119680 0 exometer_probe:loop/1
<6940.573.0> exometer_proc:'-spawn_pro '-' 610379 4119680 0 exometer_probe:loop/1
<6940.3570.0> exometer_proc:'-spawn_pro '-' 3499092 4119680 0 exometer_probe:loop/1
<6940.764.0> leveled_penciller:init/1 '-' 5027407 4119608 0 gen_server:loop/6
<6940.166.0> application_master:start_ '-' 0 4119584 0 application_master:loop_it/4
<6940.506.0> exometer_proc:'-spawn_pro '-' 1624338 2546472 0 exometer_probe:loop/1
<6940.527.0> exometer_proc:'-spawn_pro '-' 1598591 2546472 0 exometer_probe:loop/1
<6940.530.0> exometer_proc:'-spawn_pro '-' 1506774 2546472 0 exometer_probe:loop/1
<6940.3716.0> exometer_proc:'-spawn_pro '-' 188218 2546472 0 exometer_probe:loop/1
<6940.3728.0> exometer_proc:'-spawn_pro '-' 188422 2546472 0 exometer_probe:loop/1
from leveled.
martinsumner
commented on August 20, 2024
Resolved by #70
from leveled.
Related Issues (20)
- OTP 24 Support HOT 1
- Reduce open files HOT 5
- Inker delete_confirmed call not cast HOT 1
- Selective sync HOT 2
- Microseconds LMD HOT 1
- Recalc and revolving objects HOT 1
- confirm_delete race HOT 2
- Redact server state on crash HOT 2
- Adjust defaults to reflect higher vnode counts HOT 2
- Handle hash of 0 correctly in leveled_cdb HOT 1
- Experiment with zstd HOT 1
- Memory utilisation for large stores HOT 2
- Snapshot time
- Outdated deps preventing build HOT 3
- Potential Issue re delete post-compaction HOT 1
- Use standard erlang logger HOT 1
- Rebuilding ledger - but first 50K gone HOT 1
- leveled_monitor - sample_period HOT 1
- Penciller work backlog HOT 2
- Never stalling is an issue HOT 1
Recommend Projects
-
React
A declarative, efficient, and flexible JavaScript library for building user interfaces.
-
Vue.js
🖖 Vue.js is a progressive, incrementally-adoptable JavaScript framework for building UI on the web.
-
Typescript
TypeScript is a superset of JavaScript that compiles to clean JavaScript output.
-
TensorFlow
An Open Source Machine Learning Framework for Everyone
-
Django
The Web framework for perfectionists with deadlines.
-
Laravel
A PHP framework for web artisans
-
D3
Bring data to life with SVG, Canvas and HTML. 📊📈🎉
-
Recommend Topics
-
javascript
JavaScript (JS) is a lightweight interpreted programming language with first-class functions.
-
web
Some thing interesting about web. New door for the world.
-
server
A server is a program made to process requests and deliver data to clients.
-
Machine learning
Machine learning is a way of modeling and interpreting data that allows a piece of software to respond intelligently.
-
Visualization
Some thing interesting about visualization, use data art
-
Game
Some thing interesting about game, make everyone happy.
Recommend Org
-
Facebook
We are working to build community through open source technology. NB: members must have two-factor auth.
-
Microsoft
Open source projects and samples from Microsoft.
-
Google
Google ❤️ Open Source for everyone.
-
Alibaba
Alibaba Open Source for everyone
-
D3
Data-Driven Documents codes.
-
Tencent
China tencent open source team.
from leveled.