Comments (8)
([email protected])4> ProcFun8 = fun(P) -> process_info(P, [binary, dictionary]) end.
#Fun<erl_eval.6.80484245>
([email protected])5> BM8 = lists:map(ProcFun8, PL1).
* 1: variable 'PL1' is unbound
([email protected])6> PL1 = processes() -- [self()].
[<0.0.0>,<0.0.2649>,<0.1.2753>,<0.2.2098>,<0.3.0>,<0.6.0>,
<0.7.0>,<0.7.91>,<0.9.0>,<0.10.0>,<0.11.0>,<0.11.2905>,
<0.11.3797>,<0.12.0>,<0.13.0>,<0.14.0>,<0.15.0>,<0.16.0>,
<0.17.2763>,<0.18.0>,<0.19.0>,<0.19.875>,<0.20.0>,
<0.20.4194>,<0.21.0>,<0.22.0>,<0.22.2533>,<0.23.0>,<0.24.0>|...]
([email protected])7> BM8 = lists:map(ProcFun8, PL1).
[[{binary,[{139823559737576,87,1},{139823559737448,84,1}]},
{dictionary,[]}],
[{binary,[{139808459837480,280,1},
{139801014952912,280,1},
{139801014952592,280,1},
{139801014952272,280,1},
{139809177094184,280,1},
{139809177093864,280,1},
{139809177093544,280,1},
{139809177093224,280,1},
{139802293643112,280,1},
{139802293642792,280,1},
{139802293642472,280,1},
{139802293642152,280,1},
{139800721671432,280,1},
{139800721671112,280,1},
{139800721670792,280,1},
{139811778210928,280,1},
{139811778210608,280,1},
{139811778210288,280,1},
{139800048539920,280,1},
{139800048539600,280,1},
{139800048539280,280,1},
{139800048820792,280,...},
{139800048820472,...},
{...}|...]},
{dictionary,[{random_seed,{29488,2847,19820}},
{'$ancestors',[<0.785.0>,<0.772.0>,<0.706.0>,<0.663.0>,
riak_core_vnode_sup,riak_core_sup,<0.166.0>]},
{'$initial_call',{leveled_sst,init,1}}]}],
[{binary,[{139801014428000,280,1},
{139810511749128,280,1},
{139805871272824,280,1},
{139798396295600,280,1},
{139800101623352,280,1},
{139798670896040,280,1},
{139800510711112,280,1},
{139800443017648,280,1},
{139800443017328,280,1},
{139800443017008,280,1},
{139800443016688,280,1},
{139800443016368,280,1},
{139799186325536,280,1},
{139799186325216,280,1},
{139799186324896,280,1},
{139799186324576,280,1},
{139799186324256,280,1},
{139799415345424,280,1},
{139799415345104,280,1},
{139799415344784,280,1},
{139799415344464,280,...},
{139799415344144,...},
{...}|...]},
{dictionary,[{random_seed,{28175,13302,16361}},
{'$ancestors',[<0.1504.0>,<0.1503.0>,<0.1438.0>,<0.1300.0>,
riak_core_vnode_sup,riak_core_sup,<0.166.0>]},
{'$initial_call',{leveled_sst,init,1}}]}],
[{binary,[{139802061964704,280,1},
{139801495629696,280,1},
{139801495629376,280,1},
{139816202822328,280,1},
{139816202822008,280,1},
{139803106021976,280,1},
{139803106021656,280,1},
{139802124006416,280,1},
{139802124006096,280,1},
{139806063706696,280,1},
{139802126249544,280,1},
{139807608154680,280,1},
{139817563750616,280,1},
{139817563750296,280,1},
{139807608154360,280,1},
{139802607793776,280,1},
{139802617285888,280,1},
{139801978470904,280,1},
{139801877093432,280,1},
{139805643337304,280,...},
{139803372959496,...},
{...}|...]},
{dictionary,[{random_seed,{19551,15211,16472}},
{'$ancestors',[<0.1158.0>,<0.1157.0>,<0.1076.0>,<0.1026.0>,
riak_core_vnode_sup,riak_core_sup,<0.166.0>]},
{'$initial_call',{leveled_sst,init,1}}]}],
[{binary,[{139815315578120,65,1},
{139789590270048,665,1},
{139798945437152,70,1},
{139794848596104,724,1}]},
{dictionary,[]}],
[{binary,[]},
{dictionary,[{'$ancestors',[<0.2.0>]},
{'$initial_call',{gen_event,init_it,6}}]}],
[{binary,[]},
{dictionary,[{'$ancestors',[<0.2.0>]},
{'$initial_call',{application_controller,start,1}}]}],
[{binary,[]},
{dictionary,[{'$ancestors',[exometer_admin,
exometer_core_sup,<0.157.0>]},
{{exometer_proc,info},
{info,<0.159.0>,[],
{sys,undefined,exometer_probe,
[riak,riak_kv,node,gets,read_repairs|...]}}},
{'$initial_call',{exometer_proc,'-spawn_process/2-fun-0-',
0}}]}],
[{binary,[]},
{dictionary,[{'$ancestors',[<0.8.0>]},
{'$initial_call',{application_master,init,4}}]}],
[{binary,[]},{dictionary,[]}],
[{binary,[]},
{dictionary,[{'$ancestors',[<0.10.0>]},
{'$initial_call',{supervisor,kernel,1}}]}],
[{binary,[{139810815085640,280,1},
{139808500896456,280,1},
{139799925096504,280,1},
{139797033947440,280,1},
{139798281388472,280,1},
{139797033947120,280,1},
{139799925096184,280,1},
{139797031271600,280,1},
{139797031271280,280,1},
{139797031270960,280,1},
{139797031270640,280,1},
{139796812932632,280,...},
{139796812932312,...},
{...}|...]},
{dictionary,[{random_seed,{916,9853,10475}},
{'$ancestors',[<0.791.0>,<0.771.0>,<0.699.0>,<0.654.0>,
riak_core_vnode_sup,riak_core_sup,<0.166.0>]},
{'$initial_call',{leveled_sst,init,1}}]}],
[{binary,[{139793185077232,280,1},
{139795392964400,280,1},
{139797106399928,280,1},
{139795390562856,280,1},
{139794488113432,280,1},
{139793610657328,280,1},
{139794956078360,280,1},
{139811497520616,280,1},
{139791499450552,280,1},
{139794718649520,280,1},
{139793571578760,280,...},
{139803054430880,...},
{...}|...]},
{dictionary,[{random_seed,{16873,9914,28119}},
{'$ancestors',[<0.1158.0>,<0.1157.0>,<0.1076.0>,<0.1026.0>,
riak_core_vnode_sup,riak_core_sup,<0.166.0>]},
{'$initial_call',{leveled_sst,init,1}}]}],
[{binary,[]},
{dictionary,[{'$ancestors',[kernel_sup,<0.10.0>]},
{'$initial_call',{rpc,init,1}}]}],
[{binary,[]},
{dictionary,[{'$ancestors',[kernel_sup,<0.10.0>]},
{'$initial_call',{global,init,1}}]}],
[{binary,[]},{dictionary,[]}],
[{binary,[]},{dictionary,[]}],
[{binary,[]},
{dictionary,[{'$ancestors',[kernel_sup,<0.10.0>]},
{'$initial_call',{inet_db,init,1}}]}],
[{binary,[{139799560590040,280,1},
{139802486803248,280,1},
{139810861316360,280,1},
{139800872136824,280,1},
{139806291244792,280,...},
{139803043005992,...},
{...}|...]},
{dictionary,[{random_seed,{699,21190,19162}},
{'$ancestors',[<0.1139.0>,<0.1137.0>|...]},
{'$initial_call',{leveled_sst,...}}]}],
[{binary,[]},
{dictionary,[{'$ancestors',[kernel_sup,<0.10.0>]},
{'$initial_call',{supervisor,...}}]}],
[{binary,[]},
{dictionary,[{'$ancestors',[net_sup|...]},
{'$initial_call',{...}}]}],
[{binary,[{139816926648648,280,1},
{139813170963232,280,...},
{139814096561912,...},
{...}|...]},
{dictionary,[{random_seed,{...}},{'$ancestors',...},{...}]}],
[{binary,[]},{dictionary,[{'$ancestors',...},{...}]}],
[{binary,[{139789413458440,...},{...}|...]},
{dictionary,[{...}|...]}],
[{binary,[]},{dictionary,[...]}],
[{binary,[]},{dictionary,...}],
[{binary,...},{...}],
[{...}|...],
[...]|...]
([email protected])8> BinTotFun8 = fun([{binary, BL}, {dictionary, DL}]) -> {DL, lists:foldl(fun({_, Mem, _}, Acc) -> Acc + Mem end, 0, BL)} end.
#Fun<erl_eval.6.80484245>
([email protected])9> Out8 = lists:map(BinTotFun8, BM8).
[{[],171},
{[{random_seed,{29488,2847,19820}},
{'$ancestors',[<0.785.0>,<0.772.0>,<0.706.0>,<0.663.0>,
riak_core_vnode_sup,riak_core_sup,<0.166.0>]},
{'$initial_call',{leveled_sst,init,1}}],
104448},
{[{random_seed,{28175,13302,16361}},
{'$ancestors',[<0.1504.0>,<0.1503.0>,<0.1438.0>,<0.1300.0>,
riak_core_vnode_sup,riak_core_sup,<0.166.0>]},
{'$initial_call',{leveled_sst,init,1}}],
104448},
{[{random_seed,{19551,15211,16472}},
{'$ancestors',[<0.1158.0>,<0.1157.0>,<0.1076.0>,<0.1026.0>,
riak_core_vnode_sup,riak_core_sup,<0.166.0>]},
{'$initial_call',{leveled_sst,init,1}}],
104448},
{[],1524},
{[{'$ancestors',[<0.2.0>]},
{'$initial_call',{gen_event,init_it,6}}],
0},
{[{'$ancestors',[<0.2.0>]},
{'$initial_call',{application_controller,start,1}}],
0},
{[{'$ancestors',[exometer_admin,exometer_core_sup,
<0.157.0>]},
{{exometer_proc,info},
{info,<0.159.0>,[],
{sys,undefined,exometer_probe,
[riak,riak_kv,node,gets,read_repairs,
'[email protected]',primary,
outofdate]}}},
{'$initial_call',{exometer_proc,'-spawn_process/2-fun-0-',
0}}],
0},
{[{'$ancestors',[<0.8.0>]},
{'$initial_call',{application_master,init,4}}],
0},
{[],0},
{[{'$ancestors',[<0.10.0>]},
{'$initial_call',{supervisor,kernel,1}}],
0},
{[{random_seed,{916,9853,10475}},
{'$ancestors',[<0.791.0>,<0.771.0>,<0.699.0>,<0.654.0>,
riak_core_vnode_sup,riak_core_sup,<0.166.0>]},
{'$initial_call',{leveled_sst,init,1}}],
104448},
{[{random_seed,{16873,9914,28119}},
{'$ancestors',[<0.1158.0>,<0.1157.0>,<0.1076.0>,<0.1026.0>,
riak_core_vnode_sup,riak_core_sup,<0.166.0>]},
{'$initial_call',{leveled_sst,init,1}}],
117023},
{[{'$ancestors',[kernel_sup,<0.10.0>]},
{'$initial_call',{rpc,init,1}}],
0},
{[{'$ancestors',[kernel_sup,<0.10.0>]},
{'$initial_call',{global,init,1}}],
0},
{[],0},
{[],0},
{[{'$ancestors',[kernel_sup,<0.10.0>]},
{'$initial_call',{inet_db,init,1}}],
0},
{[{random_seed,{699,21190,19162}},
{'$ancestors',[<0.1139.0>,<0.1137.0>,<0.1069.0>,<0.1035.0>,
riak_core_vnode_sup|...]},
{'$initial_call',{leveled_sst,init,1}}],
104448},
{[{'$ancestors',[kernel_sup,<0.10.0>]},
{'$initial_call',{supervisor,erl_distribution,1}}],
0},
{[{'$ancestors',[net_sup,kernel_sup,<0.10.0>]},
{'$initial_call',{erl_epmd,init,1}}],
0},
{[{random_seed,{29888,29165,7235}},
{'$ancestors',[<0.1520.0>,<0.1519.0>|...]},
{'$initial_call',{leveled_sst,...}}],
104448},
{[{'$ancestors',[net_sup,kernel_sup|...]},
{'$initial_call',{auth,...}}],
0},
{[{'$ancestors',[<0.1520.0>|...]},{'$initial_call',{...}}],
71769},
{[{'$ancestors',[...]},{longnames,...},{...}],0},
{[],0},
{[{...}|...],104448},
{[],...},
{...}|...]
([email protected])10>
Another 30 minutes:
===============================================================================================================================
'[email protected]' 14:16:45
Load: cpu 88 Memory: total 5667911 binary 4856336
procs 2002 processes 713611 code 15918
runq 2 atom 719 ets 50194
Pid Name or Initial Func Time Reds Memory MsgQ Current Function
-------------------------------------------------------------------------------------------------------------------------------
<6939.764.0> leveled_penciller:init/1 '-' 11762076 41263840 0 gen_server:loop/6
<6939.773.0> leveled_penciller:init/1 '-' 12210912 28655728 0 gen_server:loop/6
<6939.1843.0> leveled_penciller:init/1 '-' 11383114 23880072 0 gen:do_call/4
<6939.776.0> leveled_penciller:init/1 '-' 9663572 23879928 0 gen_server:loop/6
<6939.1159.0> leveled_penciller:init/1 '-' 10986471 23879928 0 gen_server:loop/6
<6939.1493.0> leveled_penciller:init/1 '-' 10599386 23879928 0 gen_server:loop/6
<6939.1513.0> leveled_penciller:init/1 '-' 13158039 23879928 0 gen_server:loop/6
<6939.1141.0> leveled_penciller:init/1 '-' 8415712 19900096 0 gen_server:loop/6
<6939.1186.704> leveled_penciller:init/1 '-' 0 19900096 0 gen_server:loop/6
<6939.1521.0> leveled_penciller:init/1 '-' 11993679 19900096 0 gen_server:loop/6
===============================================================================================================================
'[email protected]' 14:17:45
Load: cpu 88 Memory: total 5378872 binary 4583662
procs 2023 processes 694059 code 15918
runq 5 atom 719 ets 50215
Pid Name or Initial Func Time Reds Memory MsgQ Current Function
-------------------------------------------------------------------------------------------------------------------------------
<6939.1513.0> leveled_penciller:init/1 '-' 15206835 49516424 0 gen_server:loop/6
<6939.1843.0> leveled_penciller:init/1 '-' 13124327 34386832 0 gen:do_call/4
<6939.776.0> leveled_penciller:init/1 '-' 10752976 28655728 0 gen_server:loop/6
<6939.1141.0> leveled_penciller:init/1 '-' 9617627 28655728 0 gen_server:loop/6
<6939.1521.0> leveled_penciller:init/1 '-' 14395115 28655728 0 gen_server:loop/6
<6939.1186.704> leveled_penciller:init/1 '-' 0 19900096 0 gen_server:loop/6
<6939.1153.0> leveled_penciller:init/1 '-' 7532119 16583568 0 gen_server:loop/6
<6939.21632.622> leveled_penciller:init/1 '-' 0 16583568 0 gen_server:loop/6
<6939.1884.0> leveled_penciller:init/1 '-' 8314558 13819792 0 gen_server:loop/6
<6939.9434.838> leveled_penciller:init/1 '-' 0 13819792 0 gen_server:loop/6
===============================================================================================================================
'[email protected]' 14:18:44
Load: cpu 88 Memory: total 4599838 binary 3875696
procs 2052 processes 614380 code 15918
runq 13 atom 719 ets 58453
Pid Name or Initial Func Time Reds Memory MsgQ Current Function
-------------------------------------------------------------------------------------------------------------------------------
<6939.1141.0> leveled_penciller:init/1 '-' 11522281 34386688 0 gen_server:loop/6
<6939.1884.0> leveled_penciller:init/1 '-' 9907293 23879928 0 gen_server:loop/6
<6939.1153.0> leveled_penciller:init/1 '-' 9320885 19900096 0 gen_server:loop/6
<6939.1186.704> leveled_penciller:init/1 '-' 0 19900096 0 gen_server:loop/6
<6939.1889.0> leveled_penciller:init/1 '-' 7461791 16583568 0 gen_server:loop/6
<6939.21632.622> leveled_penciller:init/1 '-' 0 16583568 0 gen_server:loop/6
<6939.764.0> leveled_penciller:init/1 '-' 7480932 13819792 0 gen_server:loop/6
<6939.9434.838> leveled_penciller:init/1 '-' 0 13819792 0 gen_server:loop/6
<6939.1493.0> leveled_penciller:init/1 '-' 6463530 11516648 0 gen_server:loop/6
<6939.15575.195> leveled_penciller:init/1 '-' 0 11516648 0 gen_server:loop/6
from leveled.
Re-running after change the Inker clerk so that it starts a temporary assistant to manage each compaction.
After 30 minutes running
===============================================================================================================================
'[email protected]' 13:18:54
Load: cpu 82 Memory: total 3526570 binary 3004856
procs 1230 processes 408972 code 15918
runq 0 atom 719 ets 64952
Pid Name or Initial Func Time Reds Memory MsgQ Current Function
-------------------------------------------------------------------------------------------------------------------------------
<6941.1493.0> leveled_penciller:init/1 '-' 1230077 23879928 0 gen_server:loop/6
<6941.1513.0> leveled_penciller:init/1 '-' 1037029 34386688 0 gen_server:loop/6
<6941.764.0> leveled_penciller:init/1 '-' 1011927 28655728 0 gen_server:loop/6
<6941.1521.0> leveled_penciller:init/1 '-' 887950 16583672 1 gen_server:loop/6
<6941.1884.0> leveled_penciller:init/1 '-' 859656 23879928 0 gen_server:loop/6
<6941.8714.308> leveled_cdb:init/1 '-' 820659 13728 0 gen_fsm:loop/7
<6941.1045.0> riak_core_vnode:init/1 '-' 793482 34880 5 gen:do_call/4
<6941.1018.0> riak_core_vnode:init/1 '-' 790688 21856 0 gen_fsm:loop/7
<6941.658.0> riak_core_vnode:init/1 '-' 789815 21992 2 gen:do_call/4
<6941.1211.0> riak_core_vnode:init/1 '-' 787896 21992 2 gen:do_call/4
===============================================================================================================================
'[email protected]' 13:18:58
Load: cpu 84 Memory: total 3621255 binary 3087907
procs 1231 processes 422494 code 15918
runq 1 atom 719 ets 65457
Pid Name or Initial Func Time Reds Memory MsgQ Current Function
-------------------------------------------------------------------------------------------------------------------------------
<6941.1493.0> leveled_penciller:init/1 '-' 1389726 23879928 0 gen_server:loop/6
<6941.1513.0> leveled_penciller:init/1 '-' 1180412 34386688 0 gen_server:loop/6
<6941.764.0> leveled_penciller:init/1 '-' 1071182 28655800 0 gen_server:loop/6
<6941.1884.0> leveled_penciller:init/1 '-' 887051 23879928 0 gen_server:loop/6
<6941.1521.0> leveled_penciller:init/1 '-' 843496 16583568 0 gen_server:loop/6
<6941.8714.308> leveled_cdb:init/1 '-' 826813 13728 0 gen_fsm:loop/7
<6941.658.0> riak_core_vnode:init/1 '-' 825352 21856 0 gen_fsm:loop/7
<6941.17895.309> leveled_cdb:init/1 '-' 820165 13728 0 gen_fsm:loop/7
<6941.1309.0> riak_core_vnode:init/1 '-' 804719 34880 5 gen:do_call/4
<6941.22078.329> leveled_cdb:init/1 '-' 803517 13728 0 gen_fsm:loop/7
This is already at 7.4% memory for beam, early on in the test.
Then after another 30 minutes (now 1 hour into the test):
===============================================================================================================================
'[email protected]' 13:45:09
Load: cpu 87 Memory: total 4362415 binary 3687495
procs 1583 processes 569867 code 15918
runq 10 atom 719 ets 52875
Pid Name or Initial Func Time Reds Memory MsgQ Current Function
-------------------------------------------------------------------------------------------------------------------------------
<6939.773.0> leveled_penciller:init/1 '-' 12114869 41263984 0 gen:do_call/4
<6939.1141.0> leveled_penciller:init/1 '-' 12429483 34386688 0 gen_server:loop/6
<6939.1521.0> leveled_penciller:init/1 '-' 12828866 34386688 0 gen_server:loop/6
<6939.1153.0> leveled_penciller:init/1 '-' 13258810 28655872 0 gen:do_call/4
<6939.1493.0> leveled_penciller:init/1 '-' 11314641 28655728 0 gen_server:loop/6
<6939.1843.0> leveled_penciller:init/1 '-' 11273293 28655728 0 gen_server:loop/6
<6939.1884.0> leveled_penciller:init/1 '-' 12205498 28655728 0 gen_server:loop/6
<6939.1513.0> leveled_penciller:init/1 '-' 8423506 16583568 0 gen_server:loop/6
<6939.1889.0> leveled_penciller:init/1 '-' 8288297 13819792 0 gen_server:loop/6
<6939.15575.195> leveled_penciller:init/1 '-' 0 11516648 0 gen_server:loop/6
===============================================================================================================================
'[email protected]' 13:46:09
Load: cpu 87 Memory: total 4950114 binary 4238023
procs 1587 processes 606753 code 15918
runq 11 atom 719 ets 58704
Pid Name or Initial Func Time Reds Memory MsgQ Current Function
-------------------------------------------------------------------------------------------------------------------------------
<6939.1521.0> leveled_penciller:init/1 '-' 15218531 49516424 0 gen_server:loop/6
<6939.1141.0> leveled_penciller:init/1 '-' 14326448 34386832 0 gen:do_call/4
<6939.1493.0> leveled_penciller:init/1 '-' 13479516 34386688 0 gen_server:loop/6
<6939.1843.0> leveled_penciller:init/1 '-' 13463341 34386688 0 gen_server:loop/6
<6939.1884.0> leveled_penciller:init/1 '-' 14421737 34386688 0 gen_server:loop/6
<6939.773.0> leveled_penciller:init/1 '-' 13953614 28655728 0 gen_server:loop/6
<6939.1889.0> leveled_penciller:init/1 '-' 9993834 28655728 0 gen_server:loop/6
<6939.1513.0> leveled_penciller:init/1 '-' 10452796 23879928 0 gen_server:loop/6
<6939.764.0> leveled_penciller:init/1 '-' 7890123 11516648 0 gen_server:loop/6
<6939.15575.195> leveled_penciller:init/1 '-' 0 11516648 0 gen_server:loop/6
===============================================================================================================================
'[email protected]' 13:47:08
Load: cpu 89 Memory: total 3129762 binary 2592897
procs 1613 processes 438487 code 15918
runq 6 atom 719 ets 48982
Pid Name or Initial Func Time Reds Memory MsgQ Current Function
-------------------------------------------------------------------------------------------------------------------------------
<6939.1513.0> leveled_penciller:init/1 '-' 12255606 28655800 0 gen_server:loop/6
<6939.764.0> leveled_penciller:init/1 '-' 9795208 23880072 0 gen:do_call/4
<6939.1889.0> leveled_penciller:init/1 '-' 12207526 23880032 1 gen_server:loop/6
<6939.1159.0> leveled_penciller:init/1 '-' 8551101 19900096 0 gen_server:loop/6
<6939.776.0> leveled_penciller:init/1 '-' 8686412 11516648 0 gen_server:loop/6
<6939.15575.195> leveled_penciller:init/1 '-' 0 11516648 0 gen_server:loop/6
<6939.1153.0> leveled_penciller:init/1 '-' 6769969 9597360 0 gen_server:loop/6
<6939.11262.295> leveled_penciller:init/1 '-' 0 9597360 0 gen_server:loop/6
<6939.21840.495> leveled_penciller:init/1 '-' 0 9597360 0 gen_server:loop/6
<6939.1147.138> leveled_penciller:init/1 '-' 0 6665112 0 gen_server:loop/6
Now beam is using 9% of RAM - but the internal recording of use is up/down in etop.
Now at 2.5 hours into the test. Note that compactions are still not being initiated (as they're just below the threshold score).
===============================================================================================================================
'[email protected]' 15:14:48
Load: cpu 88 Memory: total 6718914 binary 5647827
procs 3130 processes 969023 code 15918
runq 6 atom 719 ets 48411
Pid Name or Initial Func Time Reds Memory MsgQ Current Function
-------------------------------------------------------------------------------------------------------------------------------
<6939.764.0> leveled_penciller:init/1 '-' 10058090 28655872 0 gen:do_call/4
<6939.1159.0> leveled_penciller:init/1 '-' 13661918 28655728 0 gen_server:loop/6
<6939.776.0> leveled_penciller:init/1 '-' 12787045 23880104 1 io:wait_io_mon_reply/2
<6939.1493.0> leveled_penciller:init/1 '-' 10190506 23879928 0 gen_server:loop/6
<6939.1889.0> leveled_penciller:init/1 '-' 9770154 23879928 0 gen_server:loop/6
<6939.7751.1261> leveled_sst:init/1 '-' 0 19900136 0 gen_fsm:loop/7
<6939.1141.0> leveled_penciller:init/1 '-' 11688693 19900096 0 gen_server:loop/6
<6939.1186.704> leveled_penciller:init/1 '-' 0 19900096 0 gen_server:loop/6
<6939.1513.0> leveled_penciller:init/1 '-' 10854254 16583712 0 gen:do_call/4
<6939.1884.0> leveled_penciller:init/1 '-' 8356204 16583672 1 gen_server:loop/6
===============================================================================================================================
'[email protected]' 15:15:48
Load: cpu 88 Memory: total 6939698 binary 5811216
procs 3144 processes 1033438 code 15918
runq 3 atom 719 ets 52212
Pid Name or Initial Func Time Reds Memory MsgQ Current Function
-------------------------------------------------------------------------------------------------------------------------------
<6939.1159.0> leveled_penciller:init/1 '-' 15304157 49516528 1 gen_server:loop/6
<6939.1141.0> leveled_penciller:init/1 '-' 12783810 41263840 0 gen_server:loop/6
<6939.1493.0> leveled_penciller:init/1 '-' 11354398 34386832 0 gen:do_call/4
<6939.764.0> leveled_penciller:init/1 '-' 11349346 34386688 0 gen_server:loop/6
<6939.1513.0> leveled_penciller:init/1 '-' 11554348 23880104 1 gen:do_call/4
<6939.1889.0> leveled_penciller:init/1 '-' 11792318 23879928 0 gen_server:loop/6
<6939.773.0> leveled_penciller:init/1 '-' 8729132 19900240 0 gen:do_call/4
<6939.7751.1261> leveled_sst:init/1 '-' 16 19900136 0 gen_fsm:loop/7
<6939.1186.704> leveled_penciller:init/1 '-' 0 19900096 0 gen_server:loop/6
<6939.1153.0> leveled_penciller:init/1 '-' 6750081 16583712 0 gen:do_call/4
===============================================================================================================================
'[email protected]' 15:16:47
Load: cpu 89 Memory: total 6824149 binary 5706816
procs 3161 processes 1022111 code 15918
runq 10 atom 719 ets 47545
Pid Name or Initial Func Time Reds Memory MsgQ Current Function
-------------------------------------------------------------------------------------------------------------------------------
<6939.1141.0> leveled_penciller:init/1 '-' 13340484 49516424 0 gen_server:loop/6
<6939.1889.0> leveled_penciller:init/1 '-' 12624280 41263840 0 gen_server:loop/6
<6939.764.0> leveled_penciller:init/1 '-' 12435187 28655872 0 gen:do_call/4
<6939.773.0> leveled_penciller:init/1 '-' 9949071 28655872 0 gen:do_call/4
<6939.1513.0> leveled_penciller:init/1 '-' 13123416 28655728 0 gen_server:loop/6
<6939.1153.0> leveled_penciller:init/1 '-' 8014141 23880072 0 gen:do_call/4
<6939.7751.1261> leveled_sst:init/1 '-' 16 19900136 0 gen_fsm:loop/7
<6939.1186.704> leveled_penciller:init/1 '-' 0 19900096 0 gen_server:loop/6
<6939.1843.0> leveled_penciller:init/1 '-' 7758890 16583568 0 gen_server:loop/6
<6939.21632.622> leveled_penciller:init/1 '-' 0 16583568 0 gen_server:loop/6
Stop load, then restart:
===============================================================================================================================
'[email protected]' 15:36:06
Load: cpu 0 Memory: total 1246263 binary 897933
procs 1541 processes 262963 code 14286
runq 0 atom 639 ets 39955
Pid Name or Initial Func Time Reds Memory MsgQ Current Function
-------------------------------------------------------------------------------------------------------------------------------
<6939.166.0> application_master:start_ '-' 1544842 4119584 0 application_master:loop_it/4
<6939.1070.0> leveled_pclerk:init/1 '-' 8218409 2546400 0 gen_server:loop/6
<6939.1108.0> leveled_penciller:init/1 '-' 134761 2546400 0 gen_server:loop/6
<6939.1178.0> leveled_penciller:init/1 '-' 136246 2546400 0 gen_server:loop/6
<6939.1189.0> leveled_pclerk:init/1 '-' 13298943 2546400 0 gen_server:loop/6
<6939.1259.0> leveled_penciller:init/1 '-' 144138 2546400 0 gen_server:loop/6
<6939.1260.0> leveled_pclerk:init/1 '-' 8352840 2546400 0 gen_server:loop/6
<6939.1378.0> leveled_penciller:init/1 '-' 148666 2546400 0 gen_server:loop/6
<6939.1160.0> leveled_sst:init/1 '-' 146218 1574144 0 gen_fsm:loop/7
<6939.1250.0> leveled_sst:init/1 '-' 138905 1574144 0 gen_fsm:loop/7
from leveled.
after the restart, start the load again:
===============================================================================================================================
'[email protected]' 15:36:06
Load: cpu 0 Memory: total 1246263 binary 897933
procs 1541 processes 262963 code 14286
runq 0 atom 639 ets 39955
Pid Name or Initial Func Time Reds Memory MsgQ Current Function
-------------------------------------------------------------------------------------------------------------------------------
<6939.166.0> application_master:start_ '-' 1544842 4119584 0 application_master:loop_it/4
<6939.1070.0> leveled_pclerk:init/1 '-' 8218409 2546400 0 gen_server:loop/6
<6939.1108.0> leveled_penciller:init/1 '-' 134761 2546400 0 gen_server:loop/6
<6939.1178.0> leveled_penciller:init/1 '-' 136246 2546400 0 gen_server:loop/6
<6939.1189.0> leveled_pclerk:init/1 '-' 13298943 2546400 0 gen_server:loop/6
<6939.1259.0> leveled_penciller:init/1 '-' 144138 2546400 0 gen_server:loop/6
<6939.1260.0> leveled_pclerk:init/1 '-' 8352840 2546400 0 gen_server:loop/6
<6939.1378.0> leveled_penciller:init/1 '-' 148666 2546400 0 gen_server:loop/6
<6939.1160.0> leveled_sst:init/1 '-' 146218 1574144 0 gen_fsm:loop/7
<6939.1250.0> leveled_sst:init/1 '-' 138905 1574144 0 gen_fsm:loop/7
===============================================================================================================================
'[email protected]' 15:37:05
Load: cpu 0 Memory: total 1322101 binary 968053
procs 2189 processes 267936 code 14655
runq 0 atom 679 ets 40555
Pid Name or Initial Func Time Reds Memory MsgQ Current Function
-------------------------------------------------------------------------------------------------------------------------------
<6939.1178.0> leveled_penciller:init/1 '-' 13600 4119608 0 gen_server:loop/6
<6939.1595.0> leveled_penciller:init/1 '-' 126001 4119608 0 gen_server:loop/6
<6939.1632.0> leveled_penciller:init/1 '-' 119850 4119608 0 gen_server:loop/6
<6939.166.0> application_master:start_ '-' 0 4119584 0 application_master:loop_it/4
<6939.1070.0> leveled_pclerk:init/1 '-' 390 2546400 0 gen_server:loop/6
<6939.1108.0> leveled_penciller:init/1 '-' 1050 2546400 0 gen_server:loop/6
<6939.1189.0> leveled_pclerk:init/1 '-' 377 2546400 0 gen_server:loop/6
<6939.1259.0> leveled_penciller:init/1 '-' 47210 2546400 0 gen_server:loop/6
<6939.1260.0> leveled_pclerk:init/1 '-' 390 2546400 0 gen_server:loop/6
<6939.1378.0> leveled_penciller:init/1 '-' 59980 2546400 0 gen_server:loop/6
===============================================================================================================================
'[email protected]' 15:38:05
Load: cpu 0 Memory: total 1326499 binary 968557
procs 2449 processes 271835 code 14680
runq 0 atom 679 ets 40468
Pid Name or Initial Func Time Reds Memory MsgQ Current Function
-------------------------------------------------------------------------------------------------------------------------------
<6939.1178.0> leveled_penciller:init/1 '-' 1050 4119608 0 gen_server:loop/6
<6939.1595.0> leveled_penciller:init/1 '-' 1015 4119608 0 gen_server:loop/6
<6939.1632.0> leveled_penciller:init/1 '-' 1050 4119608 0 gen_server:loop/6
<6939.166.0> application_master:start_ '-' 0 4119584 0 application_master:loop_it/4
<6939.1070.0> leveled_pclerk:init/1 '-' 390 2546400 0 gen_server:loop/6
<6939.1108.0> leveled_penciller:init/1 '-' 1015 2546400 0 gen_server:loop/6
<6939.1189.0> leveled_pclerk:init/1 '-' 390 2546400 0 gen_server:loop/6
<6939.1259.0> leveled_penciller:init/1 '-' 1050 2546400 0 gen_server:loop/6
<6939.1260.0> leveled_pclerk:init/1 '-' 390 2546400 0 gen_server:loop/6
<6939.1378.0> leveled_penciller:init/1 '-' 1050 2546400 0 gen_server:loop/6
===============================================================================================================================
'[email protected]' 15:39:05
Load: cpu 67 Memory: total 1272480 binary 916482
procs 1679 processes 265585 code 15692
runq 3 atom 687 ets 41867
Pid Name or Initial Func Time Reds Memory MsgQ Current Function
-------------------------------------------------------------------------------------------------------------------------------
<6939.1188.0> leveled_penciller:init/1 '-' 3579906 6665256 0 gen:do_call/4
<6939.1108.0> leveled_penciller:init/1 '-' 3864938 4119608 0 gen_server:loop/6
<6939.1178.0> leveled_penciller:init/1 '-' 3752768 4119608 0 gen_server:loop/6
<6939.1259.0> leveled_penciller:init/1 '-' 4170802 4119608 0 gen_server:loop/6
<6939.1307.0> leveled_penciller:init/1 '-' 3483376 4119608 0 gen_server:loop/6
<6939.1378.0> leveled_penciller:init/1 '-' 4006027 4119608 0 gen_server:loop/6
<6939.1553.0> leveled_penciller:init/1 '-' 3934522 4119608 0 gen_server:loop/6
<6939.1595.0> leveled_penciller:init/1 '-' 4093998 4119608 0 gen_server:loop/6
<6939.1632.0> leveled_penciller:init/1 '-' 4132536 4119608 0 gen_server:loop/6
<6939.166.0> application_master:start_ '-' 0 4119584 0 application_master:loop_it/4
===============================================================================================================================
'[email protected]' 15:40:04
Load: cpu 88 Memory: total 2109470 binary 1611083
procs 1619 processes 410850 code 15692
runq 9 atom 687 ets 42474
Pid Name or Initial Func Time Reds Memory MsgQ Current Function
-------------------------------------------------------------------------------------------------------------------------------
<6939.1595.0> leveled_penciller:init/1 '-' 6669082 16583640 0 gen_server:loop/6
<6939.1108.0> leveled_penciller:init/1 '-' 5846397 13819936 0 gen:do_call/4
<6939.1632.0> leveled_penciller:init/1 '-' 6487819 13819936 0 gen:do_call/4
<6939.1553.0> leveled_penciller:init/1 '-' 6312739 11516824 1 gen:do_call/4
<6939.1259.0> leveled_penciller:init/1 '-' 6630790 11516792 0 gen:do_call/4
<6939.1307.0> leveled_penciller:init/1 '-' 5698638 11516648 0 gen_server:loop/6
<6939.1378.0> leveled_penciller:init/1 '-' 6307640 11516648 0 gen_server:loop/6
<6939.1465.0> leveled_penciller:init/1 '-' 6230761 11516648 0 gen_server:loop/6
<6939.1178.0> leveled_penciller:init/1 '-' 6354310 9597536 1 gen:do_call/4
<6939.1069.0> leveled_penciller:init/1 '-' 6229597 9597504 0 gen:do_call/4
===============================================================================================================================
'[email protected]' 15:41:04
Load: cpu 88 Memory: total 2863781 binary 2297959
procs 1618 processes 459709 code 15692
runq 14 atom 687 ets 55221
Pid Name or Initial Func Time Reds Memory MsgQ Current Function
-------------------------------------------------------------------------------------------------------------------------------
<6939.1401.0> leveled_penciller:init/1 '-' 7579171 19900096 0 gen_server:loop/6
<6939.1465.0> leveled_penciller:init/1 '-' 7456935 19900096 0 gen_server:loop/6
<6939.1108.0> leveled_penciller:init/1 '-' 7412606 16583712 0 gen:do_call/4
<6939.1069.0> leveled_penciller:init/1 '-' 7308190 16583568 0 gen_server:loop/6
<6939.1178.0> leveled_penciller:init/1 '-' 7278689 16583568 0 gen_server:loop/6
<6939.1259.0> leveled_penciller:init/1 '-' 7963526 16583568 0 gen_server:loop/6
<6939.1378.0> leveled_penciller:init/1 '-' 7579770 16583568 0 gen_server:loop/6
<6939.1553.0> leveled_penciller:init/1 '-' 7458619 16583568 0 gen_server:loop/6
<6939.1595.0> leveled_penciller:init/1 '-' 7879662 16583568 0 gen_server:loop/6
<6939.1632.0> leveled_penciller:init/1 '-' 7711171 13819792 0 gen_server:loop/6
===============================================================================================================================
'[email protected]' 15:42:03
Load: cpu 89 Memory: total 3618586 binary 2987264
procs 1620 processes 533085 code 15692
runq 0 atom 687 ets 48546
Pid Name or Initial Func Time Reds Memory MsgQ Current Function
-------------------------------------------------------------------------------------------------------------------------------
<6939.1553.0> leveled_penciller:init/1 '-' 8705445 28655728 0 gen_server:loop/6
<6939.1178.0> leveled_penciller:init/1 '-' 8461192 23879928 0 gen_server:loop/6
<6939.1259.0> leveled_penciller:init/1 '-' 8820445 23879928 0 gen_server:loop/6
<6939.1378.0> leveled_penciller:init/1 '-' 8765942 23879928 0 gen_server:loop/6
<6939.1465.0> leveled_penciller:init/1 '-' 8523984 23879928 0 gen_server:loop/6
<6939.1595.0> leveled_penciller:init/1 '-' 8697428 23879928 0 gen_server:loop/6
<6939.1632.0> leveled_penciller:init/1 '-' 9126851 23879928 0 gen_server:loop/6
<6939.1188.0> leveled_penciller:init/1 '-' 8496312 19900200 1 gen_server:loop/6
<6939.1069.0> leveled_penciller:init/1 '-' 8414970 19900096 0 gen_server:loop/6
<6939.1307.0> leveled_penciller:init/1 '-' 7982036 19900096 0 gen_server:loop/6
===============================================================================================================================
'[email protected]' 15:43:03
Load: cpu 89 Memory: total 4375789 binary 3664650
procs 1631 processes 611795 code 15692
runq 6 atom 687 ets 50669
Pid Name or Initial Func Time Reds Memory MsgQ Current Function
-------------------------------------------------------------------------------------------------------------------------------
<6939.1595.0> leveled_penciller:init/1 '-' 9980089 41263840 0 gen_server:loop/6
<6939.1632.0> leveled_penciller:init/1 '-' 8136472 34386760 0 gen_server:loop/6
<6939.1259.0> leveled_penciller:init/1 '-' 9764615 34386688 0 gen_server:loop/6
<6939.1401.0> leveled_penciller:init/1 '-' 9741561 34386688 0 gen_server:loop/6
<6939.1378.0> leveled_penciller:init/1 '-' 9672702 28655872 0 gen:do_call/4
<6939.1465.0> leveled_penciller:init/1 '-' 9603231 28655872 0 gen:do_call/4
<6939.1553.0> leveled_penciller:init/1 '-' 9816173 28655872 0 gen:do_call/4
<6939.1178.0> leveled_penciller:init/1 '-' 9696122 23880072 0 gen:do_call/4
<6939.1069.0> leveled_penciller:init/1 '-' 9657140 23879928 0 gen_server:loop/6
<6939.1108.0> leveled_penciller:init/1 '-' 9936938 23879928 0 gen_server:loop/6
===============================================================================================================================
'[email protected]' 15:44:02
Load: cpu 89 Memory: total 4399240 binary 3711751
procs 1626 processes 581939 code 15692
runq 2 atom 687 ets 56615
Pid Name or Initial Func Time Reds Memory MsgQ Current Function
-------------------------------------------------------------------------------------------------------------------------------
<6939.1108.0> leveled_penciller:init/1 '-' 10623587 41263840 0 gen_server:loop/6
<6939.1378.0> leveled_penciller:init/1 '-' 10945328 34386832 0 gen:do_call/4
<6939.1069.0> leveled_penciller:init/1 '-' 10554834 34386688 0 gen_server:loop/6
<6939.1178.0> leveled_penciller:init/1 '-' 10512962 34386688 0 gen_server:loop/6
<6939.1188.0> leveled_penciller:init/1 '-' 10437297 34386688 0 gen_server:loop/6
<6939.1307.0> leveled_penciller:init/1 '-' 10378232 28655728 0 gen_server:loop/6
<6939.1465.0> leveled_penciller:init/1 '-' 10708127 28655728 0 gen_server:loop/6
<6939.1595.0> leveled_penciller:init/1 '-' 11015659 28655728 0 gen_server:loop/6
<6939.1401.0> leveled_penciller:init/1 '-' 10715454 23879928 0 gen_server:loop/6
<6939.1553.0> leveled_penciller:init/1 '-' 10706196 23879928 0 gen_server:loop/6
===============================================================================================================================
'[email protected]' 15:45:02
Load: cpu 90 Memory: total 4810859 binary 4072077
procs 1642 processes 655940 code 15692
runq 21 atom 687 ets 42782
Pid Name or Initial Func Time Reds Memory MsgQ Current Function
-------------------------------------------------------------------------------------------------------------------------------
<6939.1378.0> leveled_penciller:init/1 '-' 11067167 49516568 0 gen:do_call/4
<6939.1188.0> leveled_penciller:init/1 '-' 11402072 49516424 0 gen_server:loop/6
<6939.1595.0> leveled_penciller:init/1 '-' 12012806 41263984 0 gen:do_call/4
<6939.1178.0> leveled_penciller:init/1 '-' 11678635 41263840 0 gen_server:loop/6
<6939.1307.0> leveled_penciller:init/1 '-' 11174766 41263840 0 gen_server:loop/6
<6939.1401.0> leveled_penciller:init/1 '-' 11575632 41263840 0 gen_server:loop/6
<6939.1465.0> leveled_penciller:init/1 '-' 11842006 34386832 0 gen:do_call/4
<6939.1553.0> leveled_penciller:init/1 '-' 11708280 34386688 0 gen_server:loop/6
<6939.1069.0> leveled_penciller:init/1 '-' 11440593 28655832 1 gen_server:loop/6
<6939.1259.0> leveled_penciller:init/1 '-' 5030078 6665112 0 gen_server:loop/6
===============================================================================================================================
'[email protected]' 15:46:01
Load: cpu 92 Memory: total 1763149 binary 1296867
procs 1690 processes 378789 code 15692
runq 0 atom 687 ets 41402
Pid Name or Initial Func Time Reds Memory MsgQ Current Function
-------------------------------------------------------------------------------------------------------------------------------
<6939.1307.0> leveled_penciller:init/1 '-' 10174665 49516568 0 gen:do_call/4
<6939.1632.0> leveled_penciller:init/1 '-' 6365291 16583568 0 gen_server:loop/6
<6939.1259.0> leveled_penciller:init/1 '-' 5688364 11516648 0 gen_server:loop/6
<6939.1108.0> leveled_penciller:init/1 '-' 4776714 7998024 0 gen_server:loop/6
<6939.571.0> exometer_proc:'-spawn_pro '-' 3007446 6665184 0 exometer_probe:loop/1
<6939.3572.0> exometer_proc:'-spawn_pro '-' 2879775 6665184 0 exometer_probe:loop/1
<6939.1378.0> leveled_penciller:init/1 '-' 3937160 4119784 1 gen:do_call/4
<6939.1595.0> leveled_penciller:init/1 '-' 4922831 4119752 0 gen:do_call/4
<6939.1069.0> leveled_penciller:init/1 '-' 5095490 4119712 1 gen_server:loop/6
<6939.546.0> exometer_proc:'-spawn_pro '-' 1538401 4119680 0 exometer_probe:loop/1
===============================================================================================================================
'[email protected]' 15:47:01
Load: cpu 89 Memory: total 2022254 binary 1534867
procs 1636 processes 395109 code 15692
runq 17 atom 687 ets 42276
Pid Name or Initial Func Time Reds Memory MsgQ Current Function
-------------------------------------------------------------------------------------------------------------------------------
<6939.1632.0> leveled_penciller:init/1 '-' 8185205 23880032 1 gen_server:loop/6
<6939.1259.0> leveled_penciller:init/1 '-' 7448015 19900240 0 gen:do_call/4
<6939.1108.0> leveled_penciller:init/1 '-' 6341565 16583568 0 gen_server:loop/6
<6939.1553.0> leveled_penciller:init/1 '-' 5536025 9597504 0 gen:do_call/4
<6939.1178.0> leveled_penciller:init/1 '-' 5509690 9597464 1 gen_server:loop/6
<6939.1378.0> leveled_penciller:init/1 '-' 5540570 9597360 0 gen_server:loop/6
<6939.571.0> exometer_proc:'-spawn_pro '-' 3180845 7998056 1 exometer_probe:loop/1
<6939.1069.0> leveled_penciller:init/1 '-' 5566731 7997952 0 gen_server:loop/6
<6939.1595.0> leveled_penciller:init/1 '-' 5485061 7997952 0 gen_server:loop/6
<6939.1401.0> leveled_penciller:init/1 '-' 4911579 6665184 0 gen_server:loop/6
===============================================================================================================================
'[email protected]' 15:48:00
Load: cpu 89 Memory: total 2774364 binary 2216575
procs 1638 processes 469685 code 15692
runq 20 atom 687 ets 41649
Pid Name or Initial Func Time Reds Memory MsgQ Current Function
-------------------------------------------------------------------------------------------------------------------------------
<6939.1632.0> leveled_penciller:init/1 '-' 9228675 34386688 0 gen_server:loop/6
<6939.1259.0> leveled_penciller:init/1 '-' 8718346 28655872 0 gen:do_call/4
<6939.1108.0> leveled_penciller:init/1 '-' 7415775 19900096 0 gen_server:loop/6
<6939.1378.0> leveled_penciller:init/1 '-' 6577606 16583568 0 gen_server:loop/6
<6939.1069.0> leveled_penciller:init/1 '-' 6459144 13819792 0 gen_server:loop/6
<6939.1178.0> leveled_penciller:init/1 '-' 6443197 13819792 0 gen_server:loop/6
<6939.1465.0> leveled_penciller:init/1 '-' 6308713 13819792 0 gen_server:loop/6
<6939.1595.0> leveled_penciller:init/1 '-' 6377798 13819792 0 gen_server:loop/6
<6939.1553.0> leveled_penciller:init/1 '-' 6526398 11516648 0 gen_server:loop/6
<6939.1188.0> leveled_penciller:init/1 '-' 6004619 9597360 0 gen_server:loop/6
from leveled.
30 minutes after restart of load:
===============================================================================================================================
'[email protected]' 16:07:51
Load: cpu 89 Memory: total 3227262 binary 2615665
procs 1766 processes 519283 code 15692
runq 4 atom 687 ets 44577
Pid Name or Initial Func Time Reds Memory MsgQ Current Function
-------------------------------------------------------------------------------------------------------------------------------
<6939.1553.0> leveled_penciller:init/1 '-' 11581132 41264016 1 gen:do_call/4
<6939.1259.0> leveled_penciller:init/1 '-' 8115225 28655728 0 gen_server:loop/6
<6939.1307.0> leveled_penciller:init/1 '-' 11193659 28655728 0 gen_server:loop/6
<6939.1632.0> leveled_penciller:init/1 '-' 9693028 28655728 0 gen_server:loop/6
<6939.25814.108> leveled_penciller:init/1 '-' 0 28655728 0 gen_server:loop/6
<6939.1108.0> leveled_penciller:init/1 '-' 7697013 16583568 0 gen_server:loop/6
<6939.1401.0> leveled_penciller:init/1 '-' 6041415 13819968 1 gen:do_call/4
<6939.5883.104> leveled_penciller:init/1 '-' 0 11516648 0 gen_server:loop/6
<6939.1378.0> leveled_penciller:init/1 '-' 5628977 9597360 0 gen_server:loop/6
<6939.571.0> exometer_proc:'-spawn_pro '-' 3070730 6665184 0 exometer_probe:loop/1
===============================================================================================================================
'[email protected]' 16:08:50
Load: cpu 89 Memory: total 2718965 binary 2148425
procs 1763 processes 471491 code 15692
runq 16 atom 687 ets 48432
Pid Name or Initial Func Time Reds Memory MsgQ Current Function
-------------------------------------------------------------------------------------------------------------------------------
<6939.1632.0> leveled_penciller:init/1 '-' 10801673 34386832 0 gen:do_call/4
<6939.25814.108> leveled_penciller:init/1 '-' 0 28655728 0 gen_server:loop/6
<6939.1108.0> leveled_penciller:init/1 '-' 9064055 23879928 0 gen_server:loop/6
<6939.1401.0> leveled_penciller:init/1 '-' 7282878 16583568 0 gen_server:loop/6
<6939.1378.0> leveled_penciller:init/1 '-' 6737633 13819792 0 gen_server:loop/6
<6939.5883.104> leveled_penciller:init/1 '-' 0 11516648 0 gen_server:loop/6
<6939.2865.188> leveled_penciller:init/1 '-' 371018 9597504 0 gen:do_call/4
<6939.1069.0> leveled_penciller:init/1 '-' 5273273 9597360 0 gen_server:loop/6
<6939.1188.0> leveled_penciller:init/1 '-' 5316513 9597360 0 gen_server:loop/6
<6939.1178.0> leveled_penciller:init/1 '-' 5719378 7997952 0 gen_server:loop/6
===============================================================================================================================
'[email protected]' 16:09:50
Load: cpu 91 Memory: total 3514414 binary 2822651
procs 1766 processes 578956 code 15692
runq 1 atom 687 ets 64714
Pid Name or Initial Func Time Reds Memory MsgQ Current Function
-------------------------------------------------------------------------------------------------------------------------------
<6939.1108.0> leveled_penciller:init/1 '-' 9732553 34386688 0 gen_server:loop/6
<6939.1632.0> leveled_penciller:init/1 '-' 11648210 34386688 0 gen_server:loop/6
<6939.1378.0> leveled_penciller:init/1 '-' 7982211 28655728 0 gen_server:loop/6
<6939.25814.108> leveled_penciller:init/1 '-' 0 28655728 0 gen_server:loop/6
<6939.1401.0> leveled_penciller:init/1 '-' 8080965 23879928 0 gen_server:loop/6
<6939.2865.188> leveled_penciller:init/1 '-' 9757215 23879928 0 gen_server:loop/6
<6939.1178.0> leveled_penciller:init/1 '-' 6665086 19900096 0 gen_server:loop/6
<6939.1188.0> leveled_penciller:init/1 '-' 6232072 16583568 0 gen_server:loop/6
<6939.1465.0> leveled_penciller:init/1 '-' 6957384 13819968 1 gen:do_call/4
<6939.1069.0> leveled_penciller:init/1 '-' 6296522 13819792 0 gen_server:loop/6
from leveled.
Two hours on ...
===============================================================================================================================
'[email protected]' 17:57:24
Load: cpu 89 Memory: total 4241248 binary 3260079
procs 2941 processes 875105 code 15692
runq 3 atom 687 ets 52077
Pid Name or Initial Func Time Reds Memory MsgQ Current Function
-------------------------------------------------------------------------------------------------------------------------------
<6939.1188.0> leveled_penciller:init/1 '-' 8962497 34386832 0 gen:do_call/4
<6939.1378.0> leveled_penciller:init/1 '-' 9180728 28655728 0 gen_server:loop/6
<6939.25814.108> leveled_penciller:init/1 '-' 0 28655728 0 gen_server:loop/6
<6939.1553.0> leveled_penciller:init/1 '-' 7988133 23880072 0 gen:do_call/4
<6939.29138.730> leveled_penciller:init/1 '-' 0 23879928 0 gen_server:loop/6
<6939.761.0> leveled_iclerk:init/1 '-' 0 19900096 0 gen_server:loop/6
<6939.766.0> leveled_iclerk:init/1 '-' 0 19900096 0 gen_server:loop/6
<6939.1069.0> leveled_penciller:init/1 '-' 6776950 19900096 0 gen_server:loop/6
<6939.1108.0> leveled_penciller:init/1 '-' 8265675 19900096 0 gen_server:loop/6
<6939.1307.0> leveled_penciller:init/1 '-' 10034834 19900096 0 gen_server:loop/6
===============================================================================================================================
'[email protected]' 17:58:23
Load: cpu 88 Memory: total 4915227 binary 3889784
procs 2937 processes 929746 code 15692
runq 11 atom 687 ets 43310
Pid Name or Initial Func Time Reds Memory MsgQ Current Function
-------------------------------------------------------------------------------------------------------------------------------
<6939.1188.0> leveled_penciller:init/1 '-' 10374868 34386688 0 gen_server:loop/6
<6939.1378.0> leveled_penciller:init/1 '-' 10271046 28655904 1 gen:do_call/4
<6939.1069.0> leveled_penciller:init/1 '-' 8044958 28655800 0 gen_server:loop/6
<6939.1307.0> leveled_penciller:init/1 '-' 10844972 28655728 0 gen_server:loop/6
<6939.25814.108> leveled_penciller:init/1 '-' 0 28655728 0 gen_server:loop/6
<6939.1108.0> leveled_penciller:init/1 '-' 9409542 23879928 0 gen_server:loop/6
<6939.1553.0> leveled_penciller:init/1 '-' 9600560 23879928 0 gen_server:loop/6
<6939.29138.730> leveled_penciller:init/1 '-' 0 23879928 0 gen_server:loop/6
<6939.1178.0> leveled_penciller:init/1 '-' 8026100 19900272 1 gen:do_call/4
<6939.761.0> leveled_iclerk:init/1 '-' 0 19900096 0 gen_server:loop/6
from leveled.
Changing the iclerk made no difference. However it was noticeable that each node had many more penciller processes than was expected, and also a lot of logs indicating that snapshots had been timed out after 1 hour.
The iclerk/inker code was not releasing/closing the Penciller snapshot. This would lead to penciller processes hanging around, as well as historic SST files.
from leveled.
Running a new test with the inker now correct releasing snapshots:
1-hour in
===============================================================================================================================
'[email protected]' 11:10:03
Load: cpu 88 Memory: total 2822051 binary 2311309
procs 1284 processes 419432 code 15918
runq 19 atom 719 ets 43678
Pid Name or Initial Func Time Reds Memory MsgQ Current Function
-------------------------------------------------------------------------------------------------------------------------------
<6940.1847.0> leveled_penciller:init/1 '-' 13615667 41263840 0 gen_server:loop/6
<6940.766.0> leveled_penciller:init/1 '-' 15290795 34386688 0 gen_server:loop/6
<6940.2568.0> leveled_penciller:init/1 '-' 11942783 34386688 0 gen_server:loop/6
<6940.2566.0> leveled_penciller:init/1 '-' 10906979 28655728 0 gen_server:loop/6
<6940.11618.609> leveled_sst:init/1 '-' 1598310 23879936 0 lists:last/1
<6940.1143.0> leveled_penciller:init/1 '-' 9654682 19900096 0 gen_server:loop/6
<6940.1879.0> leveled_penciller:init/1 '-' 7444992 13819792 0 gen_server:loop/6
<6940.1527.0> leveled_penciller:init/1 '-' 6092460 9597360 0 gen_server:loop/6
<6940.1513.0> leveled_penciller:init/1 '-' 7112787 7997952 0 gen_server:loop/6
<6940.531.0> exometer_proc:'-spawn_pro '-' 4175181 4119680 0 exometer_probe:loop/1
<6940.3570.0> exometer_proc:'-spawn_pro '-' 4132860 4119680 0 exometer_probe:loop/1
<6940.764.0> leveled_penciller:init/1 '-' 8769377 4119608 0 gen_server:loop/6
<6940.1887.0> leveled_penciller:init/1 '-' 7454490 4119608 0 gen_server:loop/6
<6940.166.0> application_master:start_ '-' 0 4119584 0 application_master:loop_it/4
<6940.508.0> exometer_proc:'-spawn_pro '-' 2061642 2546504 1 exometer_probe:loop/1
<6940.3699.0> exometer_proc:'-spawn_pro '-' 220110 2546504 1 exometer_probe:loop/1
<6940.506.0> exometer_proc:'-spawn_pro '-' 1161223 2546472 0 exometer_probe:loop/1
<6940.527.0> exometer_proc:'-spawn_pro '-' 1103307 2546472 0 exometer_probe:loop/1
<6940.530.0> exometer_proc:'-spawn_pro '-' 1115237 2546472 0 exometer_probe:loop/1
<6940.554.0> exometer_proc:'-spawn_pro '-' 750690 2546472 0 exometer_probe:loop/1
===============================================================================================================================
'[email protected]' 11:11:02
Load: cpu 87 Memory: total 2837965 binary 2351458
procs 1259 processes 386449 code 15918
runq 10 atom 719 ets 51312
Pid Name or Initial Func Time Reds Memory MsgQ Current Function
-------------------------------------------------------------------------------------------------------------------------------
<6940.2568.0> leveled_penciller:init/1 '-' 14133178 41263984 0 gen:do_call/4
<6940.1143.0> leveled_penciller:init/1 '-' 11819457 34386688 0 gen_server:loop/6
<6940.2566.0> leveled_penciller:init/1 '-' 13423332 34386688 0 gen_server:loop/6
<6940.1879.0> leveled_penciller:init/1 '-' 9993779 16583568 0 gen_server:loop/6
<6940.1513.0> leveled_penciller:init/1 '-' 9143695 13819792 0 gen_server:loop/6
<6940.1527.0> leveled_penciller:init/1 '-' 8138719 13819792 0 gen_server:loop/6
<6940.1887.0> leveled_penciller:init/1 '-' 7035062 11516648 0 gen_server:loop/6
<6940.1486.0> leveled_penciller:init/1 '-' 6159866 9597360 0 gen_server:loop/6
<6940.1140.0> leveled_penciller:init/1 '-' 6169036 7997952 0 gen_server:loop/6
<6940.764.0> leveled_penciller:init/1 '-' 6904841 6665112 0 gen_server:loop/6
<6940.3570.0> exometer_proc:'-spawn_pro '-' 4293185 4119712 1 exometer_probe:loop/1
<6940.508.0> exometer_proc:'-spawn_pro '-' 2196992 4119680 0 exometer_probe:loop/1
<6940.531.0> exometer_proc:'-spawn_pro '-' 4322660 4119680 0 exometer_probe:loop/1
<6940.766.0> leveled_penciller:init/1 '-' 5648482 4119608 0 gen_server:loop/6
<6940.166.0> application_master:start_ '-' 0 4119584 0 application_master:loop_it/4
<6940.506.0> exometer_proc:'-spawn_pro '-' 1381152 2546472 0 exometer_probe:loop/1
<6940.527.0> exometer_proc:'-spawn_pro '-' 1144629 2546472 0 exometer_probe:loop/1
<6940.530.0> exometer_proc:'-spawn_pro '-' 1125183 2546472 0 exometer_probe:loop/1
<6940.554.0> exometer_proc:'-spawn_pro '-' 740244 2546472 0 exometer_probe:loop/1
<6940.3691.0> exometer_proc:'-spawn_pro '-' 236237 2546472 0 exometer_probe:loop/1
At 2.5 hours into the test - the memory footprint continues to be stable (and significantly better than before the change)
===============================================================================================================================
'[email protected]' 12:39:05
Load: cpu 88 Memory: total 2975311 binary 2445896
procs 1613 processes 446097 code 15918
runq 6 atom 719 ets 38004
Pid Name or Initial Func Time Reds Memory MsgQ Current Function
-------------------------------------------------------------------------------------------------------------------------------
<6940.2566.0> leveled_penciller:init/1 '-' 11361768 34386792 1 gen_server:loop/6
<6940.1140.0> leveled_penciller:init/1 '-' 9928924 34386688 0 gen_server:loop/6
<6940.1887.0> leveled_penciller:init/1 '-' 12059806 34386688 0 gen_server:loop/6
<6940.1527.0> leveled_penciller:init/1 '-' 12168039 28655832 1 gen_server:loop/6
<6940.1143.0> leveled_penciller:init/1 '-' 8064581 19900096 0 gen_server:loop/6
<6940.1486.0> leveled_penciller:init/1 '-' 7714559 16583744 1 gen:do_call/4
<6940.1513.0> leveled_penciller:init/1 '-' 8757016 16583568 0 gen_server:loop/6
<6940.1847.0> leveled_penciller:init/1 '-' 6062620 7998056 1 gen_server:loop/6
<6940.766.0> leveled_penciller:init/1 '-' 5542404 7997952 0 gen_server:loop/6
<6940.1879.0> leveled_penciller:init/1 '-' 5034953 4119712 1 gen_server:loop/6
<6940.531.0> exometer_proc:'-spawn_pro '-' 3454103 4119680 0 exometer_probe:loop/1
<6940.573.0> exometer_proc:'-spawn_pro '-' 573676 4119680 0 exometer_probe:loop/1
<6940.3570.0> exometer_proc:'-spawn_pro '-' 3492477 4119680 0 exometer_probe:loop/1
<6940.166.0> application_master:start_ '-' 0 4119584 0 application_master:loop_it/4
<6940.506.0> exometer_proc:'-spawn_pro '-' 1674209 2546472 0 exometer_probe:loop/1
<6940.508.0> exometer_proc:'-spawn_pro '-' 1782209 2546472 0 exometer_probe:loop/1
<6940.527.0> exometer_proc:'-spawn_pro '-' 1493985 2546472 0 exometer_probe:loop/1
<6940.530.0> exometer_proc:'-spawn_pro '-' 1475428 2546472 0 exometer_probe:loop/1
<6940.554.0> exometer_proc:'-spawn_pro '-' 638783 2546472 0 exometer_probe:loop/1
<6940.3691.0> exometer_proc:'-spawn_pro '-' 189807 2546472 0 exometer_probe:loop/1
===============================================================================================================================
'[email protected]' 12:40:05
Load: cpu 88 Memory: total 2944265 binary 2403827
procs 1609 processes 447289 code 15918
runq 6 atom 719 ets 45653
Pid Name or Initial Func Time Reds Memory MsgQ Current Function
-------------------------------------------------------------------------------------------------------------------------------
<6940.1887.0> leveled_penciller:init/1 '-' 11606129 49516424 0 gen_server:loop/6
<6940.1140.0> leveled_penciller:init/1 '-' 11497457 34386688 0 gen_server:loop/6
<6940.1513.0> leveled_penciller:init/1 '-' 10112276 34386688 0 gen_server:loop/6
<6940.1486.0> leveled_penciller:init/1 '-' 9090052 23879928 0 gen_server:loop/6
<6940.1143.0> leveled_penciller:init/1 '-' 9277196 19900096 0 gen_server:loop/6
<6940.1847.0> leveled_penciller:init/1 '-' 7263336 11516792 0 gen:do_call/4
<6940.766.0> leveled_penciller:init/1 '-' 6829559 11516648 0 gen_server:loop/6
<6940.1879.0> leveled_penciller:init/1 '-' 6433885 11516648 0 gen_server:loop/6
<6940.2568.0> leveled_penciller:init/1 '-' 5339053 4119752 0 gen:do_call/4
<6940.508.0> exometer_proc:'-spawn_pro '-' 1641733 4119712 1 exometer_probe:loop/1
<6940.531.0> exometer_proc:'-spawn_pro '-' 3554800 4119680 0 exometer_probe:loop/1
<6940.573.0> exometer_proc:'-spawn_pro '-' 610379 4119680 0 exometer_probe:loop/1
<6940.3570.0> exometer_proc:'-spawn_pro '-' 3499092 4119680 0 exometer_probe:loop/1
<6940.764.0> leveled_penciller:init/1 '-' 5027407 4119608 0 gen_server:loop/6
<6940.166.0> application_master:start_ '-' 0 4119584 0 application_master:loop_it/4
<6940.506.0> exometer_proc:'-spawn_pro '-' 1624338 2546472 0 exometer_probe:loop/1
<6940.527.0> exometer_proc:'-spawn_pro '-' 1598591 2546472 0 exometer_probe:loop/1
<6940.530.0> exometer_proc:'-spawn_pro '-' 1506774 2546472 0 exometer_probe:loop/1
<6940.3716.0> exometer_proc:'-spawn_pro '-' 188218 2546472 0 exometer_probe:loop/1
<6940.3728.0> exometer_proc:'-spawn_pro '-' 188422 2546472 0 exometer_probe:loop/1
from leveled.
Resolved by #70
from leveled.
Related Issues (20)
- OTP 24 Support HOT 1
- Reduce open files HOT 5
- Inker delete_confirmed call not cast HOT 1
- Selective sync HOT 2
- Microseconds LMD HOT 1
- Recalc and revolving objects HOT 1
- confirm_delete race HOT 2
- Redact server state on crash HOT 2
- Adjust defaults to reflect higher vnode counts HOT 2
- Handle hash of 0 correctly in leveled_cdb HOT 1
- Experiment with zstd HOT 1
- Memory utilisation for large stores HOT 2
- Snapshot time
- Outdated deps preventing build HOT 3
- Potential Issue re delete post-compaction HOT 1
- Use standard erlang logger HOT 1
- Rebuilding ledger - but first 50K gone HOT 1
- leveled_monitor - sample_period HOT 1
- Penciller work backlog HOT 2
- Never stalling is an issue HOT 1
Recommend Projects
-
React
A declarative, efficient, and flexible JavaScript library for building user interfaces.
-
Vue.js
🖖 Vue.js is a progressive, incrementally-adoptable JavaScript framework for building UI on the web.
-
Typescript
TypeScript is a superset of JavaScript that compiles to clean JavaScript output.
-
TensorFlow
An Open Source Machine Learning Framework for Everyone
-
Django
The Web framework for perfectionists with deadlines.
-
Laravel
A PHP framework for web artisans
-
D3
Bring data to life with SVG, Canvas and HTML. 📊📈🎉
-
Recommend Topics
-
javascript
JavaScript (JS) is a lightweight interpreted programming language with first-class functions.
-
web
Some thing interesting about web. New door for the world.
-
server
A server is a program made to process requests and deliver data to clients.
-
Machine learning
Machine learning is a way of modeling and interpreting data that allows a piece of software to respond intelligently.
-
Visualization
Some thing interesting about visualization, use data art
-
Game
Some thing interesting about game, make everyone happy.
Recommend Org
-
Facebook
We are working to build community through open source technology. NB: members must have two-factor auth.
-
Microsoft
Open source projects and samples from Microsoft.
-
Google
Google ❤️ Open Source for everyone.
-
Alibaba
Alibaba Open Source for everyone
-
D3
Data-Driven Documents codes.
-
Tencent
China tencent open source team.
from leveled.