diff --git a/apps/arweave/include/ar_config.hrl b/apps/arweave/include/ar_config.hrl index 095fb70bc..b17dc834b 100644 --- a/apps/arweave/include/ar_config.hrl +++ b/apps/arweave/include/ar_config.hrl @@ -172,6 +172,11 @@ -define(DEFAULT_COWBOY_TCP_SEND_TIMEOUT, 15_000). -define(DEFAULT_COWBOY_TCP_LISTENER_SHUTDOWN, 5000). +%% The default OOM monitor report period in milliseconds. +-define(DEFAULT_OOM_MONITOR_REPORT_PERIOD_MS, 5000). +-define(DEFAULT_OOM_MONITOR_FILENAME, "/tmp/arweave_oom_monitor.log"). +-define(DEFAULT_OOM_MONITOR_TOP_PROCS, 20). + %% @doc Startup options with default values. -record(config, { init = false, @@ -313,7 +318,12 @@ 'http_api.tcp.nodelay' = ?DEFAULT_COWBOY_TCP_NODELAY, 'http_api.tcp.num_acceptors' = ?DEFAULT_COWBOY_TCP_NUM_ACCEPTORS, 'http_api.tcp.send_timeout_close' = ?DEFAULT_COWBOY_TCP_SEND_TIMEOUT_CLOSE, - 'http_api.tcp.send_timeout' = ?DEFAULT_COWBOY_TCP_SEND_TIMEOUT + 'http_api.tcp.send_timeout' = ?DEFAULT_COWBOY_TCP_SEND_TIMEOUT, + + % OOM Monitor Configuration + oom_monitor_report_period = ?DEFAULT_OOM_MONITOR_REPORT_PERIOD_MS, + oom_monitor_filename = ?DEFAULT_OOM_MONITOR_FILENAME, + oom_monitor_top_procs = ?DEFAULT_OOM_MONITOR_TOP_PROCS }). -endif. diff --git a/apps/arweave/src/ar.erl b/apps/arweave/src/ar.erl index 24b31aaa5..f3076067f 100644 --- a/apps/arweave/src/ar.erl +++ b/apps/arweave/src/ar.erl @@ -1126,6 +1126,35 @@ parse_cli_args(["http_api.tcp.send_timeout", Timeout|Rest], C) -> parse_cli_args(Rest, C) end; +parse_cli_args(["oom_monitor_report_period", Period|Rest], C) -> + try list_to_integer(Period) of + P when P >= 0 -> + parse_cli_args(Rest, C#config{ oom_monitor_report_period = P }); + _ -> + io:format("Invalid oom_monitor_report_period ~p.", [Period]), + parse_cli_args(Rest, C) + catch + _:_ -> + io:format("Invalid oom_monitor_report_period ~p.", [Period]), + parse_cli_args(Rest, C) + end; + +parse_cli_args(["oom_monitor_filename", Filename|Rest], C) -> + parse_cli_args(Rest, C#config{ oom_monitor_filename = Filename }); + +parse_cli_args(["oom_monitor_top_procs", TopProcs|Rest], C) -> + try list_to_integer(TopProcs) of + T when T >= 0 -> + parse_cli_args(Rest, C#config{ oom_monitor_top_procs = T }); + _ -> + io:format("Invalid oom_monitor_top_procs ~p.", [TopProcs]), + parse_cli_args(Rest, C) + catch + _:_ -> + io:format("Invalid oom_monitor_top_procs ~p.", [TopProcs]), + parse_cli_args(Rest, C) + end; + %% Undocumented/unsupported options parse_cli_args(["chunk_storage_file_size", Num | Rest], C) -> parse_cli_args(Rest, C#config{ chunk_storage_file_size = list_to_integer(Num) }); diff --git a/apps/arweave/src/ar_config.erl b/apps/arweave/src/ar_config.erl index 91026d2ef..d34d870b6 100644 --- a/apps/arweave/src/ar_config.erl +++ b/apps/arweave/src/ar_config.erl @@ -2,7 +2,8 @@ -export([validate_config/1, set_dependent_flags/1, use_remote_vdf_server/0, pull_from_remote_vdf_server/0, compute_own_vdf/0, is_vdf_server/0, - is_public_vdf_server/0, parse/1, parse_storage_module/1, log_config/1]). + is_public_vdf_server/0, is_oom_monitor_enabled/0, parse/1, parse_storage_module/1, + log_config/1]). -include("../include/ar.hrl"). -include("../include/ar_consensus.hrl"). @@ -65,6 +66,10 @@ is_public_vdf_server() -> {ok, Config} = application:get_env(arweave, config), lists:member(public_vdf_server, Config#config.enable). +is_oom_monitor_enabled() -> + {ok, Config} = application:get_env(arweave, config), + lists:member(oom_monitor, Config#config.enable). + parse(Config) when is_binary(Config) -> case ar_serialize:json_decode(Config) of {ok, JsonValue} -> parse_options(JsonValue); @@ -916,6 +921,23 @@ parse_options([{<<"http_api.tcp.send_timeout">>, Timeout}|Rest], Config) -> {error, {bad_value, 'http_api.tcp.send_timeout'}, Timeout} end; +parse_options([{<<"oom_monitor_report_period">>, Period} | Rest], Config) + when is_integer(Period), Period > 0 -> + parse_options(Rest, Config#config{ oom_monitor_report_period = Period * 1000 }); +parse_options([{<<"oom_monitor_report_period">>, Period} | _], _) -> + {error, {bad_type, oom_monitor_report_period, number}, Period}; + +parse_options([{<<"oom_monitor_filename">>, Filename} | Rest], Config) when is_binary(Filename) -> + parse_options(Rest, Config#config{ oom_monitor_filename = binary_to_list(Filename) }); +parse_options([{<<"oom_monitor_filename">>, Filename} | _], _) -> + {error, {bad_type, oom_monitor_filename, string}, Filename}; + +parse_options([{<<"oom_monitor_top_procs">>, TopProcs} | Rest], Config) + when is_integer(TopProcs), TopProcs > 0 -> + parse_options(Rest, Config#config{ oom_monitor_top_procs = TopProcs }); +parse_options([{<<"oom_monitor_top_procs">>, TopProcs} | _], _) -> + {error, {bad_type, oom_monitor_top_procs, number}, TopProcs}; + parse_options([Opt | _], _) -> {error, unknown, Opt}; parse_options([], Config) -> @@ -1268,4 +1290,3 @@ set_verify_flags(Config) -> max_propagation_peers = 0, max_block_propagation_peers = 0 }. - diff --git a/apps/arweave/src/ar_oom_monitor.erl b/apps/arweave/src/ar_oom_monitor.erl new file mode 100644 index 000000000..99da46bef --- /dev/null +++ b/apps/arweave/src/ar_oom_monitor.erl @@ -0,0 +1,389 @@ +%%% @doc OOM (Out of Memory) Monitor +%%% +%%% This module implements a gen_server that periodically monitors system memory +%%% usage and logs the information to a file. It's designed to help track memory +%%% consumption patterns and diagnose potential memory issues. +%%% +%%% The monitor reads configuration from the application environment: +%%% - oom_monitor_report_period: How often to report memory usage (milliseconds) +%%% - oom_monitor_filename: The filename to write memory reports to +%%% +%%% The monitor is only started if the 'oom_monitor' configuration flag is enabled. +-module(ar_oom_monitor). + +-behaviour(gen_server). + +-export([start_link/0]). + +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2]). + +-include_lib("arweave/include/ar.hrl"). +-include_lib("arweave/include/ar_config.hrl"). + +-define(MSG_REPORT_MEMORY, {report_memory}). +-define(DEFAULT_PROC_WINDOW_MS, 200). + +-record(memory_report, { + timestamp :: non_neg_integer(), + total_memory :: non_neg_integer(), + processes_memory :: non_neg_integer(), + system_memory :: non_neg_integer(), + atom_memory :: non_neg_integer(), + binary_memory :: non_neg_integer(), + code_memory :: non_neg_integer(), + ets_memory :: non_neg_integer(), + process_count :: non_neg_integer(), + port_count :: non_neg_integer(), + top_memory_processes :: [{pid(), non_neg_integer(), [term()]}], + top_binary_processes :: [{pid(), non_neg_integer(), [term()]}], + top_mailbox_processes :: [{pid(), non_neg_integer(), [term()]}], + growing_memory_processes :: [{pid(), non_neg_integer(), [term()]}], + growing_mailbox_processes :: [{pid(), non_neg_integer(), [term()]}], + bin_leak_processes :: [{pid(), non_neg_integer(), [term()]}], + allocator_info :: #{atom() => non_neg_integer()} +}). + +-record(state, { + report_period_ms :: non_neg_integer(), + filename :: string(), + top_procs :: non_neg_integer(), + file_handle :: file:io_device() | undefined, + last_memory_report :: #memory_report{} | undefined +}). + +%%%=================================================================== +%%% Public interface. +%%%=================================================================== + +%% @doc Start the gen_server. +start_link() -> + gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). + +%%%=================================================================== +%%% Generic server callbacks. +%%%=================================================================== + +init([]) -> + ?LOG_INFO([{event, ar_oom_monitor_init}]), + %% Trap exit to ensure we close the file handle cleanly + process_flag(trap_exit, true), + {ok, Config} = application:get_env(arweave, config), + ReportPeriod = Config#config.oom_monitor_report_period, + Filename = Config#config.oom_monitor_filename, + TopProcs = Config#config.oom_monitor_top_procs, + ok = filelib:ensure_dir(filename:dirname(Filename)), + + case file:open(Filename, [write, raw, {encoding, utf8}]) of + {ok, FileHandle} -> + ?LOG_INFO([ + {event, ar_oom_monitor_started}, {report_period_ms, ReportPeriod}, {filename, Filename}]), + write_header(FileHandle), + erlang:send_after(ReportPeriod, self(), ?MSG_REPORT_MEMORY), + {ok, #state{ + report_period_ms = ReportPeriod, + filename = Filename, + top_procs = TopProcs, + file_handle = FileHandle, + last_memory_report = undefined + }}; + {error, Reason} -> + ?LOG_ERROR([{event, ar_oom_monitor_file_open_error}, + {filename, Filename}, + {reason, Reason}]), + {stop, {file_open_error, Reason}} + end. + +handle_call(Request, _From, State) -> + ?LOG_WARNING([{event, unhandled_call}, {module, ?MODULE}, {request, Request}]), + {reply, ok, State}. + +handle_cast(Cast, State) -> + ?LOG_WARNING([{event, unhandled_cast}, {module, ?MODULE}, {cast, Cast}]), + {noreply, State}. + +handle_info(?MSG_REPORT_MEMORY, State) -> + #state{ + report_period_ms = ReportPeriod, + file_handle = FileHandle, + top_procs = TopProcs, + last_memory_report = LastMemoryReport + } = State, + MemoryInfo = collect_memory_info(TopProcs), + write_memory_report(FileHandle, MemoryInfo, LastMemoryReport), + erlang:send_after(ReportPeriod, self(), ?MSG_REPORT_MEMORY), + {noreply, State#state{last_memory_report = MemoryInfo}}; + +handle_info(Info, State) -> + ?LOG_WARNING([{event, unhandled_info}, {module, ?MODULE}, {info, Info}]), + {noreply, State}. + +terminate(_Reason, #state{file_handle = FileHandle}) -> + case FileHandle of + undefined -> + ok; + Handle -> + file:close(Handle) + end, + ok. + +%%%=================================================================== +%%% Private functions. +%%%=================================================================== + +write_header(FileHandle) -> + Timestamp = format_timestamp(erlang:system_time(millisecond)), + Header = io_lib:format( + "# ~s Arweave OOM Monitor~n" + "This file is automatically generated by the Arweave OOM Monitor.~n" + "It contains information about the memory usage of the Erlang VM and the processes running on it.~n" + "It is used to diagnose potential memory issues and to help prevent Out of Memory crashes.~n" + "This file will be overwritten upon node restart.~n~n~n", + [Timestamp]), + ok = file:write(FileHandle, Header), + ok = file:sync(FileHandle). + +collect_memory_info(TopProcs) -> + Memory = erlang:memory(), + + ProcessCount = erlang:system_info(process_count), + PortCount = erlang:system_info(port_count), + + TopMemoryProcs = get_top_processes(memory, TopProcs), + TopMailboxProcs = get_top_processes(message_queue_len, TopProcs), + TopBinaryProcs = get_top_binary_processes(TopProcs), + + GrowingMemoryProcs = get_growing_processes(memory, TopProcs), + GrowingMailboxProcs = get_growing_processes(message_queue_len, TopProcs), + BinLeakProcs = recon:bin_leak(TopProcs), + + AllocatorInfo = get_allocator_info(), + + #memory_report{ + timestamp = erlang:system_time(millisecond), + total_memory = proplists:get_value(total, Memory, 0), + processes_memory = proplists:get_value(processes, Memory, 0), + system_memory = proplists:get_value(system, Memory, 0), + atom_memory = proplists:get_value(atom, Memory, 0), + binary_memory = proplists:get_value(binary, Memory, 0), + code_memory = proplists:get_value(code, Memory, 0), + ets_memory = proplists:get_value(ets, Memory, 0), + process_count = ProcessCount, + port_count = PortCount, + top_memory_processes = TopMemoryProcs, + top_mailbox_processes = TopMailboxProcs, + top_binary_processes = TopBinaryProcs, + growing_memory_processes = GrowingMemoryProcs, + growing_mailbox_processes = GrowingMailboxProcs, + bin_leak_processes = BinLeakProcs, + allocator_info = AllocatorInfo + }. + +get_top_processes(Attribute, N) -> + recon:proc_count(Attribute, N). + +get_top_binary_processes(TopProcs) -> + Procs = lists:map(fun(Pid) -> + case erlang:process_info(Pid, [binary, registered_name, initial_call, current_function]) of + undefined -> {Pid, 0, []}; + Props -> + TotalSize = lists:sum([Size || {_, Size, _} <- proplists:get_value(binary, Props, [])]), + InitialCall = proplists:get_value(initial_call, Props, {unknown, unknown, 0}), + CurrentFunction = proplists:get_value(current_function, Props, {unknown, unknown, 0}), + case proplists:get_value(registered_name, Props, undefined) of + undefined -> {Pid, TotalSize, [{initial_call, InitialCall}, {current_function, CurrentFunction}]}; + RegName -> {Pid, TotalSize, [RegName, {initial_call, InitialCall}, {current_function, CurrentFunction}]} + end + end + end, erlang:processes()), + lists:sublist(lists:reverse(lists:keysort(2, Procs)), TopProcs). + +get_growing_processes(Attribute, Count) -> + try + recon:proc_window(Attribute, Count, ?DEFAULT_PROC_WINDOW_MS) + catch + _:_ -> + [] + end. + +get_allocator_info() -> + AllocData = recon_alloc:memory(allocated_types), + maps:from_list([{Type, Size} || {Type, Size} <- AllocData]). + +write_memory_report(FileHandle, MemoryReport, LastMemoryReport) -> + Timestamp = format_timestamp(MemoryReport#memory_report.timestamp), + BasicMemoryInfoStr = format_basic_memory_info(MemoryReport, LastMemoryReport), + TopMemoryProcsStr = format_process_list("Top Memory", MemoryReport#memory_report.top_memory_processes), + TopMailboxProcsStr = format_process_mailbox_list("Top Mailbox", MemoryReport#memory_report.top_mailbox_processes), + TopBinaryProcsStr = format_process_list("Top Binary", MemoryReport#memory_report.top_binary_processes), + GrowingMemoryProcsStr = format_process_list("Growing Memory", MemoryReport#memory_report.growing_memory_processes), + GrowingMailboxProcsStr = format_process_mailbox_list("Growing Mailbox", MemoryReport#memory_report.growing_mailbox_processes), + BinLeakProcsStr = format_bin_leak_list("Binary Leak", MemoryReport#memory_report.bin_leak_processes), + AllocatorStr = format_allocator_info(MemoryReport#memory_report.allocator_info), + + Report = io_lib:format( + "## ~s Memory Report~n~n" + "~s~n" + "~s~n~n" + "~s~n~n" + "~s~n~n" + "~s~n~n" + "~s~n~n" + "~s~n~n" + "~s~n~n", + [ + Timestamp, + BasicMemoryInfoStr, + TopMemoryProcsStr, + TopBinaryProcsStr, + TopMailboxProcsStr, + GrowingMemoryProcsStr, + GrowingMailboxProcsStr, + BinLeakProcsStr, + AllocatorStr + ]), + ok = file:write(FileHandle, Report), + ok = file:sync(FileHandle). + + +format_basic_memory_info(MemoryReport, LastMemoryReport) -> + TotalMemoryStr = format_memory_with_comparison("Total Memory", + MemoryReport#memory_report.total_memory, + case LastMemoryReport of undefined -> undefined; _ -> LastMemoryReport#memory_report.total_memory end), + ProcessesMemoryStr = format_memory_with_comparison("Processes Memory", + MemoryReport#memory_report.processes_memory, + case LastMemoryReport of undefined -> undefined; _ -> LastMemoryReport#memory_report.processes_memory end), + SystemMemoryStr = format_memory_with_comparison("System Memory", + MemoryReport#memory_report.system_memory, + case LastMemoryReport of undefined -> undefined; _ -> LastMemoryReport#memory_report.system_memory end), + AtomMemoryStr = format_memory_with_comparison("Atom Memory", + MemoryReport#memory_report.atom_memory, + case LastMemoryReport of undefined -> undefined; _ -> LastMemoryReport#memory_report.atom_memory end), + BinaryMemoryStr = format_memory_with_comparison("Binary Memory", + MemoryReport#memory_report.binary_memory, + case LastMemoryReport of undefined -> undefined; _ -> LastMemoryReport#memory_report.binary_memory end), + CodeMemoryStr = format_memory_with_comparison("Code Memory", + MemoryReport#memory_report.code_memory, + case LastMemoryReport of undefined -> undefined; _ -> LastMemoryReport#memory_report.code_memory end), + ETSMemoryStr = format_memory_with_comparison("ETS Memory", + MemoryReport#memory_report.ets_memory, + case LastMemoryReport of undefined -> undefined; _ -> LastMemoryReport#memory_report.ets_memory end), + ProcessCountStr = format_count_with_comparison("Process Count", + MemoryReport#memory_report.process_count, + case LastMemoryReport of undefined -> undefined; _ -> LastMemoryReport#memory_report.process_count end), + PortCountStr = format_count_with_comparison("Port Count", + MemoryReport#memory_report.port_count, + case LastMemoryReport of undefined -> undefined; _ -> LastMemoryReport#memory_report.port_count end), + + io_lib:format( + "### Basic Memory Info~n" + "~s~n~s~n~s~n~s~n~s~n~s~n~s~n~s~n~s~n", + [ + TotalMemoryStr, ProcessesMemoryStr, SystemMemoryStr, AtomMemoryStr, + BinaryMemoryStr, CodeMemoryStr, ETSMemoryStr, ProcessCountStr, PortCountStr + ]). + + +format_process_list(Title, []) -> + io_lib:format("### ~s Processes~nNo info~n", [Title]); +format_process_list(Title, ProcessList) -> + ProcessLines = lists:map( + fun({Pid, Value, Info}) -> + Name = case Info of + [RegName | _] when is_atom(RegName) -> atom_to_list(RegName); + _ -> pid_to_list(Pid) + end, + {M, F, A} = case proplists:get_value(initial_call, Info) of + {Module, Function, Arity} -> {Module, Function, Arity}; + undefined -> {unknown, unknown, 0} + end, + ProcessInfo = io_lib:format("~s (~s:~s/~w)", [Name, M, F, A]), + io_lib:format(" ~s: ~w bytes (~w MiB)~n", [ProcessInfo, Value, Value div (1024*1024)]) + end, + ProcessList + ), + io_lib:format("### ~s Processes~n~s", [Title, ProcessLines]). + +format_process_mailbox_list(Title, []) -> + io_lib:format("### ~s Processes~nNo info~n", [Title]); +format_process_mailbox_list(Title, ProcessList) -> + ProcessLines = lists:map( + fun({Pid, Value, Info}) -> + Name = case Info of + [RegName | _] when is_atom(RegName) -> atom_to_list(RegName); + _ -> pid_to_list(Pid) + end, + {M, F, A} = case proplists:get_value(initial_call, Info) of + {Module, Function, Arity} -> {Module, Function, Arity}; + undefined -> {unknown, unknown, 0} + end, + ProcessInfo = io_lib:format("~s (~s:~s/~w)", [Name, M, F, A]), + io_lib:format(" ~s: ~w messages~n", [ProcessInfo, Value]) + end, + ProcessList + ), + io_lib:format("### ~s Processes~n~s", [Title, ProcessLines]). + +format_bin_leak_list(_Title,[]) -> "### Binary Leak Processes\nNo info\n"; +format_bin_leak_list(Title,ProcessList)-> + ProcLines = lists:map(fun({Pid, Bytes, Info}) -> + AbsBytes = abs(Bytes), + Name= case Info of [Reg|_] when is_atom(Reg) -> atom_to_list(Reg); _-> pid_to_list(Pid) end, + {M,F,A}= case proplists:get_value(initial_call,Info) of {Mo,Fu,Ar}-> {Mo,Fu,Ar}; _->{unknown,unknown,0} end, + MiB= AbsBytes div (1024*1024), + io_lib:format(" ~s (~s:~s/~w): reclaimed ~w bytes (~w MiB)~n",[Name,M,F,A,AbsBytes,MiB]) + end,ProcessList), + io_lib:format("### ~s Processes~n~s",[Title,ProcLines]). + +format_allocator_info(AllocatorInfo) -> + case maps:size(AllocatorInfo) of + 0 -> + "### Allocator Info~nNo info~n"; + _ -> + AllocLines = maps:fold( + fun(Type, Size, Acc) -> + Line = io_lib:format(" ~s: ~w bytes (~w MB)~n", + [Type, Size, Size div (1024*1024)]), + [Line | Acc] + end, + [], + AllocatorInfo + ), + io_lib:format("### Allocator Info~n~s", [lists:reverse(AllocLines)]) + end. + +format_memory_with_comparison(Name, CurrentBytes, undefined) -> + CurrentMiB = CurrentBytes div (1024*1024), + io_lib:format("~s: ~w bytes (~w MiB)", [Name, CurrentBytes, CurrentMiB]); +format_memory_with_comparison(Name, CurrentBytes, LastBytes) -> + CurrentMiB = CurrentBytes div (1024*1024), + DiffBytes = CurrentBytes - LastBytes, + DiffMiB = DiffBytes div (1024*1024), + PercentChange = case LastBytes of + 0 -> case CurrentBytes of 0 -> 0.0; _ -> 100.0 end; + _ -> (DiffBytes / LastBytes) * 100.0 + end, + Sign = case DiffBytes >= 0 of true -> "+"; false -> "" end, + io_lib:format( + "~s: ~w bytes (~w MiB); since last: ~s~.1f% ~s~w bytes (~s~w MiB)", + [Name, CurrentBytes, CurrentMiB, Sign, PercentChange, Sign, DiffBytes, Sign, DiffMiB]). + +format_count_with_comparison(Name, CurrentCount, undefined) -> + io_lib:format("~s: ~w", [Name, CurrentCount]); +format_count_with_comparison(Name, CurrentCount, LastCount) -> + DiffCount = CurrentCount - LastCount, + PercentChange = case LastCount of + 0 -> case CurrentCount of 0 -> 0.0; _ -> 100.0 end; + _ -> (DiffCount / LastCount) * 100.0 + end, + Sign = case DiffCount >= 0 of true -> "+"; false -> "" end, + io_lib:format("~s: ~w; since last: ~s~.1f% ~s~w", [Name, CurrentCount, Sign, PercentChange, Sign, DiffCount]). + +format_timestamp(MillisecondsSinceEpoch) -> + {{Year, Month, Day}, {Hour, Minute, Second}} = + calendar:gregorian_seconds_to_datetime( + MillisecondsSinceEpoch div 1000 + + calendar:datetime_to_gregorian_seconds({{1970, 1, 1}, {0, 0, 0}}) + ), + Milliseconds = MillisecondsSinceEpoch rem 1000, + io_lib:format("~4..0w-~2..0w-~2..0wT~2..0w:~2..0w:~2..0w.~3..0wZ", + [Year, Month, Day, Hour, Minute, Second, Milliseconds]). diff --git a/apps/arweave/src/ar_sup.erl b/apps/arweave/src/ar_sup.erl index a2c3c6674..03f5f3059 100644 --- a/apps/arweave/src/ar_sup.erl +++ b/apps/arweave/src/ar_sup.erl @@ -107,4 +107,8 @@ init([]) -> true -> [?CHILD(ar_process_sampler, worker)]; false -> [] end, - {ok, {{one_for_one, 5, 10}, Children ++ DebugChildren}}. + OOMMonitorChildren = case ar_config:is_oom_monitor_enabled() of + true -> [?CHILD(ar_oom_monitor, worker)]; + false -> [] + end, + {ok, {{one_for_one, 5, 10}, Children ++ DebugChildren ++ OOMMonitorChildren}}.