File 2203-Reduce-memory-use-of-system_information-to... of Package erlang

Overview Repositories Revisions Requests Users Attributes Meta

File 2203-Reduce-memory-use-of-system_information-to_file-1.patch of Package erlang (Revision 9cfc7361ef11aa2c9e03afdf25519fb7)

Currently displaying revision 9cfc7361ef11aa2c9e03afdf25519fb7 , Show latest

From d468fcbb1dfeef6c68f6033df9bff57dfcd748af Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?John=20H=C3=B6gberg?= <john@erlang.org>
Date: Wed, 10 Jan 2018 13:34:05 +0100
Subject: [PATCH] Reduce memory use of system_information:to_file/1

The previous implementation generated a term, converted it to plain
text with io_lib:format/2, and then converted that to a binary
before writing it to disk.

We now emit the term as we go, which should make it a bit safer to
extract this information under load.
---
 lib/runtime_tools/src/system_information.erl | 232 ++++++++++++++++++---------
 1 file changed, 158 insertions(+), 74 deletions(-)

diff --git a/lib/runtime_tools/src/system_information.erl b/lib/runtime_tools/src/system_information.erl
index df25297eb9..6a9f3ff58a 100644
--- a/lib/runtime_tools/src/system_information.erl
+++ b/lib/runtime_tools/src/system_information.erl
@@ -75,43 +75,37 @@ load_report(file, File)   -> load_report(data, from_file(File));
 load_report(data, Report) ->
     ok = start_internal(), gen_server:call(?SERVER, {load_report, Report}, infinity).
 
-report() -> [
-	{init_arguments,    init:get_arguments()},
-	{code_paths,        code:get_path()},
-	{code,              code()},
-	{system_info,       erlang_system_info()},
-	{erts_compile_info, erlang:system_info(compile_info)},
-	{beam_dynamic_libraries, get_dynamic_libraries()},
-	{environment_erts,  os_getenv_erts_specific()},
-	{environment,       [split_env(Env) || Env <- os:getenv()]},
-	{sanity_check,      sanity_check()}	     
-    ].
+report() ->
+    %% This is ugly but beats having to maintain two distinct implementations,
+    %% and we don't really care about memory use since it's internal and
+    %% undocumented.
+    {ok, Fd} = file:open([], [ram, read, write]),
+    to_fd(Fd),
+    {ok, _} = file:position(Fd, bof),
+    from_fd(Fd).
 
 -spec to_file(FileName) -> ok | {error, Reason} when
       FileName :: file:name_all(),
       Reason :: file:posix() | badarg | terminated | system_limit.
 
 to_file(File) ->
-    file:write_file(File, iolist_to_binary([
-		io_lib:format("{system_information_version, ~p}.~n", [
-			?REPORT_FILE_VSN
-		    ]),
-		io_lib:format("{system_information, ~p}.~n", [
-			report()
-		    ])
-	    ])).
+    case file:open(File, [raw, write, binary, delayed_write]) of
+        {ok, Fd} ->
+            try
+                to_fd(Fd)
+            after
+                file:close(Fd)
+            end;
+        {error, Reason} ->
+            {error, Reason}
+    end.
 
 from_file(File) ->
-    case file:consult(File) of
-	{ok, Data} ->
-	    case get_value([system_information_version], Data) of
-		?REPORT_FILE_VSN ->
-		    get_value([system_information], Data);
-		Vsn ->
-		    erlang:error({unknown_version, Vsn})
-	    end;
-	_ ->
-	    erlang:error(bad_report_file)
+    {ok, Fd} = file:open(File, [raw, read]),
+    try
+        from_fd(Fd)
+    after
+        file:close(Fd)
     end.
 
 applications() -> applications([]).
@@ -457,61 +451,151 @@ split_env([$=|Vs], Key) -> {lists:reverse(Key), Vs};
 split_env([I|Vs], Key)  -> split_env(Vs, [I|Key]);
 split_env([], KV)       -> lists:reverse(KV). % should not happen.
 
-%% get applications
+from_fd(Fd) ->
+    try
+        [{system_information_version, "1.0"},
+         {system_information, Data}] = consult_fd(Fd),
+        Data
+    catch
+        _:_ -> erlang:error(bad_report_file)
+    end.
 
-code() ->
-    % order is important
-    get_code_from_paths(code:get_path()).
+consult_fd(Fd) ->
+    consult_fd_1(Fd, [], {ok, []}).
+consult_fd_1(Fd, Cont0, ReadResult) ->
+    Data =
+        case ReadResult of
+            {ok, Characters} -> Characters;
+            eof -> eof
+        end,
+    case erl_scan:tokens(Cont0, Data, 1) of
+        {done, {ok, Tokens, _}, Next} ->
+            {ok, Term} = erl_parse:parse_term(Tokens),
+            [Term | consult_fd_1(Fd, [], {ok, Next})];
+        {more, Cont} ->
+            consult_fd_1(Fd, Cont, file:read(Fd, 1 bsl 20));
+        {done, {eof, _}, eof} -> []
+    end.
 
-get_code_from_paths([]) -> [];
-get_code_from_paths([Path|Paths]) ->
-    case is_application_path(Path) of
-	true -> 
-	    [{application, get_application_from_path(Path)}|get_code_from_paths(Paths)];
-	false ->
-	    [{code, [
-			{path,    Path},
-			{modules, get_modules_from_path(Path)}
-		    ]}|get_code_from_paths(Paths)]
+%%
+%% Dumps a system_information tuple to the given Fd, writing the term in chunks
+%% to avoid eating too much memory on large systems.
+%%
+
+to_fd(Fd) ->
+    EmitChunk =
+        fun(Format, Args) ->
+            ok = file:write(Fd, io_lib:format(Format, Args))
+        end,
+
+    EmitChunk("{system_information_version, ~p}.~n"
+              "{system_information,["
+                  "{init_arguments,~p},"
+                  "{code_paths,~p},",
+        [?REPORT_FILE_VSN,
+         init:get_arguments(),
+         code:get_path()]),
+
+    emit_code_info(EmitChunk),
+
+    EmitChunk(    ","  %% Note the leading comma!
+                  "{system_info,~p},"
+                  "{erts_compile_info,~p},"
+                  "{beam_dynamic_libraries,~p},"
+                  "{environment_erts,~p},"
+                  "{environment,~p},"
+                  "{sanity_check,~p}"
+              "]}.~n",
+        [erlang_system_info(),
+         erlang:system_info(compile_info),
+         get_dynamic_libraries(),
+         os_getenv_erts_specific(),
+         [split_env(Env) || Env <- os:getenv()],
+         sanity_check()]).
+
+%% Emits all modules/applications in the *code path order*
+emit_code_info(EmitChunk) ->
+    EmitChunk("{code, [", []),
+    comma_separated_foreach(EmitChunk,
+        fun(Path) ->
+            case is_application_path(Path) of
+                true -> emit_application_info(EmitChunk, Path);
+                false -> emit_code_path_info(EmitChunk, Path)
+            end
+        end, code:get_path()),
+    EmitChunk("]}", []).
+
+emit_application_info(EmitChunk, Path) ->
+    [Appfile|_] = filelib:wildcard(filename:join(Path, "*.app")),
+    case file:consult(Appfile) of
+        {ok, [{application, App, Info}]} ->
+            RtDeps = proplists:get_value(runtime_dependencies, Info, []),
+            Description = proplists:get_value(description, Info, []),
+            Version = proplists:get_value(vsn, Info, []),
+
+            EmitChunk("{application, {~p,["
+                          "{description,~p},"
+                          "{vsn,~p},"
+                          "{path,~p},"
+                          "{runtime_dependencies,~p},",
+                [App, Description, Version, Path, RtDeps]),
+            emit_module_info_from_path(EmitChunk, Path),
+            EmitChunk("]}}", [])
     end.
 
+emit_code_path_info(EmitChunk, Path) ->
+    EmitChunk("{code, ["
+                  "{path, ~p},", [Path]),
+    emit_module_info_from_path(EmitChunk, Path),
+    EmitChunk("]}", []).
+
+emit_module_info_from_path(EmitChunk, Path) ->
+    BeamFiles = filelib:wildcard(filename:join(Path, "*.beam")),
+
+    EmitChunk("{modules, [", []),
+    comma_separated_foreach(EmitChunk,
+        fun(Beam) ->
+            emit_module_info(EmitChunk, Beam)
+        end, BeamFiles),
+    EmitChunk("]}", []).
+
+emit_module_info(EmitChunk, Beam) ->
+    %% FIXME: The next three calls load *all* significant chunks onto the heap,
+    %% which may cause us to run out of memory if there's a huge module in the
+    %% code path.
+    {ok,{Mod, Md5}} = beam_lib:md5(Beam),
+
+    CompilerVersion = get_compiler_version(Beam),
+    Native = beam_is_native_compiled(Beam),
+
+    Loaded = case code:is_loaded(Mod) of
+        false -> false;
+        _     -> true
+    end,
+
+    EmitChunk("{~p,["
+                  "{loaded,~p},"
+                  "{native,~p},"
+                  "{compiler,~p},"
+                  "{md5,~p}"
+              "]}",
+        [Mod, Loaded, Native, CompilerVersion, hexstring(Md5)]).
+
+comma_separated_foreach(_EmitChunk, _Fun, []) ->
+    ok;
+comma_separated_foreach(_EmitChunk, Fun, [H]) ->
+    Fun(H);
+comma_separated_foreach(EmitChunk, Fun, [H | T]) ->
+    Fun(H),
+    EmitChunk(",", []),
+    comma_separated_foreach(EmitChunk, Fun, T).
+
 is_application_path(Path) ->
     case filelib:wildcard(filename:join(Path, "*.app")) of
 	[] -> false;
 	_  -> true
     end.
 
-get_application_from_path(Path) ->
-    [Appfile|_] = filelib:wildcard(filename:join(Path, "*.app")),
-    case file:consult(Appfile) of
-	{ok, [{application, App, Info}]} ->
-	    {App, [
-		    {description, proplists:get_value(description, Info, [])},
-		    {vsn,         proplists:get_value(vsn, Info, [])},
-		    {path,        Path},
-		    {runtime_dependencies,
-		     proplists:get_value(runtime_dependencies, Info, [])},
-		    {modules,     get_modules_from_path(Path)}
-		]}
-    end.
-
-get_modules_from_path(Path) ->
-    [ 
-	begin
-		{ok,{Mod, Md5}} = beam_lib:md5(Beam),
-		Loaded = case code:is_loaded(Mod) of
-		    false -> false;
-		    _     -> true
-		end,
-		{Mod, [
-			{loaded,   Loaded},
-			{native,   beam_is_native_compiled(Beam)},
-			{compiler, get_compiler_version(Beam)},
-			{md5,      hexstring(Md5)}
-		    ]}
-	end || Beam <- filelib:wildcard(filename:join(Path, "*.beam"))
-    ].
-
 hexstring(Bin) when is_binary(Bin) ->
     lists:flatten([io_lib:format("~2.16.0b", [V]) || <<V>> <= Bin]).
 
-- 
2.15.1

Places

File 2203-Reduce-memory-use-of-system_information-to_file-1.patch of Package erlang (Revision 9cfc7361ef11aa2c9e03afdf25519fb7)

Places