File 4411-Add-dist_util_spawn_options-in-kernel.patch of Package erlang
From e44b432f3e0331b6be0625f9f713329edd559773 Mon Sep 17 00:00:00 2001
From: Zeyu Zhang <zeyu@fb.com>
Date: Tue, 20 Jul 2021 16:25:01 -0700
Subject: [PATCH] Add dist_util_spawn_options in kernel
This change allows user to specify spawn options when dist protocol
starts dist_util:handshake_other_started or handshake_we_started
processes.
These processes will be alive until the connections are down. With
thousands of ssl dist connections it spends quite amount of memory,
and it would be great if we can change garbage collection, as well
as other spawn options for dist_util processes.
---
lib/kernel/doc/src/kernel_app.xml | 9 +++++
lib/kernel/src/inet_tcp_dist.erl | 12 +++++-
lib/kernel/test/erl_distribution_SUITE.erl | 45 ++++++++++++++++++++++
lib/ssl/src/inet_tls_dist.erl | 11 +++++-
lib/ssl/test/ssl_dist_SUITE.erl | 45 ++++++++++++++++++++++
5 files changed, 118 insertions(+), 4 deletions(-)
diff --git a/lib/kernel/doc/src/kernel_app.xml b/lib/kernel/doc/src/kernel_app.xml
index c17b327f32..5a9539e6ef 100644
--- a/lib/kernel/doc/src/kernel_app.xml
+++ b/lib/kernel/doc/src/kernel_app.xml
@@ -293,6 +293,15 @@
other distributed Erlang nodes.
See <seemfa marker="gen_tcp#connect/4"><c>gen_tcp:connect/4</c></seemfa>.</p>
</item>
+ <tag><c>{net_ticker_spawn_options, Opts}</c></tag>
+ <item>
+ <marker id="net_ticker_spawn_options"></marker>
+ <p>Defines a list of extra spawn options for dist_util:con_loop processes.
+ When there is a large number of dist connections, setting up garbage collection
+ options can be helpful to reduce memory usage. Default is [link, {priority, max}],
+ and these two options cannot be changed.
+ See <seemfa marker="erts:erlang#spawn_opt/2"><c>erlang:spawn_opt/2</c></seemfa>.</p>
+ </item>
<tag><c>inet_parse_error_log = silent</c></tag>
<item>
<p>If set, no log events are issued when erroneous lines are
diff --git a/lib/kernel/src/inet_tcp_dist.erl b/lib/kernel/src/inet_tcp_dist.erl
index ef0f06aef7..1a12d758ff 100644
--- a/lib/kernel/src/inet_tcp_dist.erl
+++ b/lib/kernel/src/inet_tcp_dist.erl
@@ -208,7 +208,7 @@ accept_connection(AcceptPid, Socket, MyNode, Allowed, SetupTime) ->
gen_accept_connection(Driver, AcceptPid, Socket, MyNode, Allowed, SetupTime) ->
spawn_opt(?MODULE, do_accept,
[Driver, self(), AcceptPid, Socket, MyNode, Allowed, SetupTime],
- [link, {priority, max}]).
+ net_ticker_spawn_options()).
do_accept(Driver, Kernel, AcceptPid, Socket, MyNode, Allowed, SetupTime) ->
receive
@@ -275,6 +275,14 @@ nodelay() ->
end.
+% we may want different spawn options for dist_util processes
+
+net_ticker_spawn_options() ->
+ Opts = application:get_env(kernel, net_ticker_spawn_options, []),
+ Opts1 = [{priority, max} | proplists:delete(priority, Opts)],
+ [link | proplists:delete(link, Opts1)].
+
+
%% ------------------------------------------------------------
%% Get remote information about a Socket.
%% ------------------------------------------------------------
@@ -304,7 +312,7 @@ setup(Node, Type, MyNode, LongOrShortNames,SetupTime) ->
gen_setup(Driver, Node, Type, MyNode, LongOrShortNames, SetupTime) ->
spawn_opt(?MODULE, do_setup,
[Driver, self(), Node, Type, MyNode, LongOrShortNames, SetupTime],
- [link, {priority, max}]).
+ net_ticker_spawn_options()).
do_setup(Driver, Kernel, Node, Type, MyNode, LongOrShortNames, SetupTime) ->
?trace("~p~n",[{inet_tcp_dist,self(),setup,Node}]),
diff --git a/lib/kernel/test/erl_distribution_SUITE.erl b/lib/kernel/test/erl_distribution_SUITE.erl
index 4f2d4d393d..b5def00b44 100644
--- a/lib/kernel/test/erl_distribution_SUITE.erl
+++ b/lib/kernel/test/erl_distribution_SUITE.erl
@@ -34,6 +34,7 @@
setopts/1,
table_waste/1, net_setuptime/1,
inet_dist_options_options/1,
+ net_ticker_spawn_options/1,
monitor_nodes_nodedown_reason/1,
monitor_nodes_complex_nodedown_reason/1,
@@ -52,6 +53,7 @@
%% Performs the test at another node.
-export([get_socket_priorities/0,
+ get_net_ticker_fullsweep_option/1,
tick_cli_test/3, tick_cli_test1/3,
tick_serv_test/2, tick_serv_test1/1,
run_remote_test/1,
@@ -92,6 +94,7 @@ all() ->
epmd_reconnect,
hidden_node, setopts,
table_waste, net_setuptime, inet_dist_options_options,
+ net_ticker_spawn_options,
{group, monitor_nodes},
erl_uds_dist_smoke_test,
erl_1424, net_kernel_start,
@@ -1060,6 +1063,48 @@ get_socket_priorities() ->
+%% check net_ticker_spawn_options
+net_ticker_spawn_options(Config) when is_list(Config) ->
+ FullsweepString0 = "[{fullsweep_after,0}]",
+ FullsweepString =
+ case os:cmd("echo [{a,1}]") of
+ "[{a,1}]"++_ ->
+ FullsweepString0;
+ _ ->
+ %% Some shells need quoting of [{}]
+ "'"++FullsweepString0++"'"
+ end,
+ InetDistOptions =
+ "-hidden "
+ "-kernel net_ticker_spawn_options "++FullsweepString,
+ {ok,Node1} =
+ start_node("", net_ticker_spawn_options_1, InetDistOptions),
+ {ok,Node2} =
+ start_node("", net_ticker_spawn_options_2, InetDistOptions),
+ %%
+ pong =
+ rpc:call(Node1, net_adm, ping, [Node2]),
+ FullsweepOptionNode1 =
+ rpc:call(Node1, ?MODULE, get_net_ticker_fullsweep_option, [Node2]),
+ FullsweepOptionNode2 =
+ rpc:call(Node2, ?MODULE, get_net_ticker_fullsweep_option, [Node1]),
+ io:format("FullsweepOptionNode1 = ~p", [FullsweepOptionNode1]),
+ io:format("FullsweepOptionNode2 = ~p", [FullsweepOptionNode2]),
+ 0 = FullsweepOptionNode1,
+ 0 = FullsweepOptionNode2,
+ %%
+ stop_node(Node2),
+ stop_node(Node1),
+ ok.
+
+get_net_ticker_fullsweep_option(Node) ->
+ Port = proplists:get_value(Node, erlang:system_info(dist_ctrl)),
+ {links, [DistUtilPid, _NetKernelPid]} = erlang:port_info(Port, links),
+ {garbage_collection, GCOpts} = erlang:process_info(DistUtilPid, garbage_collection),
+ proplists:get_value(fullsweep_after, GCOpts).
+
+
+
%%
%% Testcase:
%% monitor_nodes_nodedown_reason
diff --git a/lib/ssl/src/inet_tls_dist.erl b/lib/ssl/src/inet_tls_dist.erl
index 5cfbeff387..d413d7738b 100644
--- a/lib/ssl/src/inet_tls_dist.erl
+++ b/lib/ssl/src/inet_tls_dist.erl
@@ -448,7 +448,7 @@ gen_accept_connection(
Driver, AcceptPid, DistCtrl,
MyNode, Allowed, SetupTime, Kernel)
end,
- [link, {priority, max}])).
+ net_ticker_spawn_options())).
do_accept(
_Driver, AcceptPid, DistCtrl, MyNode, Allowed, SetupTime, Kernel) ->
@@ -540,7 +540,7 @@ gen_setup(Driver, Node, Type, MyNode, LongOrShortNames, SetupTime) ->
Kernel = self(),
monitor_pid(
spawn_opt(setup_fun(Driver, Kernel, Node, Type, MyNode, LongOrShortNames, SetupTime),
- [link, {priority, max}])).
+ net_ticker_spawn_options())).
-spec setup_fun(_,_,_,_,_,_,_) -> fun(() -> no_return()).
setup_fun(Driver, Kernel, Node, Type, MyNode, LongOrShortNames, SetupTime) ->
@@ -808,6 +808,13 @@ nodelay() ->
end.
+% we may want different spawn options for dist_util processes
+
+net_ticker_spawn_options() ->
+ Opts = application:get_env(kernel, net_ticker_spawn_options, []),
+ Opts1 = [{priority, max} | proplists:delete(priority, Opts)],
+ [link | proplists:delete(link, Opts1)].
+
get_ssl_options(Type) ->
try ets:lookup(ssl_dist_opts, Type) of
[{Type, Opts0}] ->
diff --git a/lib/ssl/test/ssl_dist_SUITE.erl b/lib/ssl/test/ssl_dist_SUITE.erl
index 67be91196f..24698b7f56 100644
--- a/lib/ssl/test/ssl_dist_SUITE.erl
+++ b/lib/ssl/test/ssl_dist_SUITE.erl
@@ -53,6 +53,8 @@
listen_options/1,
connect_options/0,
connect_options/1,
+ net_ticker_spawn_options/0,
+ net_ticker_spawn_options/1,
use_interface/0,
use_interface/1,
verify_fun_fail/0,
@@ -70,6 +72,7 @@
listen_options_test/3,
do_connect_options/2,
connect_options_test/3,
+ net_ticker_spawn_options_test/3,
verify_fun_fail_test/3,
verify_fun_pass_test/3,
verify_pass_always/3,
@@ -100,6 +103,7 @@ all() ->
listen_port_options,
listen_options,
connect_options,
+ net_ticker_spawn_options,
use_interface,
verify_fun_fail,
verify_fun_pass,
@@ -322,6 +326,22 @@ connect_options() ->
connect_options(Config) when is_list(Config) ->
try_setting_priority(fun do_connect_options/2, Config).
+%%--------------------------------------------------------------------
+net_ticker_spawn_options() ->
+ [{doc, "Test net_ticker_spawn_options"}].
+net_ticker_spawn_options(Config) when is_list(Config) ->
+ FullsweepString0 = "[{fullsweep_after,0}]",
+ FullsweepString =
+ case os:cmd("echo [{a,1}]") of
+ "[{a,1}]"++_ ->
+ FullsweepString0;
+ _ ->
+ %% Some shells need quoting of [{}]
+ "'"++FullsweepString0++"'"
+ end,
+ Options = "-kernel net_ticker_spawn_options "++FullsweepString,
+ gen_dist_test(net_ticker_spawn_options_test, [{tls_only_basic_opts, Options} | Config]).
+
%%--------------------------------------------------------------------
use_interface() ->
@@ -594,6 +614,23 @@ connect_options_test(NH1, NH2, Config) ->
%% Node 2 will not, since it only applies to outbound connections.
[] = Elevated2.
+net_ticker_spawn_options_test(NH1, NH2, _Config) ->
+ Node1 = NH1#node_handle.nodename,
+ Node2 = NH2#node_handle.nodename,
+
+ pong = apply_on_ssl_node(NH1, fun () -> net_adm:ping(Node2) end),
+
+ FullsweepOptionNode1 =
+ apply_on_ssl_node(NH1, fun () -> get_dist_util_fullsweep_option(Node2) end),
+ FullsweepOptionNode2 =
+ apply_on_ssl_node(NH2, fun () -> get_dist_util_fullsweep_option(Node1) end),
+
+ ct:pal("FullsweepOptionNode1: ~p~n", [FullsweepOptionNode1]),
+ ct:pal("FullsweepOptionNode2: ~p~n", [FullsweepOptionNode2]),
+
+ 0 = FullsweepOptionNode1,
+ 0 = FullsweepOptionNode2.
+
verify_fun_fail_test(NH1, NH2, _) ->
Node2 = NH2#node_handle.nodename,
@@ -635,6 +672,14 @@ get_socket_priorities() ->
{ok,[{priority,Priority}]} <-
[inet:getopts(Port, [priority]) || Port <- inet_ports()]].
+get_dist_util_fullsweep_option(Node) ->
+ SenderPid = proplists:get_value(Node, erlang:system_info(dist_ctrl)),
+ {links, Links1} = erlang:process_info(SenderPid, links),
+ {links, Links2} = erlang:process_info(whereis(net_kernel), links),
+ [DistUtilPid] = [X || X <- Links1, Y <- Links2, X =:= Y],
+ {garbage_collection, GCOpts} = erlang:process_info(DistUtilPid, garbage_collection),
+ proplists:get_value(fullsweep_after, GCOpts).
+
inet_ports() ->
[Port || Port <- erlang:ports(),
element(2, erlang:port_info(Port, name)) =:= "tcp_inet"].
--
2.31.1
