File 3956-dialyzer-Parallelize-remote-types-processing-in-cont.patch of Package erlang
From 12f0567f701a2ade95b528cc64e69713583e19ba Mon Sep 17 00:00:00 2001
From: Hans Bolinder <hasse@erlang.org>
Date: Sun, 1 Aug 2021 09:09:44 +0200
Subject: [PATCH 6/7] dialyzer: Parallelize remote types processing in
contracts
This has effect if many processes can run i parallel.
---
lib/dialyzer/src/dialyzer_codeserver.erl | 19 ++--
lib/dialyzer/src/dialyzer_contracts.erl | 108 ++++++++++++++++------
lib/dialyzer/src/dialyzer_coordinator.erl | 31 +++++--
lib/dialyzer/src/dialyzer_utils.erl | 2 +-
lib/dialyzer/src/dialyzer_worker.erl | 12 ++-
lib/dialyzer/src/erl_types.erl | 15 +--
6 files changed, 134 insertions(+), 53 deletions(-)
diff --git a/lib/dialyzer/src/dialyzer_codeserver.erl b/lib/dialyzer/src/dialyzer_codeserver.erl
index 0472e2c128..b25ea791d2 100644
--- a/lib/dialyzer/src/dialyzer_codeserver.erl
+++ b/lib/dialyzer/src/dialyzer_codeserver.erl
@@ -29,7 +29,7 @@
finalize_records/1,
get_contracts/1,
get_callbacks/1,
- get_exported_types/1,
+ get_exported_types_table/1,
extract_exported_types/1,
get_exports/1,
get_records_table/1,
@@ -132,12 +132,15 @@ new() ->
CodeOptions = [compressed, public, {read_concurrency, true}],
Code = ets:new(dialyzer_codeserver_code, CodeOptions),
ReadOptions = [compressed, {read_concurrency, true}],
- [Contracts, Callbacks, Records, ExportedTypes] =
+ [Records, ExportedTypes] =
[ets:new(Name, ReadOptions) ||
- Name <- [dialyzer_codeserver_contracts,
- dialyzer_codeserver_callbacks,
- dialyzer_codeserver_records,
+ Name <- [dialyzer_codeserver_records,
dialyzer_codeserver_exported_types]],
+ ReadWriteOptions = [public | ReadOptions],
+ [Contracts, Callbacks] =
+ [ets:new(Name, ReadWriteOptions) ||
+ Name <- [dialyzer_codeserver_contracts,
+ dialyzer_codeserver_callbacks]],
TempOptions = [public, {write_concurrency, true}],
[Exports, FunMetaInfo, TempExportedTypes, TempRecords, TempContracts,
TempCallbacks] =
@@ -211,10 +214,10 @@ insert_fun_meta_info(List, #codeserver{fun_meta_info = FunMetaInfo} = CS) ->
is_exported(MFA, #codeserver{exports = Exports}) ->
ets_set_is_element(MFA, Exports).
--spec get_exported_types(codeserver()) -> sets:set(mfa()).
+-spec get_exported_types_table(codeserver()) -> map_ets().
-get_exported_types(#codeserver{exported_types = ExpTypes}) ->
- ets_set_to_set(ExpTypes).
+get_exported_types_table(#codeserver{exported_types = ExpTypes}) ->
+ ExpTypes.
-spec extract_exported_types(codeserver()) -> {codeserver(), set_ets()}.
diff --git a/lib/dialyzer/src/dialyzer_contracts.erl b/lib/dialyzer/src/dialyzer_contracts.erl
index 5adcc78a0e..e434c64e4e 100644
--- a/lib/dialyzer/src/dialyzer_contracts.erl
+++ b/lib/dialyzer/src/dialyzer_contracts.erl
@@ -27,12 +27,23 @@
process_contract_remote_types/1,
store_tmp_contract/6]).
--export_type([file_contract/0, plt_contracts/0]).
+%% For dialyzer_worker.
+-export([process_contract_remote_types_module/2]).
+
+-export_type([file_contract/0, plt_contracts/0,
+ contract_remote_types_init_data/0,
+ contract_remote_types_result/0]).
%%-----------------------------------------------------------------------
-include("dialyzer.hrl").
+-type ext_types_message() :: {pid(), 'ext_types',
+ {mfa(), {file:filename(), erl_anno:location()}}}
+ | {'error', io_lib:chars()}.
+-type contract_remote_types_init_data() :: dialyzer_codeserver:codeserver().
+-type contract_remote_types_result() :: [ext_types_message()].
+
%%-----------------------------------------------------------------------
%% Types used in other parts of the system below
%%-----------------------------------------------------------------------
@@ -143,35 +154,76 @@ sequence([H|T], Delimiter) -> H ++ Delimiter ++ sequence(T, Delimiter).
dialyzer_codeserver:codeserver().
process_contract_remote_types(CodeServer) ->
- Mods = dialyzer_codeserver:all_temp_modules(CodeServer),
+ case dialyzer_codeserver:all_temp_modules(CodeServer) of
+ [] ->
+ CodeServer;
+ Mods ->
+ %% CodeServer is updated by each worker, but is still valid
+ %% after updates. Workers call
+ %% process_contract_remote_types_module/2 below.
+ Return =
+ dialyzer_coordinator:parallel_job(contract_remote_types,
+ Mods,
+ _InitData=CodeServer,
+ _Timing=none),
+ %% We need to pass on messages and thrown errors from erl_types:
+ _ = [self() ! {self(), ext_types, ExtType} ||
+ {_, ext_types, ExtType} <- Return],
+ case [Error || {error, _} = Error <- Return] of
+ [] ->
+ dialyzer_codeserver:finalize_contracts(CodeServer);
+ [Error | _] ->
+ throw(Error)
+ end
+ end.
+
+-spec process_contract_remote_types_module(module(),
+ dialyzer_codeserver:codeserver()) -> [ext_types_message()].
+
+process_contract_remote_types_module(ModuleName, CodeServer) ->
RecordTable = dialyzer_codeserver:get_records_table(CodeServer),
- ExpTypes = dialyzer_codeserver:get_exported_types(CodeServer),
- ModuleFun =
- fun(ModuleName) ->
- ContractFun =
- fun({MFA, {File, TmpContract, Xtra}}, C0) ->
- #tmp_contract{contract_funs = CFuns, forms = Forms} = TmpContract,
- {NewCs, C2} = lists:mapfoldl(fun(CFun, C1) ->
- CFun(ExpTypes, RecordTable, C1)
- end, C0, CFuns),
- Args = general_domain(NewCs),
- Contract = #contract{contracts = NewCs, args = Args, forms = Forms},
- {{MFA, {File, Contract, Xtra}}, C2}
- end,
- Cache = erl_types:cache__new(),
- {ContractMap, CallbackMap} =
- dialyzer_codeserver:get_temp_contracts(ModuleName, CodeServer),
- {NewContractList, Cache1} =
- lists:mapfoldl(ContractFun, Cache, maps:to_list(ContractMap)),
- {NewCallbackList, _NewCache} =
- lists:mapfoldl(ContractFun, Cache1, maps:to_list(CallbackMap)),
- dialyzer_codeserver:store_contracts(ModuleName,
- maps:from_list(NewContractList),
- maps:from_list(NewCallbackList),
- CodeServer)
+ ExpTypes = dialyzer_codeserver:get_exported_types_table(CodeServer),
+ ContractFun =
+ fun({MFA, {File, TmpContract, Xtra}}, C0) ->
+ #tmp_contract{contract_funs = CFuns, forms = Forms} = TmpContract,
+ {NewCs, C2} = lists:mapfoldl(fun(CFun, C1) ->
+ CFun(ExpTypes, RecordTable, C1)
+ end, C0, CFuns),
+ Args = general_domain(NewCs),
+ Contract = #contract{contracts = NewCs, args = Args, forms = Forms},
+ {{MFA, {File, Contract, Xtra}}, C2}
end,
- lists:foreach(ModuleFun, Mods),
- dialyzer_codeserver:finalize_contracts(CodeServer).
+ Cache = erl_types:cache__new(),
+ {ContractMap, CallbackMap} =
+ dialyzer_codeserver:get_temp_contracts(ModuleName, CodeServer),
+ try
+ {NewContractList, Cache1} =
+ lists:mapfoldl(ContractFun, Cache, maps:to_list(ContractMap)),
+ {NewCallbackList, _NewCache} =
+ lists:mapfoldl(ContractFun, Cache1, maps:to_list(CallbackMap)),
+ _NewCodeServer =
+ dialyzer_codeserver:store_contracts(ModuleName,
+ maps:from_list(NewContractList),
+ maps:from_list(NewCallbackList),
+ CodeServer),
+ rcv_ext_types()
+ catch
+ throw:{error, _}=Error ->
+ [Error] ++ rcv_ext_types()
+ end.
+
+rcv_ext_types() ->
+ Self = self(),
+ Self ! {Self, done},
+ rcv_ext_types(Self, []).
+
+rcv_ext_types(Self, ExtTypes) ->
+ receive
+ {Self, ext_types, _} = ExtType ->
+ rcv_ext_types(Self, [ExtType | ExtTypes]);
+ {Self, done} ->
+ lists:usort(ExtTypes)
+ end.
-type fun_types() :: orddict:orddict(label(), erl_types:erl_type()).
diff --git a/lib/dialyzer/src/dialyzer_coordinator.erl b/lib/dialyzer/src/dialyzer_coordinator.erl
index 7c1bc1de5a..f8a078364b 100644
--- a/lib/dialyzer/src/dialyzer_coordinator.erl
+++ b/lib/dialyzer/src/dialyzer_coordinator.erl
@@ -43,34 +43,44 @@
-type timing() :: dialyzer_timing:timing_server().
-type scc() :: [mfa_or_funlbl()].
--type mode() :: 'typesig' | 'dataflow' | 'compile' | 'warnings'.
+-type mode() :: 'typesig' | 'dataflow' | 'compile' | 'warnings' |
+ 'contract_remote_types'.
-type compile_job() :: file:filename().
-type typesig_job() :: scc().
-type dataflow_job() :: module().
-type warnings_job() :: module().
+-type contract_remote_types_job() :: module().
--type job() :: compile_job() | typesig_job() | dataflow_job() | warnings_job().
+-type job() :: compile_job() | typesig_job() | dataflow_job() |
+ warnings_job() | contract_remote_types_job().
-type compile_init_data() :: dialyzer_analysis_callgraph:compile_init_data().
-type typesig_init_data() :: dialyzer_succ_typings:typesig_init_data().
-type dataflow_init_data() :: dialyzer_succ_typings:dataflow_init_data().
-type warnings_init_data() :: dialyzer_succ_typings:warnings_init_data().
+-type contract_remote_types_init_data() ::
+ dialyzer_contracts:contract_remote_types_init_data().
-type compile_result() :: dialyzer_analysis_callgraph:compile_result().
-type typesig_result() :: [mfa_or_funlbl()].
-type dataflow_result() :: [mfa_or_funlbl()].
-type warnings_result() :: [dial_warning()].
+-type contract_remote_types_result() ::
+ dialyzer_contracts:contract_remote_types_result().
-type init_data() :: compile_init_data() | typesig_init_data() |
- dataflow_init_data() | warnings_init_data().
+ dataflow_init_data() | warnings_init_data() |
+ contract_remote_types_init_data().
-type result() :: compile_result() | typesig_result() |
- dataflow_result() | warnings_result().
+ dataflow_result() | warnings_result() |
+ contract_remote_types_result().
-type job_result() :: dialyzer_analysis_callgraph:one_file_mid_error() |
dialyzer_analysis_callgraph:one_file_result_ok() |
- typesig_result() | dataflow_result() | warnings_result().
+ typesig_result() | dataflow_result() |
+ warnings_result() | contract_remote_types_result().
-record(state, {mode :: mode(),
active = 0 :: integer(),
@@ -94,7 +104,10 @@
('dataflow', [dataflow_job()], dataflow_init_data(),
timing()) -> dataflow_result();
('warnings', [warnings_job()], warnings_init_data(),
- timing()) -> warnings_result().
+ timing()) -> warnings_result();
+ ('contract_remote_types', [contract_remote_types_job()],
+ contract_remote_types_init_data(), timing()) ->
+ contract_remote_types_result().
parallel_job(Mode, Jobs, InitData, Timing) ->
State = spawn_jobs(Mode, Jobs, InitData, Timing),
@@ -158,7 +171,9 @@ collect_result(#state{mode = Mode, active = Active, result = Result,
ets:delete(SCCtoPID),
NewResult;
'warnings' ->
- NewResult
+ NewResult;
+ 'contract_remote_types' ->
+ NewResult
end;
N ->
case TypesigOrDataflow of
@@ -187,6 +202,8 @@ update_result(Mode, InitData, Job, Data, Result) ->
X when X =:= 'typesig'; X =:= 'dataflow' ->
dialyzer_succ_typings:lookup_names(Data, InitData) ++ Result;
'warnings' ->
+ Data ++ Result;
+ 'contract_remote_types' ->
Data ++ Result
end.
diff --git a/lib/dialyzer/src/dialyzer_utils.erl b/lib/dialyzer/src/dialyzer_utils.erl
index c4ac96c9dc..e3a3976366 100644
--- a/lib/dialyzer/src/dialyzer_utils.erl
+++ b/lib/dialyzer/src/dialyzer_utils.erl
@@ -233,7 +233,7 @@ get_record_fields([], _RecDict, Acc) ->
%% The field types are cached. Used during analysis when handling records.
process_record_remote_types(CServer) ->
- ExpTypes = dialyzer_codeserver:get_exported_types(CServer),
+ ExpTypes = dialyzer_codeserver:get_exported_types_table(CServer),
Mods = dialyzer_codeserver:all_temp_modules(CServer),
process_opaque_types0(Mods, CServer, ExpTypes),
VarTable = erl_types:var_table__new(),
diff --git a/lib/dialyzer/src/dialyzer_worker.erl b/lib/dialyzer/src/dialyzer_worker.erl
index af0f2e9e08..64de596da7 100644
--- a/lib/dialyzer/src/dialyzer_worker.erl
+++ b/lib/dialyzer/src/dialyzer_worker.erl
@@ -66,7 +66,8 @@ init(#state{job = SCC, mode = Mode, init_data = InitData,
DependsOn = [{Pid, erlang:monitor(process, Pid)} || Pid <- Pids],
loop(updating, State#state{depends_on = DependsOn});
init(#state{mode = Mode} = State) when
- Mode =:= 'compile'; Mode =:= 'warnings' ->
+ Mode =:= 'compile'; Mode =:= 'warnings';
+ Mode =:= 'contract_remote_types' ->
loop(running, State).
loop(updating, #state{mode = Mode} = State) when
@@ -83,6 +84,11 @@ loop(waiting, #state{mode = Mode} = State) when
?debug("~w: Wait: ~p\n", [self(), State#state.job]),
NewState = wait_for_success_typings(State),
loop(updating, NewState);
+loop(running, #state{mode = 'contract_remote_types'} = State) ->
+ request_activation(State),
+ ?debug("~w: Remote types: ~p\n", [self(), State#state.job]),
+ Result = do_work(State),
+ report_to_coordinator(Result, State);
loop(running, #state{mode = 'compile'} = State) ->
request_activation(State),
?debug("Compile: ~s\n",[State#state.job]),
@@ -127,7 +133,9 @@ request_activation(#state{coordinator = Coordinator}) ->
do_work(#state{mode = Mode, job = Job, init_data = InitData}) ->
case Mode of
typesig -> dialyzer_succ_typings:find_succ_types_for_scc(Job, InitData);
- dataflow -> dialyzer_succ_typings:refine_one_module(Job, InitData)
+ dataflow -> dialyzer_succ_typings:refine_one_module(Job, InitData);
+ contract_remote_types ->
+ dialyzer_contracts:process_contract_remote_types_module(Job, InitData)
end.
report_to_coordinator(Result, #state{job = Job, coordinator = Coordinator}) ->
diff --git a/lib/hipe/cerl/erl_types.erl b/lib/hipe/cerl/erl_types.erl
index e381bfb64e..e218d2b051 100644
--- a/lib/hipe/cerl/erl_types.erl
+++ b/lib/hipe/cerl/erl_types.erl
@@ -4663,6 +4663,7 @@ mod_name(Mod, Name) ->
[erl_type()], type_names()}.
-type mod_type_table() :: ets:tid().
-type mod_records() :: dict:dict(module(), type_table()).
+-type exported_type_table() :: ets:tid().
-record(cache,
{
types = maps:new() :: #{cache_key() => {erl_type(), expand_limit()}},
@@ -4671,7 +4672,7 @@ mod_name(Mod, Name) ->
-opaque cache() :: #cache{}.
--spec t_from_form(parse_form(), sets:set(mfa()), site(), mod_type_table(),
+-spec t_from_form(parse_form(), exported_type_table(), site(), mod_type_table(),
var_table(), cache()) -> {erl_type(), cache()}.
t_from_form(Form, ExpTypes, Site, RecDict, VarTab, Cache) ->
@@ -4696,12 +4697,12 @@ t_from_form_without_remote(Form, Site, TypeTable) ->
-type expand_depth() :: integer().
-record(from_form, {site :: site() | {'check', mta()},
- xtypes :: sets:set(mfa()) | 'replace_by_none',
+ xtypes :: exported_type_table() | 'replace_by_none',
mrecs :: 'undefined' | mod_type_table(),
vtab :: var_table(),
tnames :: type_names()}).
--spec t_from_form_check_remote(parse_form(), sets:set(mfa()), mta(),
+-spec t_from_form_check_remote(parse_form(), exported_type_table(), mta(),
mod_type_table()) -> 'ok'.
t_from_form_check_remote(Form, ExpTypes, MTA, RecDict) ->
State = #from_form{site = {check, MTA},
@@ -4722,7 +4723,7 @@ t_from_form_check_remote(Form, ExpTypes, Site, RecDict) ->
%% types balanced (unions will otherwise collapse to any()) by limiting
%% the depth the same way as t_limit/2 does.
--spec t_from_form1(parse_form(), sets:set(mfa()) | 'replace_by_none',
+-spec t_from_form1(parse_form(), exported_type_table() | 'replace_by_none',
site(), 'undefined' | mod_type_table(), var_table(),
cache()) -> {erl_type(), cache()}.
@@ -5074,7 +5075,7 @@ remote_from_form(Anno, RemMod, Name, Args, S, D, L, C) ->
self() ! {self(), ext_types, MFA},
{t_any(), L, C};
{RemDict, C1} ->
- case sets:is_element(MFA, ET) of
+ case ets:member(ET, MFA) of
true ->
RemType = {type, MFA},
case can_unfold_more(RemType, TypeNames) of
@@ -5345,7 +5346,7 @@ recur_limit(Fun, D, L, TypeName, TypeNames) ->
Fun(D, L)
end.
--spec t_check_record_fields(parse_form(), sets:set(mfa()), site(),
+-spec t_check_record_fields(parse_form(), exported_type_table(), site(),
mod_type_table(), var_table(), cache()) -> cache().
t_check_record_fields(Form, ExpTypes, Site, RecDict, VarTable, Cache) ->
@@ -5576,7 +5577,7 @@ t_form_to_string({type, _Anno, Name, []} = T) ->
V = var_table__new(),
C = cache__new(),
State = #from_form{site = Site,
- xtypes = sets:new(),
+ xtypes = replace_by_none,
mrecs = 'undefined',
vtab = V,
tnames = []},
--
2.31.1
