File 4156-dialyzer-Parallelize-remote-types-processi... of Package erlang

Overview Repositories Revisions Requests Users Attributes Meta

File 4156-dialyzer-Parallelize-remote-types-processing-in-cont.patch of Package erlang (Revision 5bfcaba6253c6d17475b8ba23b952738)

Currently displaying revision 5bfcaba6253c6d17475b8ba23b952738 , Show latest

From 12f0567f701a2ade95b528cc64e69713583e19ba Mon Sep 17 00:00:00 2001
From: Hans Bolinder <hasse@erlang.org>
Date: Sun, 1 Aug 2021 09:09:44 +0200
Subject: [PATCH 6/7] dialyzer: Parallelize remote types processing in
 contracts

This has effect if many processes can run i parallel.
---
 lib/dialyzer/src/dialyzer_codeserver.erl  |  19 ++--
 lib/dialyzer/src/dialyzer_contracts.erl   | 108 ++++++++++++++++------
 lib/dialyzer/src/dialyzer_coordinator.erl |  31 +++++--
 lib/dialyzer/src/dialyzer_utils.erl       |   2 +-
 lib/dialyzer/src/dialyzer_worker.erl      |  12 ++-
 lib/dialyzer/src/erl_types.erl            |  15 +--
 6 files changed, 134 insertions(+), 53 deletions(-)

diff --git a/lib/dialyzer/src/dialyzer_codeserver.erl b/lib/dialyzer/src/dialyzer_codeserver.erl
index 0472e2c128..b25ea791d2 100644
--- a/lib/dialyzer/src/dialyzer_codeserver.erl
+++ b/lib/dialyzer/src/dialyzer_codeserver.erl
@@ -29,7 +29,7 @@
 	 finalize_records/1,
 	 get_contracts/1,
 	 get_callbacks/1,
-         get_exported_types/1,
+         get_exported_types_table/1,
          extract_exported_types/1,
 	 get_exports/1,
 	 get_records_table/1,
@@ -132,12 +132,15 @@ new() ->
   CodeOptions = [compressed, public, {read_concurrency, true}],
   Code = ets:new(dialyzer_codeserver_code, CodeOptions),
   ReadOptions = [compressed, {read_concurrency, true}],
-  [Contracts, Callbacks, Records, ExportedTypes] =
+  [Records, ExportedTypes] =
     [ets:new(Name, ReadOptions) ||
-      Name <- [dialyzer_codeserver_contracts,
-               dialyzer_codeserver_callbacks,
-               dialyzer_codeserver_records,
+      Name <- [dialyzer_codeserver_records,
                dialyzer_codeserver_exported_types]],
+  ReadWriteOptions = [public | ReadOptions],
+  [Contracts, Callbacks] =
+    [ets:new(Name, ReadWriteOptions) ||
+      Name <- [dialyzer_codeserver_contracts,
+               dialyzer_codeserver_callbacks]],
   TempOptions = [public, {write_concurrency, true}],
   [Exports, FunMetaInfo, TempExportedTypes, TempRecords, TempContracts,
    TempCallbacks] =
@@ -211,10 +214,10 @@ insert_fun_meta_info(List, #codeserver{fun_meta_info = FunMetaInfo} = CS) ->
 is_exported(MFA, #codeserver{exports = Exports}) ->
   ets_set_is_element(MFA, Exports).
 
--spec get_exported_types(codeserver()) -> sets:set(mfa()).
+-spec get_exported_types_table(codeserver()) -> map_ets().
 
-get_exported_types(#codeserver{exported_types = ExpTypes}) ->
-  ets_set_to_set(ExpTypes).
+get_exported_types_table(#codeserver{exported_types = ExpTypes}) ->
+  ExpTypes.
 
 -spec extract_exported_types(codeserver()) -> {codeserver(), set_ets()}.
 
diff --git a/lib/dialyzer/src/dialyzer_contracts.erl b/lib/dialyzer/src/dialyzer_contracts.erl
index 5adcc78a0e..e434c64e4e 100644
--- a/lib/dialyzer/src/dialyzer_contracts.erl
+++ b/lib/dialyzer/src/dialyzer_contracts.erl
@@ -27,12 +27,23 @@
 	 process_contract_remote_types/1,
 	 store_tmp_contract/6]).
 
--export_type([file_contract/0, plt_contracts/0]).
+%% For dialyzer_worker.
+-export([process_contract_remote_types_module/2]).
+
+-export_type([file_contract/0, plt_contracts/0,
+              contract_remote_types_init_data/0,
+              contract_remote_types_result/0]).
 
 %%-----------------------------------------------------------------------
 
 -include("dialyzer.hrl").
 
+-type ext_types_message() :: {pid(), 'ext_types',
+                              {mfa(), {file:filename(), erl_anno:location()}}}
+                           | {'error', io_lib:chars()}.
+-type contract_remote_types_init_data() :: dialyzer_codeserver:codeserver().
+-type contract_remote_types_result() :: [ext_types_message()].
+
 %%-----------------------------------------------------------------------
 %% Types used in other parts of the system below
 %%-----------------------------------------------------------------------
@@ -143,35 +154,76 @@ sequence([H|T], Delimiter) -> H ++ Delimiter ++ sequence(T, Delimiter).
                                        dialyzer_codeserver:codeserver().
 
 process_contract_remote_types(CodeServer) ->
-  Mods = dialyzer_codeserver:all_temp_modules(CodeServer),
+  case dialyzer_codeserver:all_temp_modules(CodeServer) of
+    [] ->
+      CodeServer;
+    Mods ->
+      %% CodeServer is updated by each worker, but is still valid
+      %% after updates. Workers call
+      %% process_contract_remote_types_module/2 below.
+      Return =
+        dialyzer_coordinator:parallel_job(contract_remote_types,
+                                          Mods,
+                                          _InitData=CodeServer,
+                                          _Timing=none),
+      %% We need to pass on messages and thrown errors from erl_types:
+      _ = [self() ! {self(), ext_types, ExtType} ||
+            {_, ext_types, ExtType} <- Return],
+      case [Error || {error, _} = Error <- Return] of
+        [] ->
+          dialyzer_codeserver:finalize_contracts(CodeServer);
+        [Error | _] ->
+          throw(Error)
+      end
+  end.
+
+-spec process_contract_remote_types_module(module(),
+                                           dialyzer_codeserver:codeserver()) -> [ext_types_message()].
+
+process_contract_remote_types_module(ModuleName, CodeServer) ->
   RecordTable = dialyzer_codeserver:get_records_table(CodeServer),
-  ExpTypes = dialyzer_codeserver:get_exported_types(CodeServer),
-  ModuleFun =
-    fun(ModuleName) ->
-        ContractFun =
-          fun({MFA, {File, TmpContract, Xtra}}, C0) ->
-              #tmp_contract{contract_funs = CFuns, forms = Forms} = TmpContract,
-              {NewCs, C2} = lists:mapfoldl(fun(CFun, C1) ->
-                                               CFun(ExpTypes, RecordTable, C1)
-                                           end, C0, CFuns),
-              Args = general_domain(NewCs),
-              Contract = #contract{contracts = NewCs, args = Args, forms = Forms},
-              {{MFA, {File, Contract, Xtra}}, C2}
-          end,
-        Cache = erl_types:cache__new(),
-        {ContractMap, CallbackMap} =
-          dialyzer_codeserver:get_temp_contracts(ModuleName, CodeServer),
-        {NewContractList, Cache1} =
-          lists:mapfoldl(ContractFun, Cache, maps:to_list(ContractMap)),
-        {NewCallbackList, _NewCache} =
-          lists:mapfoldl(ContractFun, Cache1, maps:to_list(CallbackMap)),
-        dialyzer_codeserver:store_contracts(ModuleName,
-                                            maps:from_list(NewContractList),
-                                            maps:from_list(NewCallbackList),
-                                            CodeServer)
+  ExpTypes = dialyzer_codeserver:get_exported_types_table(CodeServer),
+  ContractFun =
+    fun({MFA, {File, TmpContract, Xtra}}, C0) ->
+        #tmp_contract{contract_funs = CFuns, forms = Forms} = TmpContract,
+        {NewCs, C2} = lists:mapfoldl(fun(CFun, C1) ->
+                                         CFun(ExpTypes, RecordTable, C1)
+                                     end, C0, CFuns),
+        Args = general_domain(NewCs),
+        Contract = #contract{contracts = NewCs, args = Args, forms = Forms},
+        {{MFA, {File, Contract, Xtra}}, C2}
     end,
-  lists:foreach(ModuleFun, Mods),
-  dialyzer_codeserver:finalize_contracts(CodeServer).
+  Cache = erl_types:cache__new(),
+  {ContractMap, CallbackMap} =
+    dialyzer_codeserver:get_temp_contracts(ModuleName, CodeServer),
+  try
+    {NewContractList, Cache1} =
+      lists:mapfoldl(ContractFun, Cache, maps:to_list(ContractMap)),
+    {NewCallbackList, _NewCache} =
+      lists:mapfoldl(ContractFun, Cache1, maps:to_list(CallbackMap)),
+    _NewCodeServer =
+      dialyzer_codeserver:store_contracts(ModuleName,
+                                          maps:from_list(NewContractList),
+                                          maps:from_list(NewCallbackList),
+                                          CodeServer),
+    rcv_ext_types()
+  catch
+    throw:{error, _}=Error ->
+      [Error] ++ rcv_ext_types()
+  end.
+
+rcv_ext_types() ->
+  Self = self(),
+  Self ! {Self, done},
+  rcv_ext_types(Self, []).
+
+rcv_ext_types(Self, ExtTypes) ->
+  receive
+    {Self, ext_types, _} = ExtType ->
+      rcv_ext_types(Self, [ExtType | ExtTypes]);
+    {Self, done} ->
+      lists:usort(ExtTypes)
+  end.
 
 -type fun_types() :: orddict:orddict(label(), erl_types:erl_type()).
 
diff --git a/lib/dialyzer/src/dialyzer_coordinator.erl b/lib/dialyzer/src/dialyzer_coordinator.erl
index 7c1bc1de5a..f8a078364b 100644
--- a/lib/dialyzer/src/dialyzer_coordinator.erl
+++ b/lib/dialyzer/src/dialyzer_coordinator.erl
@@ -43,34 +43,44 @@
 -type timing() :: dialyzer_timing:timing_server().
 
 -type scc()     :: [mfa_or_funlbl()].
--type mode()    :: 'typesig' | 'dataflow' | 'compile' | 'warnings'.
+-type mode()    :: 'typesig' | 'dataflow' | 'compile' | 'warnings' |
+                   'contract_remote_types'.
 
 -type compile_job()  :: file:filename().
 -type typesig_job()  :: scc().
 -type dataflow_job() :: module().
 -type warnings_job() :: module().
+-type contract_remote_types_job() :: module().
 
--type job() :: compile_job() | typesig_job() | dataflow_job() | warnings_job().
+-type job() :: compile_job() | typesig_job() | dataflow_job() |
+               warnings_job() | contract_remote_types_job().
 
 -type compile_init_data()  :: dialyzer_analysis_callgraph:compile_init_data().
 -type typesig_init_data()  :: dialyzer_succ_typings:typesig_init_data().
 -type dataflow_init_data() :: dialyzer_succ_typings:dataflow_init_data().
 -type warnings_init_data() :: dialyzer_succ_typings:warnings_init_data().
+-type contract_remote_types_init_data() ::
+                      dialyzer_contracts:contract_remote_types_init_data().
 
 -type compile_result()  :: dialyzer_analysis_callgraph:compile_result().
 -type typesig_result()  :: [mfa_or_funlbl()].
 -type dataflow_result() :: [mfa_or_funlbl()].
 -type warnings_result() :: [dial_warning()].
+-type contract_remote_types_result() ::
+        dialyzer_contracts:contract_remote_types_result().
 
 -type init_data() :: compile_init_data() | typesig_init_data() |
-		     dataflow_init_data() | warnings_init_data().
+		     dataflow_init_data() | warnings_init_data() |
+                     contract_remote_types_init_data().
 
 -type result() :: compile_result() | typesig_result() |
-		  dataflow_result() | warnings_result().
+		  dataflow_result() | warnings_result() |
+                  contract_remote_types_result().
 
 -type job_result() :: dialyzer_analysis_callgraph:one_file_mid_error() |
                       dialyzer_analysis_callgraph:one_file_result_ok() |
-                      typesig_result() | dataflow_result() | warnings_result().
+                      typesig_result() | dataflow_result() |
+                      warnings_result() | contract_remote_types_result().
 
 -record(state, {mode           :: mode(),
 		active     = 0 :: integer(),
@@ -94,7 +104,10 @@
 		  ('dataflow', [dataflow_job()], dataflow_init_data(),
 		   timing()) -> dataflow_result();
 		  ('warnings', [warnings_job()], warnings_init_data(),
-		   timing()) -> warnings_result().
+		   timing()) -> warnings_result();
+                  ('contract_remote_types', [contract_remote_types_job()],
+                   contract_remote_types_init_data(), timing()) ->
+                      contract_remote_types_result().
 
 parallel_job(Mode, Jobs, InitData, Timing) ->
   State = spawn_jobs(Mode, Jobs, InitData, Timing),
@@ -158,7 +171,9 @@ collect_result(#state{mode = Mode, active = Active, result = Result,
 	      ets:delete(SCCtoPID),
 	      NewResult;
 	    'warnings' ->
-	      NewResult
+	      NewResult;
+            'contract_remote_types' ->
+              NewResult
 	  end;
 	N ->
           case TypesigOrDataflow of
@@ -187,6 +202,8 @@ update_result(Mode, InitData, Job, Data, Result) ->
     X when X =:= 'typesig'; X =:= 'dataflow' ->
       dialyzer_succ_typings:lookup_names(Data, InitData) ++ Result;
     'warnings' ->
+      Data ++ Result;
+    'contract_remote_types' ->
       Data ++ Result
   end.
 
diff --git a/lib/dialyzer/src/dialyzer_utils.erl b/lib/dialyzer/src/dialyzer_utils.erl
index c4ac96c9dc..e3a3976366 100644
--- a/lib/dialyzer/src/dialyzer_utils.erl
+++ b/lib/dialyzer/src/dialyzer_utils.erl
@@ -233,7 +233,7 @@ get_record_fields([], _RecDict, Acc) ->
 
 %% The field types are cached. Used during analysis when handling records.
 process_record_remote_types(CServer) ->
-  ExpTypes = dialyzer_codeserver:get_exported_types(CServer),
+  ExpTypes = dialyzer_codeserver:get_exported_types_table(CServer),
   Mods = dialyzer_codeserver:all_temp_modules(CServer),
   process_opaque_types0(Mods, CServer, ExpTypes),
   VarTable = erl_types:var_table__new(),
diff --git a/lib/dialyzer/src/dialyzer_worker.erl b/lib/dialyzer/src/dialyzer_worker.erl
index af0f2e9e08..64de596da7 100644
--- a/lib/dialyzer/src/dialyzer_worker.erl
+++ b/lib/dialyzer/src/dialyzer_worker.erl
@@ -66,7 +66,8 @@ init(#state{job = SCC, mode = Mode, init_data = InitData,
   DependsOn = [{Pid, erlang:monitor(process, Pid)} || Pid <- Pids],
   loop(updating, State#state{depends_on = DependsOn});
 init(#state{mode = Mode} = State) when
-    Mode =:= 'compile'; Mode =:= 'warnings' ->
+    Mode =:= 'compile'; Mode =:= 'warnings';
+    Mode =:= 'contract_remote_types' ->
   loop(running, State).
 
 loop(updating, #state{mode = Mode} = State) when
@@ -83,6 +84,11 @@ loop(waiting, #state{mode = Mode} = State) when
   ?debug("~w: Wait: ~p\n", [self(), State#state.job]),
   NewState = wait_for_success_typings(State),
   loop(updating, NewState);
+loop(running, #state{mode = 'contract_remote_types'} = State) ->
+  request_activation(State),
+  ?debug("~w: Remote types: ~p\n", [self(), State#state.job]),
+  Result = do_work(State),
+  report_to_coordinator(Result, State);
 loop(running, #state{mode = 'compile'} = State) ->
   request_activation(State),
   ?debug("Compile: ~s\n",[State#state.job]),
@@ -127,7 +133,9 @@ request_activation(#state{coordinator = Coordinator}) ->
 do_work(#state{mode = Mode, job = Job, init_data = InitData}) ->
   case Mode of
     typesig -> dialyzer_succ_typings:find_succ_types_for_scc(Job, InitData);
-    dataflow -> dialyzer_succ_typings:refine_one_module(Job, InitData)
+    dataflow -> dialyzer_succ_typings:refine_one_module(Job, InitData);
+    contract_remote_types ->
+      dialyzer_contracts:process_contract_remote_types_module(Job, InitData)
   end.
 
 report_to_coordinator(Result, #state{job = Job, coordinator = Coordinator}) ->
diff --git a/lib/hipe/cerl/erl_types.erl b/lib/hipe/cerl/erl_types.erl
index e381bfb64e..e218d2b051 100644
--- a/lib/hipe/cerl/erl_types.erl
+++ b/lib/hipe/cerl/erl_types.erl
@@ -4663,6 +4663,7 @@ mod_name(Mod, Name) ->
                       [erl_type()], type_names()}.
 -type mod_type_table() :: ets:tid().
 -type mod_records() :: dict:dict(module(), type_table()).
+-type exported_type_table() :: ets:tid().
 -record(cache,
         {
           types = maps:new() :: #{cache_key() => {erl_type(), expand_limit()}},
@@ -4671,7 +4672,7 @@ mod_name(Mod, Name) ->
 
 -opaque cache() :: #cache{}.
 
--spec t_from_form(parse_form(), sets:set(mfa()), site(), mod_type_table(),
+-spec t_from_form(parse_form(), exported_type_table(), site(), mod_type_table(),
                   var_table(), cache()) -> {erl_type(), cache()}.
 
 t_from_form(Form, ExpTypes, Site, RecDict, VarTab, Cache) ->
@@ -4696,12 +4697,12 @@ t_from_form_without_remote(Form, Site, TypeTable) ->
 -type expand_depth() :: integer().
 
 -record(from_form, {site   :: site() | {'check', mta()},
-                    xtypes :: sets:set(mfa()) | 'replace_by_none',
+                    xtypes :: exported_type_table() | 'replace_by_none',
                     mrecs  :: 'undefined' | mod_type_table(),
                     vtab   :: var_table(),
                     tnames :: type_names()}).
 
--spec t_from_form_check_remote(parse_form(), sets:set(mfa()), mta(),
+-spec t_from_form_check_remote(parse_form(), exported_type_table(), mta(),
                                mod_type_table()) -> 'ok'.
 t_from_form_check_remote(Form, ExpTypes, MTA, RecDict) ->
   State = #from_form{site   = {check, MTA},
@@ -4722,7 +4723,7 @@ t_from_form_check_remote(Form, ExpTypes, Site, RecDict) ->
 %% types balanced (unions will otherwise collapse to any()) by limiting
 %% the depth the same way as t_limit/2 does.
 
--spec t_from_form1(parse_form(), sets:set(mfa()) | 'replace_by_none',
+-spec t_from_form1(parse_form(), exported_type_table() | 'replace_by_none',
                    site(), 'undefined' | mod_type_table(), var_table(),
                    cache()) -> {erl_type(), cache()}.
 
@@ -5074,7 +5075,7 @@ remote_from_form(Anno, RemMod, Name, Args, S, D, L, C) ->
           self() ! {self(), ext_types, MFA},
           {t_any(), L, C};
         {RemDict, C1} ->
-          case sets:is_element(MFA, ET) of
+          case ets:member(ET, MFA) of
             true ->
               RemType = {type, MFA},
               case can_unfold_more(RemType, TypeNames) of
@@ -5345,7 +5346,7 @@ recur_limit(Fun, D, L, TypeName, TypeNames) ->
       Fun(D, L)
   end.
 
--spec t_check_record_fields(parse_form(), sets:set(mfa()), site(),
+-spec t_check_record_fields(parse_form(), exported_type_table(), site(),
                             mod_type_table(), var_table(), cache()) -> cache().
 
 t_check_record_fields(Form, ExpTypes, Site, RecDict, VarTable, Cache) ->
@@ -5576,7 +5577,7 @@ t_form_to_string({type, _Anno, Name, []} = T) ->
      V = var_table__new(),
      C = cache__new(),
      State = #from_form{site   = Site,
-                        xtypes = sets:new(),
+                        xtypes = replace_by_none,
                         mrecs  = 'undefined',
                         vtab   = V,
                         tnames = []},
-- 
2.31.1

Places

File 4156-dialyzer-Parallelize-remote-types-processing-in-cont.patch of Package erlang (Revision 5bfcaba6253c6d17475b8ba23b952738)

Places