File 0553-ct-Allow-post_end_per_testcase-hook-to-change-tc-sta.patch of Package erlang
From d35233878fd0b65c563374bd4383a122b51a48ec Mon Sep 17 00:00:00 2001
From: Siri Hansen <siri@erlang.org>
Date: Wed, 19 Dec 2018 14:29:22 +0100
Subject: [PATCH] [ct] Allow post_end_per_testcase hook to change tc status
after timetrap
---
lib/common_test/src/test_server.erl | 47 ++++++---
lib/common_test/test/ct_hooks_SUITE.erl | 106 ++++++++++++++++++++-
.../ct_cth_update_result_post_end_tc_SUITE.erl | 101 ++++++++++++++++++++
.../cth/tests/update_result_post_end_tc_cth.erl | 98 +++++++++++++++++++
4 files changed, 336 insertions(+), 16 deletions(-)
create mode 100644 lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_cth_update_result_post_end_tc_SUITE.erl
create mode 100644 lib/common_test/test/ct_hooks_SUITE_data/cth/tests/update_result_post_end_tc_cth.erl
diff --git a/lib/common_test/src/test_server.erl b/lib/common_test/src/test_server.erl
index e56106408f..ecb2ee7caf 100644
--- a/lib/common_test/src/test_server.erl
+++ b/lib/common_test/src/test_server.erl
@@ -849,17 +849,23 @@ spawn_fw_call(Mod,EPTC={end_per_testcase,Func},EndConf,Pid,
"WARNING: end_per_testcase failed!</font>",
{died,W}
end,
- try do_end_tc_call(Mod,EPTC,{Pid,Report,[EndConf]}, Why) of
- _ -> ok
- catch
- _:FwEndTCErr ->
- exit({fw_notify_done,end_tc,FwEndTCErr})
- end,
- FailLoc = proplists:get_value(tc_fail_loc, EndConf),
+ FailLoc0 = proplists:get_value(tc_fail_loc, EndConf),
+ {RetVal1,FailLoc} =
+ try do_end_tc_call(Mod,EPTC,{Pid,Report,[EndConf]}, Why) of
+ Why ->
+ {RetVal,FailLoc0};
+ {failed,_} = R ->
+ {R,[{Mod,Func}]};
+ R ->
+ {R,FailLoc0}
+ catch
+ _:FwEndTCErr ->
+ exit({fw_notify_done,end_tc,FwEndTCErr})
+ end,
%% finished, report back (if end_per_testcase fails, a warning
%% should be printed as part of the comment)
SendTo ! {self(),fw_notify_done,
- {Time,RetVal,FailLoc,[],Warn}}
+ {Time,RetVal1,FailLoc,[],Warn}}
end,
spawn_link(FwCall);
@@ -901,14 +907,25 @@ spawn_fw_call(Mod,Func,CurrConf,Pid,Error,Loc,SendTo) ->
FwErrorNotifyErr})
end,
Conf = [{tc_status,{failed,Error}}|CurrConf],
- try do_end_tc_call(Mod,EndTCFunc,{Pid,Error,[Conf]},Error) of
- _ -> ok
- catch
- _:FwEndTCErr ->
- exit({fw_notify_done,end_tc,FwEndTCErr})
- end,
+ {Time,RetVal,Loc1} =
+ try do_end_tc_call(Mod,EndTCFunc,{Pid,Error,[Conf]},Error) of
+ Error ->
+ {died, Error, Loc};
+ {failed,Reason} = NewReturn ->
+ fw_error_notify(Mod,Func1,Conf,Reason),
+ {died, NewReturn, [{Mod,Func}]};
+ NewReturn ->
+ T = case Error of
+ {timetrap_timeout,TT} -> TT;
+ _ -> 0
+ end,
+ {T, NewReturn, Loc}
+ catch
+ _:FwEndTCErr ->
+ exit({fw_notify_done,end_tc,FwEndTCErr})
+ end,
%% finished, report back
- SendTo ! {self(),fw_notify_done,{died,Error,Loc,[],undefined}}
+ SendTo ! {self(),fw_notify_done,{Time,RetVal,Loc1,[],undefined}}
end,
spawn_link(FwCall).
diff --git a/lib/common_test/test/ct_hooks_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE.erl
index 8ba14e63bc..7900fcd5bc 100644
--- a/lib/common_test/test/ct_hooks_SUITE.erl
+++ b/lib/common_test/test/ct_hooks_SUITE.erl
@@ -84,7 +84,7 @@ all(suite) ->
fail_post_suite_cth, skip_pre_suite_cth, skip_pre_end_cth,
skip_pre_init_tc_cth,
skip_post_suite_cth, recover_post_suite_cth, update_config_cth,
- state_update_cth, options_cth, same_id_cth,
+ state_update_cth, update_result_cth, options_cth, same_id_cth,
fail_n_skip_with_minimal_cth, prio_cth, no_config,
no_init_suite_config, no_init_config, no_end_config,
failed_sequence, repeat_force_stop, config_clash,
@@ -209,6 +209,10 @@ state_update_cth(Config) when is_list(Config) ->
do_test(state_update_cth, "ct_cth_fail_one_skip_one_SUITE.erl",
[state_update_cth,state_update_cth],Config).
+update_result_cth(Config) ->
+ do_test(update_result_cth, "ct_cth_update_result_post_end_tc_SUITE.erl",
+ [update_result_post_end_tc_cth],Config).
+
options_cth(Config) when is_list(Config) ->
do_test(options_cth, "ct_cth_empty_SUITE.erl",
[{empty_cth,[test]}],Config).
@@ -1094,6 +1098,106 @@ test_events(state_update_cth) ->
{?eh,stop_logging,[]}
];
+test_events(update_result_cth) ->
+ Suite = ct_cth_update_result_post_end_tc_SUITE,
+ [
+ {?eh,start_logging,'_'},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,cth,{'_',init,['_',[]]}},
+ {?eh,tc_start,{Suite,init_per_suite}},
+ {?eh,tc_done,{Suite,init_per_suite,ok}},
+
+ {?eh,tc_start,{Suite,tc_ok_to_fail}},
+ {?eh,cth,{'_',post_end_per_testcase,[Suite,tc_ok_to_fail,'_',ok,[]]}},
+ {?eh,tc_done,{Suite,tc_ok_to_fail,{failed,{error,"Test failure"}}}},
+ {?eh,cth,{'_',on_tc_fail,'_'}},
+ {?eh,test_stats,{0,1,{0,0}}},
+
+ {?eh,tc_start,{Suite,tc_ok_to_skip}},
+ {?eh,cth,{'_',post_end_per_testcase,[Suite,tc_ok_to_skip,'_',ok,[]]}},
+ {?eh,tc_done,{Suite,tc_ok_to_skip,{skipped,"Test skipped"}}},
+ {?eh,cth,{'_',on_tc_skip,'_'}},
+ {?eh,test_stats,{0,1,{1,0}}},
+
+ {?eh,tc_start,{Suite,tc_fail_to_ok}},
+ {?eh,cth,{'_',post_end_per_testcase,
+ [Suite,tc_fail_to_ok,'_',
+ {error,{test_case_failed,"should be changed to ok"}},[]]}},
+ {?eh,tc_done,{Suite,tc_fail_to_ok,ok}},
+ {?eh,test_stats,{1,1,{1,0}}},
+
+ {?eh,tc_start,{Suite,tc_fail_to_skip}},
+ {?eh,cth,{'_',post_end_per_testcase,
+ [Suite,tc_fail_to_skip,'_',
+ {error,{test_case_failed,"should be changed to skip"}},[]]}},
+ {?eh,tc_done,{Suite,tc_fail_to_skip,{skipped,"Test skipped"}}},
+ {?eh,cth,{'_',on_tc_skip,'_'}},
+ {?eh,test_stats,{1,1,{2,0}}},
+
+ {?eh,tc_start,{Suite,tc_timetrap_to_ok}},
+ {?eh,cth,{'_',post_end_per_testcase,
+ [Suite,tc_timetrap_to_ok,'_',{timetrap_timeout,3000},[]]}},
+ {?eh,tc_done,{Suite,tc_timetrap_to_ok,ok}},
+ {?eh,test_stats,{2,1,{2,0}}},
+
+ {?eh,tc_start,{Suite,tc_timetrap_to_skip}},
+ {?eh,cth,{'_',post_end_per_testcase,
+ [Suite,tc_timetrap_to_skip,'_',{timetrap_timeout,3000},[]]}},
+ {?eh,tc_done,{Suite,tc_timetrap_to_skip,{skipped,"Test skipped"}}},
+ {?eh,cth,{'_',on_tc_skip,'_'}},
+ {?eh,test_stats,{2,1,{3,0}}},
+
+ {?eh,tc_start,{Suite,tc_skip_to_fail}},
+ {?eh,cth,{'_',post_end_per_testcase,
+ [Suite,tc_skip_to_fail,'_',
+ {skip,"should be changed to fail"},[]]}},
+ {?eh,tc_done,{Suite,tc_skip_to_fail,{failed,{error,"Test failure"}}}},
+ {?eh,cth,{'_',on_tc_fail,'_'}},
+ {?eh,test_stats,{2,2,{3,0}}},
+
+ {?eh,tc_start,{Suite,end_fail_to_fail}},
+ {?eh,cth,{'_',post_end_per_testcase,
+ [Suite,end_fail_to_fail,'_',
+ {failed,
+ {Suite,end_per_testcase,
+ {'EXIT',{test_case_failed,"change result when end fails"}}}},[]]}},
+ {?eh,tc_done,{Suite,end_fail_to_fail,{failed,{error,"Test failure"}}}},
+ {?eh,cth,{'_',on_tc_fail,'_'}},
+ {?eh,test_stats,{2,3,{3,0}}},
+
+ {?eh,tc_start,{Suite,end_fail_to_skip}},
+ {?eh,cth,{'_',post_end_per_testcase,
+ [Suite,end_fail_to_skip,'_',
+ {failed,
+ {Suite,end_per_testcase,
+ {'EXIT',{test_case_failed,"change result when end fails"}}}},[]]}},
+ {?eh,tc_done,{Suite,end_fail_to_skip,{skipped,"Test skipped"}}},
+ {?eh,cth,{'_',on_tc_skip,'_'}},
+ {?eh,test_stats,{2,3,{4,0}}},
+
+ {?eh,tc_start,{Suite,end_timetrap_to_fail}},
+ {?eh,cth,{'_',post_end_per_testcase,
+ [Suite,end_timetrap_to_fail,'_',
+ {failed,{Suite,end_per_testcase,{timetrap_timeout,3000}}},[]]}},
+ {?eh,tc_done,{Suite,end_timetrap_to_fail,{failed,{error,"Test failure"}}}},
+ {?eh,cth,{'_',on_tc_fail,'_'}},
+ {?eh,test_stats,{2,4,{4,0}}},
+
+ {?eh,tc_start,{Suite,end_timetrap_to_skip}},
+ {?eh,cth,{'_',post_end_per_testcase,
+ [Suite,end_timetrap_to_skip,'_',
+ {failed,{Suite,end_per_testcase,{timetrap_timeout,3000}}},[]]}},
+ {?eh,tc_done,{Suite,end_timetrap_to_skip,{skipped,"Test skipped"}}},
+ {?eh,cth,{'_',on_tc_skip,'_'}},
+ {?eh,test_stats,{2,4,{5,0}}},
+
+ {?eh,tc_start,{Suite,end_per_suite}},
+ {?eh,tc_done,{Suite,end_per_suite,ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,cth,{'_',terminate,[[]]}},
+ {?eh,stop_logging,[]}
+ ];
+
test_events(options_cth) ->
[
{?eh,start_logging,{'DEF','RUNDIR'}},
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_cth_update_result_post_end_tc_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_cth_update_result_post_end_tc_SUITE.erl
new file mode 100644
index 0000000000..a16138ce6f
--- /dev/null
+++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_cth_update_result_post_end_tc_SUITE.erl
@@ -0,0 +1,101 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2017. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%% http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(ct_cth_update_result_post_end_tc_SUITE).
+
+-compile(export_all).
+
+-include("ct.hrl").
+
+suite() ->
+ [{timetrap,{seconds,3}}].
+
+init_per_suite(Config) ->
+ Config.
+
+end_per_suite(Config) ->
+ ok.
+
+init_per_group(_,Config) ->
+ Config.
+
+end_per_group(_,_) ->
+ ok.
+
+init_per_testcase(_,Config) ->
+ Config.
+
+end_per_testcase(EndTimetrap,_) when EndTimetrap==end_timetrap_to_fail;
+ EndTimetrap==end_timetrap_to_skip->
+ timer:sleep(10000);
+end_per_testcase(EndFail,_) when EndFail==end_fail_to_fail;
+ EndFail==end_fail_to_skip->
+ ct:fail("change result when end fails");
+end_per_testcase(_,_) ->
+ ok.
+
+all() ->
+ [tc_ok_to_fail,
+ tc_ok_to_skip,
+ tc_fail_to_ok,
+ tc_fail_to_skip,
+ tc_timetrap_to_ok,
+ tc_timetrap_to_skip,
+ tc_skip_to_fail,
+ end_fail_to_fail,
+ end_fail_to_skip,
+ end_timetrap_to_fail,
+ end_timetrap_to_skip].
+
+%% Test cases starts here.
+tc_ok_to_fail(_Config) ->
+ ok.
+
+tc_ok_to_skip(_Config) ->
+ ok.
+
+tc_fail_to_ok(_Config) ->
+ ct:fail("should be changed to ok").
+
+tc_fail_to_skip(_Config) ->
+ ct:fail("should be changed to skip").
+
+tc_timetrap_to_ok(_Config) ->
+ timer:sleep(10000), % will time out after 3 sek
+ ok.
+
+tc_timetrap_to_skip(_Config) ->
+ timer:sleep(10000), % will time out after 3 sek
+ ok.
+
+tc_skip_to_fail(_Config) ->
+ {skip,"should be changed to fail"}.
+
+end_fail_to_fail(_Config) ->
+ ok.
+
+end_fail_to_skip(_Config) ->
+ ok.
+
+end_timetrap_to_fail(_Config) ->
+ ok.
+
+end_timetrap_to_skip(_Config) ->
+ ok.
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/update_result_post_end_tc_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/update_result_post_end_tc_cth.erl
new file mode 100644
index 0000000000..7afb3d8781
--- /dev/null
+++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/update_result_post_end_tc_cth.erl
@@ -0,0 +1,98 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2017. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%% http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+
+-module(update_result_post_end_tc_cth).
+
+
+-include_lib("common_test/src/ct_util.hrl").
+-include_lib("common_test/include/ct_event.hrl").
+
+
+%% CT Hooks
+-compile(export_all).
+
+init(Id, Opts) ->
+ empty_cth:init(Id, Opts).
+
+pre_init_per_suite(Suite, Config, State) ->
+ empty_cth:pre_init_per_suite(Suite,Config,State).
+
+post_init_per_suite(Suite,Config,Return,State) ->
+ empty_cth:post_init_per_suite(Suite,Config,Return,State).
+
+pre_end_per_suite(Suite,Config,State) ->
+ empty_cth:pre_end_per_suite(Suite,Config,State).
+
+post_end_per_suite(Suite,Config,Return,State) ->
+ empty_cth:post_end_per_suite(Suite,Config,Return,State).
+
+pre_init_per_group(Suite,Group,Config,State) ->
+ empty_cth:pre_init_per_group(Suite,Group,Config,State).
+
+post_init_per_group(Suite,Group,Config,Return,State) ->
+ empty_cth:post_init_per_group(Suite,Group,Config,Return,State).
+
+pre_end_per_group(Suite,Group,Config,State) ->
+ empty_cth:pre_end_per_group(Suite,Group,Config,State).
+
+post_end_per_group(Suite,Group,Config,Return,State) ->
+ empty_cth:post_end_per_group(Suite,Group,Config,Return,State).
+
+pre_init_per_testcase(Suite,TC,Config,State) ->
+ empty_cth:pre_init_per_testcase(Suite,TC,Config,State).
+
+post_end_per_testcase(Suite,TC,Config,Return,State) ->
+ empty_cth:post_end_per_testcase(Suite,TC,Config,Return,State),
+ change_result(TC,Config,State).
+
+on_tc_fail(Suite,TC, Reason, State) ->
+ empty_cth:on_tc_fail(Suite,TC,Reason,State).
+
+on_tc_skip(Suite,TC, Reason, State) ->
+ empty_cth:on_tc_skip(Suite,TC,Reason,State).
+
+terminate(State) ->
+ empty_cth:terminate(State).
+
+%%%-----------------------------------------------------------------
+%%%
+change_result(tc_ok_to_fail,_Config,State) ->
+ {{fail, "Test failure"}, State};
+change_result(tc_ok_to_skip,_Config,State) ->
+ {{skip, "Test skipped"}, State};
+change_result(tc_fail_to_ok,Config,State) ->
+ {lists:keydelete(tc_status,1,Config),State};
+change_result(tc_fail_to_skip,Config,State) ->
+ {{skip,"Test skipped"},State};
+change_result(tc_timetrap_to_ok,Config,State) ->
+ {lists:keydelete(tc_status,1,Config),State};
+change_result(tc_timetrap_to_skip,Config,State) ->
+ {{skip,"Test skipped"},State};
+change_result(tc_skip_to_fail,_Config,State) ->
+ {{fail, "Test failure"}, State};
+change_result(end_fail_to_fail,_Config,State) ->
+ {{fail, "Test failure"}, State};
+change_result(end_fail_to_skip,_Config,State) ->
+ {{skip, "Test skipped"}, State};
+change_result(end_timetrap_to_fail,_Config,State) ->
+ {{fail, "Test failure"}, State};
+change_result(end_timetrap_to_skip,_Config,State) ->
+ {{skip, "Test skipped"}, State}.
--
2.16.4
