File 1303-ct-extend-ct_hooks_SUITE.patch of Package erlang

Overview Repositories Revisions Requests Users Attributes Meta

File 1303-ct-extend-ct_hooks_SUITE.patch of Package erlang (Revision 2587bae7bf0dfb77beed2069b73c8f2a)

Currently displaying revision 2587bae7bf0dfb77beed2069b73c8f2a , Show latest

From a1846e936acdcdb5b4b4cfa9adacb75b1ac249fe Mon Sep 17 00:00:00 2001
From: Jakub Witczak <kuba@erlang.org>
Date: Wed, 14 Jun 2023 15:16:34 +0200
Subject: [PATCH 3/4] ct: extend ct_hooks_SUITE

---
 lib/common_test/test/ct_hooks_SUITE.erl       | 289 ++++++++++++------
 .../cth/tests/ct_update_config_SUITE.erl      |  36 ++-
 .../cth/tests/ct_update_config_SUITE2.erl     |  62 ++++
 lib/common_test/test/ct_test_support.erl      |   4 +-
 4 files changed, 285 insertions(+), 106 deletions(-)
 create mode 100644 lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_update_config_SUITE2.erl

diff --git a/lib/common_test/test/ct_hooks_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE.erl
index 79960224eb..63ff69eef8 100644
--- a/lib/common_test/test/ct_hooks_SUITE.erl
+++ b/lib/common_test/test/ct_hooks_SUITE.erl
@@ -95,7 +95,7 @@ all(suite) ->
        fail_pre_suite_cth, double_fail_pre_suite_cth,
        fail_post_suite_cth, skip_pre_suite_cth, skip_pre_end_cth,
        skip_pre_init_tc_cth, fail_post_init_tc_cth,
-       skip_post_suite_cth, recover_post_suite_cth, update_config_cth,
+       skip_post_suite_cth, recover_post_suite_cth, update_config_cth, update_config_cth2,
        state_update_cth, update_result_cth, options_cth, same_id_cth,
        fail_n_skip_with_minimal_cth, prio_cth, no_config,
        no_init_suite_config, no_init_config, no_end_config,
@@ -226,6 +226,10 @@ recover_post_suite_cth(Config) when is_list(Config) ->
 update_config_cth(Config) when is_list(Config) ->
     do_test(update_config_cth, "ct_update_config_SUITE.erl",
 	    [update_config_cth],Config).
+%% no init/end_per_testcase functions in suite
+update_config_cth2(Config) when is_list(Config) ->
+    do_test(update_config_cth2, "ct_update_config_SUITE2.erl",
+	    [update_config_cth],Config).
 
 state_update_cth(Config) when is_list(Config) ->
     do_test(state_update_cth, "ct_cth_fail_one_skip_one_SUITE.erl",
@@ -1193,12 +1197,13 @@ test_events(recover_post_suite_cth) ->
     ];
 
 test_events(update_config_cth) ->
+    Suite = ct_update_config_SUITE,
     TestCaseEvents =
         fun(Case, Result) ->
                 [
-                 {?eh,tc_start,{ct_update_config_SUITE,Case}},
+                 {?eh,tc_start,{Suite,Case}},
                  {?eh,cth,{'_',pre_init_per_testcase,
-                           [ct_update_config_SUITE,
+                           [Suite,
                             Case,contains(
                                         [post_init_per_group,
                                          init_per_group,
@@ -1208,7 +1213,7 @@ test_events(update_config_cth) ->
                                          pre_init_per_suite]),
                             []]}},
                  {?eh,cth,{'_',post_init_per_testcase,
-                           [ct_update_config_SUITE,
+                           [Suite,
                             Case,contains(
                                         [init_per_testcase,
                                          pre_init_per_testcase,
@@ -1220,7 +1225,7 @@ test_events(update_config_cth) ->
                                          pre_init_per_suite]),
                             ok,[]]}},
                  {?eh,cth,{'_',pre_end_per_testcase,
-                           [ct_update_config_SUITE,
+                           [Suite,
                             Case,contains(
                                         [post_init_per_testcase,
                                          init_per_testcase,
@@ -1233,7 +1238,7 @@ test_events(update_config_cth) ->
                                          pre_init_per_suite]),
                             []]}},
                  {?eh,cth,{'_',post_end_per_testcase,
-                           [ct_update_config_SUITE,
+                           [Suite,
                             Case,contains(
                                         [pre_end_per_testcase,
                                          post_init_per_testcase,
@@ -1246,101 +1251,80 @@ test_events(update_config_cth) ->
                                          init_per_suite,
                                          pre_init_per_suite]),
                             Result,[]]}},
-                 {?eh,tc_done,{ct_update_config_SUITE,Case,ok}}
+                 {?eh,tc_done,{Suite,Case,ok}}
                 ]
         end,
-    [
-     {?eh,start_logging,{'DEF','RUNDIR'}},
-     {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
-     {?eh,cth,{'_',init,['_',[]]}},
-
-     {?eh,tc_start,{ct_update_config_SUITE,init_per_suite}},
-     {?eh,cth,{'_',pre_init_per_suite,
-	       [ct_update_config_SUITE,contains([]),[]]}},
-     {?eh,cth,{'_',post_init_per_suite,
-	       [ct_update_config_SUITE,
-		'$proplist',
-		contains(
-                  [init_per_suite,
-                   pre_init_per_suite]),
-		[]]}},
-     {?eh,tc_done,{ct_update_config_SUITE,init_per_suite,ok}},
-
-     {?eh,tc_start,{ct_update_config_SUITE, {init_per_group,group1,[]}}},
-     {?eh,cth,{'_',pre_init_per_group,
-	       [ct_update_config_SUITE,
-                group1,contains(
-			 [post_init_per_suite,
-			  init_per_suite,
-			  pre_init_per_suite]),
-		[]]}},
-     {?eh,cth,{'_',post_init_per_group,
-	       [ct_update_config_SUITE,
-                group1,
-		contains(
-		  [post_init_per_suite,
-		   init_per_suite,
-		   pre_init_per_suite]),
-		contains(
-		  [init_per_group,
-		   pre_init_per_group,
-		   post_init_per_suite,
-		   init_per_suite,
-		   pre_init_per_suite]),
-                []]}},
-     {?eh,tc_done,{ct_update_config_SUITE,{init_per_group,group1,[]},ok}}] ++
-        TestCaseEvents(test_case, ok) ++
-        TestCaseEvents(test_case_timetrap, {timetrap_timeout,1000}) ++
-        [{?eh,tc_start,{ct_update_config_SUITE, {end_per_group,group1,[]}}},
-         {?eh,cth,{'_',pre_end_per_group,
-                   [ct_update_config_SUITE,
-                    group1,contains(
-                             [post_init_per_group,
-                              init_per_group,
-                              pre_init_per_group,
-                              post_init_per_suite,
-                              init_per_suite,
-                              pre_init_per_suite]),
-                    []]}},
-         {?eh,cth,{'_',post_end_per_group,
-                   [ct_update_config_SUITE,
-                    group1,
-                    contains(
-                      [pre_end_per_group,
-                       post_init_per_group,
-                       init_per_group,
-                       pre_init_per_group,
-                       post_init_per_suite,
-                       init_per_suite,
-                       pre_init_per_suite]),
-                    ok,[]]}},
-         {?eh,tc_done,{ct_update_config_SUITE,{end_per_group,group1,[]},ok}},
-
-         {?eh,tc_start,{ct_update_config_SUITE,end_per_suite}},
-         {?eh,cth,{'_',pre_end_per_suite,
-                   [ct_update_config_SUITE,contains(
-                                             [post_init_per_suite,
-                                              init_per_suite,
-                                              pre_init_per_suite]),
-                    []]}},
-         {?eh,cth,{'_',post_end_per_suite,
-                   [ct_update_config_SUITE,contains(
-                                             [pre_end_per_suite,
-                                              post_init_per_suite,
-                                              init_per_suite,
-                                              pre_init_per_suite]),
-                    '_',[]]}},
-         {?eh,tc_done,{ct_update_config_SUITE,end_per_suite,ok}},
-         {?eh,test_done,{'DEF','STOP_TIME'}},
-         {?eh,cth,{'_',terminate,[contains(
-                                    [post_end_per_suite,
-                                     pre_end_per_suite,
-                                     post_init_per_suite,
-                                     init_per_suite,
-                                     pre_init_per_suite])]}},
-         {?eh,stop_logging,[]}
-        ];
-
+    update_config_cth_test_events(TestCaseEvents, Suite);
+test_events(update_config_cth2) ->
+    Suite = ct_update_config_SUITE2,
+    TestCaseEvents =
+        fun(Case, Result) ->
+                {PreEndPerTestcaseHookEventAdd, PostEndPerTestcaseEventAdd} =
+                    %% Case below is unexpected thing which needs clarification
+                    %% test_case_timetrap should behave the same
+                    case lists:member(Case, [test_case_timetrap]) of
+                        true ->
+                            {[], []};
+                        _ ->
+                            {
+                             [{?eh,cth,{'_',pre_end_per_testcase,
+                                        [Suite,
+                                         Case,contains(
+                                                [post_init_per_testcase,
+                                                 %% init_per_testcase,
+                                                 pre_init_per_testcase,
+                                                 post_init_per_group,
+                                                 init_per_group,
+                                                 pre_init_per_group,
+                                                 post_init_per_suite,
+                                                 init_per_suite,
+                                                 pre_init_per_suite]),
+                                         []]}}], [pre_end_per_testcase]}
+                    end,
+                [
+                 {?eh,tc_start,{Suite,Case}},
+                 {?eh,cth,{'_',pre_init_per_testcase,
+                           [Suite,
+                            Case,contains(
+                                        [post_init_per_group,
+                                         init_per_group,
+                                         pre_init_per_group,
+                                         post_init_per_suite,
+                                         init_per_suite,
+                                         pre_init_per_suite]),
+                            []]}},
+                 {?eh,cth,{'_',post_init_per_testcase,
+                           [Suite,
+                            Case,contains(
+                                        [
+                                         %% init_per_testcase,
+                                         pre_init_per_testcase,
+                                         post_init_per_group,
+                                         init_per_group,
+                                         pre_init_per_group,
+                                         post_init_per_suite,
+                                         init_per_suite,
+                                         pre_init_per_suite
+                                        ]),
+                            ok,[]]}}] ++
+                    PreEndPerTestcaseHookEventAdd ++
+                 [{?eh,cth,{'_',post_end_per_testcase,
+                           [Suite,
+                            Case,contains(PostEndPerTestcaseEventAdd ++
+                                        [post_init_per_testcase,
+                                         %% init_per_testcase,
+                                         pre_init_per_testcase,
+                                         post_init_per_group,
+                                         init_per_group,
+                                         pre_init_per_group,
+                                         post_init_per_suite,
+                                         init_per_suite,
+                                         pre_init_per_suite]),
+                            Result,[]]}},
+                 {?eh,tc_done,{Suite,Case,ok}}
+                ]
+        end,
+    update_config_cth_test_events(TestCaseEvents, Suite);
 test_events(state_update_cth) ->
     [
      {?eh,start_logging,{'DEF','RUNDIR'}},
@@ -2855,6 +2839,113 @@ test_events(crash_all) ->
 test_events(ok) ->
     ok.
 
+update_config_cth_test_events(TestCaseEvents, Suite) ->
+    MaybeEndPerTestcaseCrashEvents =
+        case Suite of
+            ct_update_config_SUITE ->
+                TestCaseEvents(test_case_timetrap_end_per_testcase_crash,
+                               {timetrap_timeout,1000}) ++
+                    TestCaseEvents(test_case_badmatch,
+                                   {error, {{badmatch,2}, '_'}}) ++
+                    TestCaseEvents(test_case_spawn_crash, {'EXIT',bam});
+            _ ->
+                []
+        end,
+    [
+     {?eh,start_logging,{'DEF','RUNDIR'}},
+     {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+     {?eh,cth,{'_',init,['_',[]]}},
+
+     {?eh,tc_start,{Suite,init_per_suite}},
+     {?eh,cth,{'_',pre_init_per_suite,
+	       [Suite,contains([]),[]]}},
+     {?eh,cth,{'_',post_init_per_suite,
+	       [Suite,
+		'$proplist',
+		contains(
+                  [init_per_suite,
+                   pre_init_per_suite]),
+		[]]}},
+     {?eh,tc_done,{Suite,init_per_suite,ok}},
+
+     {?eh,tc_start,{Suite, {init_per_group,group1,[]}}},
+     {?eh,cth,{'_',pre_init_per_group,
+	       [Suite,
+                group1,contains(
+			 [post_init_per_suite,
+			  init_per_suite,
+			  pre_init_per_suite]),
+		[]]}},
+     {?eh,cth,{'_',post_init_per_group,
+	       [Suite,
+                group1,
+		contains(
+		  [post_init_per_suite,
+		   init_per_suite,
+		   pre_init_per_suite]),
+		contains(
+		  [init_per_group,
+		   pre_init_per_group,
+		   post_init_per_suite,
+		   init_per_suite,
+		   pre_init_per_suite]),
+                []]}},
+     {?eh,tc_done,{Suite,{init_per_group,group1,[]},ok}}] ++
+        TestCaseEvents(test_case, ok) ++
+        TestCaseEvents(test_case_fail,
+                       {error,{test_case_failed,because_i_want_failure}}) ++
+        TestCaseEvents(test_case_timetrap, {timetrap_timeout,1000}) ++
+        MaybeEndPerTestcaseCrashEvents ++
+        [{?eh,tc_start,{Suite, {end_per_group,group1,[]}}},
+         {?eh,cth,{'_',pre_end_per_group,
+                   [Suite,
+                    group1,contains(
+                             [post_init_per_group,
+                              init_per_group,
+                              pre_init_per_group,
+                              post_init_per_suite,
+                              init_per_suite,
+                              pre_init_per_suite]),
+                    []]}},
+         {?eh,cth,{'_',post_end_per_group,
+                   [Suite,
+                    group1,
+                    contains(
+                      [pre_end_per_group,
+                       post_init_per_group,
+                       init_per_group,
+                       pre_init_per_group,
+                       post_init_per_suite,
+                       init_per_suite,
+                       pre_init_per_suite]),
+                    ok,[]]}},
+         {?eh,tc_done,{Suite,{end_per_group,group1,[]},ok}},
+
+         {?eh,tc_start,{Suite,end_per_suite}},
+         {?eh,cth,{'_',pre_end_per_suite,
+                   [Suite,contains(
+                                             [post_init_per_suite,
+                                              init_per_suite,
+                                              pre_init_per_suite]),
+                    []]}},
+         {?eh,cth,{'_',post_end_per_suite,
+                   [Suite,contains(
+                                             [pre_end_per_suite,
+                                              post_init_per_suite,
+                                              init_per_suite,
+                                              pre_init_per_suite]),
+                    '_',[]]}},
+         {?eh,tc_done,{Suite,end_per_suite,ok}},
+         {?eh,test_done,{'DEF','STOP_TIME'}},
+         {?eh,cth,{'_',terminate,[contains(
+                                    [post_end_per_suite,
+                                     pre_end_per_suite,
+                                     post_init_per_suite,
+                                     init_per_suite,
+                                     pre_init_per_suite])]}},
+         {?eh,stop_logging,[]}
+        ].
+
 %% test events help functions
 contains(List) ->
     fun(Proplist) when is_list(Proplist) ->
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_update_config_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_update_config_SUITE.erl
index 61fd1a329d..2205bb465e 100644
--- a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_update_config_SUITE.erl
+++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_update_config_SUITE.erl
@@ -23,7 +23,7 @@
 -suite_defaults([{timetrap, {minutes, 10}}]).
 
 %% Note: This directive should only be used in test suites.
--compile(export_all).
+-compile([export_all, nowarn_export_all]).
 
 -include("ct.hrl").
 
@@ -39,27 +39,53 @@ end_per_suite(_Config) ->
 init_per_testcase(_TestCase, Config) ->
     [{init_per_testcase,?now}|Config].
 
+end_per_testcase(test_case_timetrap_end_per_testcase_crash, _Config) ->
+    1/0, % invoke code crash
+    ok;
 end_per_testcase(_TestCase, _Config) ->
     ok.
 
-init_per_group(GroupName, Config) ->
+init_per_group(_GroupName, Config) ->
     [{init_per_group,?now}|Config].
 
-end_per_group(GroupName, Config) ->
+end_per_group(_GroupName, _Config) ->
     ok.
 
 all() ->
     [{group,group1}].
 
 groups() ->
-    [{group1,[],[test_case, test_case_timetrap]}].
+    [{group1,[],[test_case,
+                 test_case_fail,
+                 test_case_timetrap,
+                 test_case_timetrap_end_per_testcase_crash,
+                 test_case_badmatch,
+                 test_case_spawn_crash]}].
 
-%% Test cases starts here.
 test_case(Config) when is_list(Config) ->
     ok.
 
+test_case_fail(Config) when is_list(Config) ->
+    ct:fail(because_i_want_failure).
+
 test_case_timetrap() ->
     [{timetrap, {seconds, 1}}].
 
 test_case_timetrap(_) ->
     ct:sleep(infinity).
+
+test_case_badmatch(_) ->
+    1 = 2.
+
+%% test case is a slightly different flow, but at the moment behavior should be
+%% the same as for test_case_timetrap, as crash in end_per_testcase is catched
+%% in test_server module
+test_case_timetrap_end_per_testcase_crash() ->
+    [{timetrap, {seconds, 1}}].
+
+test_case_timetrap_end_per_testcase_crash(Config) when is_list(Config) ->
+    ct:sleep(infinity).
+
+test_case_spawn_crash(_Config) ->
+    spawn_link(fun() -> error(bam) end),
+    ct:sleep(infinity).
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_update_config_SUITE2.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_update_config_SUITE2.erl
new file mode 100644
index 0000000000..7fda6354b5
--- /dev/null
+++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_update_config_SUITE2.erl
@@ -0,0 +1,62 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2023. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(ct_update_config_SUITE2).
+
+-suite_defaults([{timetrap, {minutes, 10}}]).
+
+%% Note: This directive should only be used in test suites.
+-compile([export_all, nowarn_export_all]).
+-include("ct.hrl").
+
+-define(now, ct_test_support:unique_timestamp()).
+
+%% Test server callback functions
+init_per_suite(Config) ->
+    [{init_per_suite,?now}|Config].
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(_GroupName, Config) ->
+    [{init_per_group,?now}|Config].
+
+end_per_group(_GroupName, _Config) ->
+    ok.
+
+all() ->
+    [{group,group1}].
+
+groups() ->
+    [{group1,[],[test_case,
+                 test_case_fail,
+                 test_case_timetrap]}].
+
+test_case(Config) when is_list(Config) ->
+    ok.
+
+test_case_fail(Config) when is_list(Config) ->
+    ct:fail(because_i_want_failure).
+
+test_case_timetrap() ->
+    [{timetrap, {seconds, 1}}].
+
+test_case_timetrap(_) ->
+    ct:sleep(infinity).
diff --git a/lib/common_test/test/ct_test_support.erl b/lib/common_test/test/ct_test_support.erl
index 3b7fc70313..4dcb1863ce 100644
--- a/lib/common_test/test/ct_test_support.erl
+++ b/lib/common_test/test/ct_test_support.erl
@@ -522,7 +522,7 @@ verify_events(TEvs, Evs, Node, Config) ->
 
 verify_events1([TestEv|_], [{TEH,#event{name=stop_logging,node=Node,data=_}}|_], Node, _)
   when element(1,TestEv) == TEH, element(2,TestEv) =/= stop_logging ->
-    test_server:format("Failed to find ~tp in the list of events!~n", [TestEv]),
+    test_server:format("(I) Failed to find ~tp in the list of events!~n", [TestEv]),
     exit({event_not_found,TestEv});
 
 verify_events1(TEvs = [TestEv | TestEvs], Evs = [_|Events], Node, Config) ->
@@ -545,7 +545,7 @@ verify_events1(TEvs = [TestEv | TestEvs], Evs = [_|Events], Node, Config) ->
     end;
 
 verify_events1([TestEv|_], [], _, _) ->
-    test_server:format("Failed to find ~tp in the list of events!~n", [TestEv]),
+    test_server:format("(II) Failed to find ~tp in the list of events!~n", [TestEv]),
     exit({event_not_found,TestEv});
 
 verify_events1([], Evs, _, Config) ->
-- 
2.35.3

Places

File 1303-ct-extend-ct_hooks_SUITE.patch of Package erlang (Revision 2587bae7bf0dfb77beed2069b73c8f2a)

Places