File 1285-Update-the-doc-CT-test_spec-grammar-to-mat... of Package erlang

Overview Repositories Revisions Requests Users Attributes Meta

File 1285-Update-the-doc-CT-test_spec-grammar-to-match-actual-.patch of Package erlang (Revision 84136f040b8e10a77a20fc548a9bd409)

Currently displaying revision 84136f040b8e10a77a20fc548a9bd409 , Show latest

From da1224651c206e4ef33a1575fb37ff0877ab4847 Mon Sep 17 00:00:00 2001
From: acw224 <thomasdcuvillier@gmail.com>
Date: Tue, 3 Aug 2021 08:26:13 -0700
Subject: [PATCH 1/2] Update the doc CT test_spec grammar to match actual
 behavior

The grammar in the doc doesn't match actual behavior
Two problems:
Groups = GroupPath, GroupPath = [GroupName] leads to think that
[group1, group2] is the group path group1-group2 whereas it is
understood as an array of GroupSpec.
GroupSpec = [group, properties, GroupSpec] was not recognised, only
[group, properties, [GroupSpec]

An updated grammar is proposed, together with additional tests
to validate the pertinence of the new grammar.
---
 lib/common_test/doc/src/run_test_chapter.xml  |   8 +-
 lib/common_test/test/ct_testspec_1_SUITE.erl  | 189 ++++++++++++++++--
 .../groups_1/groups_11_SUITE.erl              |  50 ++++-
 3 files changed, 222 insertions(+), 25 deletions(-)

diff --git a/lib/common_test/doc/src/run_test_chapter.xml b/lib/common_test/doc/src/run_test_chapter.xml
index 0b8657ced3..1d88abc27e 100644
--- a/lib/common_test/doc/src/run_test_chapter.xml
+++ b/lib/common_test/doc/src/run_test_chapter.xml
@@ -955,9 +955,9 @@
  Dir             = string()
  Suites          = atom() | [atom()] | all
  Suite           = atom()
- Groups          = GroupPath | [GroupPath] | GroupSpec | [GroupSpec] | all
- GroupPath       = [GroupName]
- GroupSpec       = GroupName | {GroupName,Properties} | {GroupName,Properties,GroupSpec}
+ Groups          = GroupPath | GroupSpec | [GroupSpec] | all
+ GroupPath       = [[GroupName]]
+ GroupSpec       = GroupName | {GroupName,Properties} | {GroupName,Properties,[GroupSpec]}
  GroupName       = atom()
  GroupNames      = GroupName | [GroupName]
  Cases           = atom() | [atom()] | all
@@ -1608,5 +1608,3 @@ div.error pre { color:white }</pre>
       
   </section>
 </chapter>
-
-
diff --git a/lib/common_test/test/ct_testspec_1_SUITE.erl b/lib/common_test/test/ct_testspec_1_SUITE.erl
index 2d2c42999f..31ae19a033 100644
--- a/lib/common_test/test/ct_testspec_1_SUITE.erl
+++ b/lib/common_test/test/ct_testspec_1_SUITE.erl
@@ -63,6 +63,8 @@ all() ->
     [all_suites, skip_all_suites, suite, skip_suite,
      all_testcases, skip_all_testcases, testcase,
      skip_testcase, all_groups, skip_all_groups, group,
+     group_path, group_config,
+     group_spec, multi_group, multi_group_config,
      skip_group, group_all_testcases,
      skip_group_all_testcases, group_testcase,
      skip_group_testcase, topgroup, subgroup, skip_subgroup,
@@ -207,12 +209,41 @@ skip_all_groups(Config) when is_list(Config) ->
 
 group(Config) when is_list(Config) ->
     DataDir = ?config(data_dir, Config),
-
     TestDir = filename:join(DataDir, "groups_1"),
     TestSpec = [{groups,TestDir,groups_11_SUITE,test_group_1a}],
-
     setup_and_execute(group, TestSpec, Config).
 
+group_path(Config) when is_list(Config) ->
+    DataDir = ?config(data_dir, Config),
+    TestDir = filename:join(DataDir, "groups_1"),
+    TestSpec = [{groups,TestDir,groups_11_SUITE,[[test_group_2, test_group_3]]}],
+    setup_and_execute(group_path, TestSpec, Config).
+
+group_config(Config) when is_list(Config) ->
+    DataDir = ?config(data_dir, Config),
+    TestDir = filename:join(DataDir, "groups_1"),
+    TestSpec = [{groups,TestDir,groups_11_SUITE, {test_group_1c, [parallel]}}],
+    setup_and_execute(group_config, TestSpec, Config).
+
+group_spec(Config) when is_list(Config) ->
+    DataDir = ?config(data_dir, Config),
+    TestDir = filename:join(DataDir, "groups_1"),
+    TestSpec = [{groups,TestDir,groups_11_SUITE, {test_group_2, [parallel], [{test_group_3, [sequence]}]}}],
+    setup_and_execute(group_spec, TestSpec, Config).
+
+multi_group(Config) when is_list(Config) ->
+    DataDir = ?config(data_dir, Config),
+    TestDir = filename:join(DataDir, "groups_1"),
+    TestSpec = [{groups,TestDir,groups_11_SUITE, [test_group_1a, test_group_3]}],
+    setup_and_execute(multi_group, TestSpec, Config).
+
+multi_group_config(Config) when is_list(Config) ->
+    DataDir = ?config(data_dir, Config),
+    TestDir = filename:join(DataDir, "groups_1"),
+    TestSpec = [{groups,TestDir,groups_11_SUITE, [{test_group_2, [parallel], [{test_group_3, [sequence]}]},
+     {test_group_9, [sequence], [{test_group_8, [parallel]}]}]}],
+    setup_and_execute(multi_group_config, TestSpec, Config).
+
 skip_group(Config) when is_list(Config) ->
     DataDir = ?config(data_dir, Config),
 
@@ -224,7 +255,6 @@ skip_group(Config) when is_list(Config) ->
 
     setup_and_execute(skip_group, TestSpec, Config).
 
-
 %%%-----------------------------------------------------------------
 %%%
 
@@ -753,7 +783,7 @@ test_events(all_groups) ->
     [
      {?eh,start_logging,'_'},
      {?eh,tc_start,{groups_11_SUITE,init_per_suite}},
-     {?eh,test_stats,{12,0,{0,0}}},
+     {?eh,test_stats,{16,0,{0,0}}},
      {?eh,tc_done,{groups_11_SUITE,end_per_suite,'_'}},
      {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}}
     ];
@@ -761,7 +791,7 @@ test_events(all_groups) ->
 test_events(skip_all_groups) ->
     [
      {?eh,start_logging,'_'},
-     {?eh,start_info,{1,1,12}},
+     {?eh,start_info,{1,1,16}},
      {?eh,tc_start,{groups_11_SUITE,init_per_suite}},
      {?eh,tc_user_skip,{groups_11_SUITE,{init_per_group,test_group_1a},"SKIPPED!"}},
      {?eh,tc_user_skip,{groups_11_SUITE,{testcase_1a,test_group_1a},"SKIPPED!"}},
@@ -775,26 +805,38 @@ test_events(skip_all_groups) ->
      {?eh,tc_user_skip,{groups_11_SUITE,{testcase_1b,test_group_1b},"SKIPPED!"}},
      {?eh,test_stats,{0,0,{4,0}}},
      {?eh,tc_user_skip,{groups_11_SUITE,{end_per_group,test_group_1b},"SKIPPED!"}},
+     {?eh,tc_user_skip,{groups_11_SUITE,{init_per_group,test_group_1c},"SKIPPED!"}},
+     {?eh,tc_user_skip,{groups_11_SUITE,{testcase_1a,test_group_1c},"SKIPPED!"}},
+     {?eh,test_stats,{0,0,{5,0}}},
+     {?eh,tc_user_skip,{groups_11_SUITE,{testcase_1b,test_group_1c},"SKIPPED!"}},
+     {?eh,test_stats,{0,0,{6,0}}},
+     {?eh,tc_user_skip,{groups_11_SUITE,{end_per_group,test_group_1c},"SKIPPED!"}},
      {?eh,tc_user_skip,{groups_11_SUITE,{init_per_group,test_group_2},"SKIPPED!"}},
      {?eh,tc_user_skip,{groups_11_SUITE,{testcase_2a,test_group_2},"SKIPPED!"}},
-     {?eh,test_stats,{0,0,{5,0}}},
+     {?eh,test_stats,{0,0,{7,0}}},
      {?eh,tc_user_skip,{groups_11_SUITE,{testcase_3a,test_group_3},"SKIPPED!"}},
-     {?eh,test_stats,{0,0,{6,0}}},
+     {?eh,test_stats,{0,0,{8,0}}},
      {?eh,tc_user_skip,{groups_11_SUITE,{testcase_3b,test_group_3},"SKIPPED!"}},
-     {?eh,test_stats,{0,0,{7,0}}},
+     {?eh,test_stats,{0,0,{9,0}}},
      {?eh,tc_user_skip,{groups_11_SUITE,{testcase_2b,test_group_2},"SKIPPED!"}},
-     {?eh,test_stats,{0,0,{8,0}}},
+     {?eh,test_stats,{0,0,{10,0}}},
      {?eh,tc_user_skip,{groups_11_SUITE,{end_per_group,test_group_2},"SKIPPED!"}},
      {?eh,tc_user_skip,{groups_11_SUITE,{init_per_group,test_group_4},"SKIPPED!"}},
      {?eh,tc_user_skip,{groups_11_SUITE,{testcase_5a,test_group_5},"SKIPPED!"}},
-     {?eh,test_stats,{0,0,{9,0}}},
+     {?eh,test_stats,{0,0,{11,0}}},
      {?eh,tc_user_skip,{groups_11_SUITE,{testcase_7a,test_group_7},"SKIPPED!"}},
-     {?eh,test_stats,{0,0,{10,0}}},
+     {?eh,test_stats,{0,0,{12,0}}},
      {?eh,tc_user_skip,{groups_11_SUITE,{testcase_7b,test_group_7},"SKIPPED!"}},
-     {?eh,test_stats,{0,0,{11,0}}},
+     {?eh,test_stats,{0,0,{13,0}}},
      {?eh,tc_user_skip,{groups_11_SUITE,{testcase_5b,test_group_5},"SKIPPED!"}},
-     {?eh,test_stats,{0,0,{12,0}}},
+     {?eh,test_stats,{0,0,{14,0}}},
      {?eh,tc_user_skip,{groups_11_SUITE,{end_per_group,test_group_4},"SKIPPED!"}},
+     {?eh,tc_user_skip,{groups_11_SUITE,{init_per_group,test_group_9},"SKIPPED!"}},
+     {?eh,tc_user_skip,{groups_11_SUITE,{testcase_9,test_group_9},"SKIPPED!"}},
+     {?eh,test_stats,{0,0,{15,0}}},
+     {?eh,tc_user_skip,{groups_11_SUITE,{testcase_8,test_group_8},"SKIPPED!"}},
+     {?eh,test_stats,{0,0,{16,0}}},
+     {?eh,tc_user_skip,{groups_11_SUITE,{end_per_group,test_group_9},"SKIPPED!"}},
      {?eh,tc_start,{groups_11_SUITE,end_per_suite}},
      {?eh,tc_done,{groups_11_SUITE,end_per_suite,ok}},
      {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}}
@@ -813,6 +855,29 @@ test_events(group) ->
      {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}}
     ];
  
+test_events(group_path) ->
+    [
+     {?eh,start_logging,'_'},
+     {?eh,tc_start,{groups_11_SUITE,init_per_suite}},
+     {?eh,tc_start,{groups_11_SUITE,{init_per_group,test_group_2,[sequence]}}},
+     {?eh,tc_start,{groups_11_SUITE,{init_per_group,test_group_3,[parallel]}}},
+     {?eh,tc_done,{groups_11_SUITE,{end_per_group,test_group_3,[parallel]},'_'}},
+     {?eh,tc_done,{groups_11_SUITE,{end_per_group,test_group_2,[sequence]},'_'}},
+     {?eh,tc_done,{groups_11_SUITE,end_per_suite,'_'}},
+     {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}}
+    ];
+
+test_events(group_config) ->
+    [
+     {?eh,start_logging,'_'},
+     {?eh,tc_start,{groups_11_SUITE,init_per_suite}},
+     {?eh,tc_start,{groups_11_SUITE,{init_per_group,test_group_1c,[parallel]}}},
+     {?eh,tc_done,{groups_11_SUITE,{end_per_group,test_group_1c,[parallel]},'_'}},
+     {?eh,tc_done,{groups_11_SUITE,end_per_suite,'_'}},
+     {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}}
+    ];
+
+
 test_events(skip_group) ->
     [
       {?eh,start_logging,'_'},
@@ -858,6 +923,104 @@ test_events(group_all_testcases) ->
      {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}}
     ];
  
+test_events(group_spec) ->
+    [
+     {?eh,start_logging,'_'},
+     {?eh,tc_start,{groups_11_SUITE,init_per_suite}},
+     {parallel, [
+     {?eh,tc_start,{groups_11_SUITE,{init_per_group,test_group_2,[parallel]}}},
+     {?eh,tc_done,{groups_11_SUITE,{init_per_group,test_group_2,[parallel]},ok}},
+     {?eh,tc_start,{groups_11_SUITE,testcase_2a}},
+     {?eh,tc_done,{groups_11_SUITE,testcase_2a,ok}},
+     [
+     {?eh,tc_start,{groups_11_SUITE,{init_per_group,test_group_3,[sequence]}}},
+     {?eh,tc_start,{groups_11_SUITE,testcase_3a}},
+     {?eh,tc_start,{groups_11_SUITE,testcase_3b}},
+     {?eh,test_stats,{3,0,{0,0}}},
+     {?eh,tc_done,{groups_11_SUITE,{end_per_group,test_group_3,[sequence]},'_'}}
+     ],
+     {?eh,tc_start,{groups_11_SUITE,testcase_2b}},
+     {?eh,tc_done,{groups_11_SUITE,testcase_2b,ok}},
+     {?eh,test_stats,{4,0,{0,0}}},
+     {?eh,tc_start,{groups_11_SUITE,{end_per_group,test_group_2,[parallel]}}},
+     {?eh,tc_done,{groups_11_SUITE,{end_per_group,test_group_2,[parallel]},ok}}
+     ]},
+     {?eh,tc_done,{groups_11_SUITE,end_per_suite,'_'}},
+     {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}}
+    ];
+
+test_events(multi_group) ->
+    [
+     {?eh,start_logging,'_'},
+     {?eh,tc_start,{groups_11_SUITE,init_per_suite}},
+     {?eh,tc_start,{groups_11_SUITE,{init_per_group,test_group_1a,[]}}},
+     {?eh,tc_start,{groups_11_SUITE,testcase_1a}},
+     {?eh,tc_start,{groups_11_SUITE,testcase_1b}},
+     {?eh,test_stats,{2,0,{0,0}}},
+     {?eh,tc_done,{groups_11_SUITE,{end_per_group,test_group_1a,[]},'_'}},
+     {?eh,tc_start,{groups_11_SUITE,{init_per_group,test_group_2,[sequence]}}},
+     {parallel, [
+     {?eh,tc_start,{groups_11_SUITE,{init_per_group,test_group_3,[parallel]}}},
+     {?eh,tc_done,{groups_11_SUITE,{init_per_group,test_group_3,[parallel]}, ok}},
+     {?eh,tc_start,{groups_11_SUITE,testcase_3a}},
+     {?eh,tc_done,{groups_11_SUITE,testcase_3a,ok}},
+     {?eh,tc_start,{groups_11_SUITE,testcase_3b}},
+     {?eh,tc_done,{groups_11_SUITE,testcase_3b,ok}},
+     {?eh,test_stats,{4,0,{0,0}}},
+     {?eh,tc_start,{groups_11_SUITE,{end_per_group,test_group_3,[parallel]}}},
+     {?eh,tc_done,{groups_11_SUITE,{end_per_group,test_group_3,[parallel]}, ok}}
+     ]},
+     {?eh,tc_done,{groups_11_SUITE,{end_per_group,test_group_2,[sequence]},'_'}},
+     {?eh,tc_done,{groups_11_SUITE,end_per_suite,'_'}},
+     {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}}
+    ];
+
+test_events(multi_group_config) ->
+    [
+     {?eh,start_logging,'_'},
+     {?eh,tc_start,{groups_11_SUITE,init_per_suite}},
+     {parallel, [
+     {?eh,tc_start,{groups_11_SUITE,{init_per_group,test_group_2,[parallel]}}},
+     {?eh,tc_done,{groups_11_SUITE,{init_per_group,test_group_2,[parallel]},ok}},
+     {?eh,tc_start,{groups_11_SUITE,testcase_2a}},
+     {?eh,tc_done,{groups_11_SUITE,testcase_2a,ok}},
+     [
+     {?eh,tc_start,{groups_11_SUITE,{init_per_group,test_group_3,[sequence]}}},
+     {?eh,tc_start,{groups_11_SUITE,testcase_3a}},
+     {?eh,tc_start,{groups_11_SUITE,testcase_3b}},
+     {?eh,test_stats,{3,0,{0,0}}},
+     {?eh,tc_done,{groups_11_SUITE,{end_per_group,test_group_3,[sequence]},'_'}}
+     ],
+     {?eh,tc_start,{groups_11_SUITE,testcase_2b}},
+     {?eh,tc_done,{groups_11_SUITE,testcase_2b,ok}},
+     {?eh,test_stats,{4,0,{0,0}}},
+     {?eh,tc_start,{groups_11_SUITE,{end_per_group,test_group_2,[parallel]}}},
+     {?eh,tc_done,{groups_11_SUITE,{end_per_group,test_group_2,[parallel]},ok}}
+     ]},
+     [
+     {?eh,tc_start,{groups_11_SUITE,{init_per_group,test_group_9,[sequence]}}},
+     {?eh,tc_done,{groups_11_SUITE,{init_per_group,test_group_9,[sequence]},ok}},
+     {?eh,tc_start,{groups_11_SUITE,testcase_9}},
+     {?eh,tc_done,{groups_11_SUITE,testcase_9,ok}},
+     {?eh,test_stats,{5,0,{0,0}}},
+     {parallel,[
+     {?eh,tc_start,
+     {groups_11_SUITE,{init_per_group,test_group_8,[parallel]}}},
+     {?eh,tc_done,{groups_11_SUITE,{init_per_group,test_group_8,[parallel]},ok}},
+     {?eh,tc_start,{groups_11_SUITE,testcase_8}},
+     {?eh,tc_done,{groups_11_SUITE,testcase_8,ok}},
+     {?eh,test_stats,{6,0,{0,0}}},
+     {?eh,tc_start,{groups_11_SUITE,{end_per_group,test_group_8,[parallel]}}},
+     {?eh,tc_done,{groups_11_SUITE,{end_per_group,test_group_8,[parallel]},ok}}
+     ]},
+     {?eh,tc_start,{groups_11_SUITE,{end_per_group,test_group_9,[sequence]}}},
+     {?eh,tc_done,{groups_11_SUITE,{end_per_group,test_group_9,[sequence]},ok}}
+     ],
+     {?eh,tc_done,{groups_11_SUITE,end_per_suite,'_'}},
+     {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}}
+    ];
+
+
 test_events(skip_group_all_testcases) ->
     [
      {?eh,start_logging,'_'},
diff --git a/lib/common_test/test/ct_testspec_1_SUITE_data/groups_1/groups_11_SUITE.erl b/lib/common_test/test/ct_testspec_1_SUITE_data/groups_1/groups_11_SUITE.erl
index 4d481fe3b8..56c97d9782 100644
--- a/lib/common_test/test/ct_testspec_1_SUITE_data/groups_1/groups_11_SUITE.erl
+++ b/lib/common_test/test/ct_testspec_1_SUITE_data/groups_1/groups_11_SUITE.erl
@@ -36,9 +36,11 @@ groups() ->
 
       {test_group_1b, [], [testcase_1a,testcase_1b]},
 
-      {test_group_2, [], [testcase_2a,
+      {test_group_1c, [sequence], [testcase_1a,testcase_1b]},
 
-			  {test_group_3, [], [testcase_3a,
+      {test_group_2, [sequence], [testcase_2a,
+
+			  {test_group_3, [parallel], [testcase_3a,
 					      testcase_3b]},
 			  testcase_2b]},
 
@@ -49,7 +51,12 @@ groups() ->
 					  testcase_5b]}]},
       {test_group_6, [{group, test_group_7}]},
 
-      {test_group_7, [testcase_7a,testcase_7b]}
+      {test_group_7, [testcase_7a,testcase_7b]},
+
+      {test_group_8, [sequence], [testcase_8]},
+
+      {test_group_9, [parallel], [testcase_9, {group, test_group_8}]}
+
      ].
 
 all() ->
@@ -64,10 +71,11 @@ all() ->
 %% this func only for internal test purposes
 grs_and_tcs() ->
     {[
-      test_group_1a, test_group_1b,
+      test_group_1a, test_group_1b, test_group_1c,
       test_group_2, test_group_3,
       test_group_4, test_group_5,
-      test_group_6, test_group_7
+      test_group_6, test_group_7,
+      test_group_8, test_group_9
      ],
      [
       testcase_1,
@@ -77,7 +85,9 @@ grs_and_tcs() ->
       testcase_3a, testcase_3b,
       testcase_3,
       testcase_5a, testcase_5b,
-      testcase_7a, testcase_7b
+      testcase_7a, testcase_7b,
+      testcase_8,
+      testcase_9
      ]}.
 
 %%--------------------------------------------------------------------
@@ -95,7 +105,10 @@ end_per_suite(Config) ->
 %%--------------------------------------------------------------------
 
 init_per_group(Group, Config) ->
-    [{name,Group}] = ?config(tc_group_properties,Config),
+    Group = case ?config(tc_group_properties,Config) of
+        [{name, Group0}] -> Group0;
+        [{name, Group0}, _Props] -> Group0
+    end,
     {Grs,_} = grs_and_tcs(),
     case lists:member(Group, Grs) of
 	true ->
@@ -164,8 +177,11 @@ testcase_1a(Config) ->
 	_ ->
 	    case ?config(test_group_1b,Config) of
 		test_group_1b -> ok;
+		_ -> case ?config(test_group_1c,Config) of
+		        test_group_1c -> ok;
 		_ -> ct:fail(no_group_data)
 	    end
+	    end
     end,
     testcase_1a = ?config(testcase_1a,Config),
     ok.
@@ -178,8 +194,11 @@ testcase_1b(Config) ->
 	_ ->
 	    case ?config(test_group_1b,Config) of
 		test_group_1b -> ok;
+		_ -> case ?config(test_group_1c,Config) of
+		        test_group_1c -> ok;
 		_ -> ct:fail(no_group_data)
 	    end
+	    end
     end,
     undefined = ?config(testcase_1a,Config),
     testcase_1b = ?config(testcase_1b,Config),
@@ -191,6 +210,7 @@ testcase_2(Config) ->
     init = ?config(suite,Config),
     undefined = ?config(test_group_1a,Config),
     undefined = ?config(test_group_1b,Config),
+    undefined = ?config(test_group_1c, Config),
     testcase_2 = ?config(testcase_2,Config),
     ok.
 
@@ -280,3 +300,19 @@ testcase_7b(Config) ->
     undefined = ?config(testcase_7a,Config),
     testcase_7b = ?config(testcase_7b,Config),
     ok.
+testcase_8() ->
+    [].
+testcase_8(Config) ->
+    init = ?config(suite,Config),
+    test_group_9 = ?config(test_group_9,Config),
+    test_group_8 = ?config(test_group_8,Config),
+    testcase_8 = ?config(testcase_8,Config),
+    ok.
+testcase_9() ->
+    [].
+testcase_9(Config) ->
+    init = ?config(suite,Config),
+    test_group_9 = ?config(test_group_9,Config),
+    undefined = ?config(test_group_8,Config),
+    testcase_9 = ?config(testcase_9,Config),
+    ok.
-- 
2.31.1

Places

File 1285-Update-the-doc-CT-test_spec-grammar-to-match-actual-.patch of Package erlang (Revision 84136f040b8e10a77a20fc548a9bd409)

Places