File 1285-Update-the-doc-CT-test_spec-grammar-to-match-actual-.patch of Package erlang
From da1224651c206e4ef33a1575fb37ff0877ab4847 Mon Sep 17 00:00:00 2001
From: acw224 <thomasdcuvillier@gmail.com>
Date: Tue, 3 Aug 2021 08:26:13 -0700
Subject: [PATCH 1/2] Update the doc CT test_spec grammar to match actual
behavior
The grammar in the doc doesn't match actual behavior
Two problems:
Groups = GroupPath, GroupPath = [GroupName] leads to think that
[group1, group2] is the group path group1-group2 whereas it is
understood as an array of GroupSpec.
GroupSpec = [group, properties, GroupSpec] was not recognised, only
[group, properties, [GroupSpec]
An updated grammar is proposed, together with additional tests
to validate the pertinence of the new grammar.
---
lib/common_test/doc/src/run_test_chapter.xml | 8 +-
lib/common_test/test/ct_testspec_1_SUITE.erl | 189 ++++++++++++++++--
.../groups_1/groups_11_SUITE.erl | 50 ++++-
3 files changed, 222 insertions(+), 25 deletions(-)
diff --git a/lib/common_test/doc/src/run_test_chapter.xml b/lib/common_test/doc/src/run_test_chapter.xml
index 0b8657ced3..1d88abc27e 100644
--- a/lib/common_test/doc/src/run_test_chapter.xml
+++ b/lib/common_test/doc/src/run_test_chapter.xml
@@ -955,9 +955,9 @@
Dir = string()
Suites = atom() | [atom()] | all
Suite = atom()
- Groups = GroupPath | [GroupPath] | GroupSpec | [GroupSpec] | all
- GroupPath = [GroupName]
- GroupSpec = GroupName | {GroupName,Properties} | {GroupName,Properties,GroupSpec}
+ Groups = GroupPath | GroupSpec | [GroupSpec] | all
+ GroupPath = [[GroupName]]
+ GroupSpec = GroupName | {GroupName,Properties} | {GroupName,Properties,[GroupSpec]}
GroupName = atom()
GroupNames = GroupName | [GroupName]
Cases = atom() | [atom()] | all
@@ -1608,5 +1608,3 @@ div.error pre { color:white }</pre>
</section>
</chapter>
-
-
diff --git a/lib/common_test/test/ct_testspec_1_SUITE.erl b/lib/common_test/test/ct_testspec_1_SUITE.erl
index 2d2c42999f..31ae19a033 100644
--- a/lib/common_test/test/ct_testspec_1_SUITE.erl
+++ b/lib/common_test/test/ct_testspec_1_SUITE.erl
@@ -63,6 +63,8 @@ all() ->
[all_suites, skip_all_suites, suite, skip_suite,
all_testcases, skip_all_testcases, testcase,
skip_testcase, all_groups, skip_all_groups, group,
+ group_path, group_config,
+ group_spec, multi_group, multi_group_config,
skip_group, group_all_testcases,
skip_group_all_testcases, group_testcase,
skip_group_testcase, topgroup, subgroup, skip_subgroup,
@@ -207,12 +209,41 @@ skip_all_groups(Config) when is_list(Config) ->
group(Config) when is_list(Config) ->
DataDir = ?config(data_dir, Config),
-
TestDir = filename:join(DataDir, "groups_1"),
TestSpec = [{groups,TestDir,groups_11_SUITE,test_group_1a}],
-
setup_and_execute(group, TestSpec, Config).
+group_path(Config) when is_list(Config) ->
+ DataDir = ?config(data_dir, Config),
+ TestDir = filename:join(DataDir, "groups_1"),
+ TestSpec = [{groups,TestDir,groups_11_SUITE,[[test_group_2, test_group_3]]}],
+ setup_and_execute(group_path, TestSpec, Config).
+
+group_config(Config) when is_list(Config) ->
+ DataDir = ?config(data_dir, Config),
+ TestDir = filename:join(DataDir, "groups_1"),
+ TestSpec = [{groups,TestDir,groups_11_SUITE, {test_group_1c, [parallel]}}],
+ setup_and_execute(group_config, TestSpec, Config).
+
+group_spec(Config) when is_list(Config) ->
+ DataDir = ?config(data_dir, Config),
+ TestDir = filename:join(DataDir, "groups_1"),
+ TestSpec = [{groups,TestDir,groups_11_SUITE, {test_group_2, [parallel], [{test_group_3, [sequence]}]}}],
+ setup_and_execute(group_spec, TestSpec, Config).
+
+multi_group(Config) when is_list(Config) ->
+ DataDir = ?config(data_dir, Config),
+ TestDir = filename:join(DataDir, "groups_1"),
+ TestSpec = [{groups,TestDir,groups_11_SUITE, [test_group_1a, test_group_3]}],
+ setup_and_execute(multi_group, TestSpec, Config).
+
+multi_group_config(Config) when is_list(Config) ->
+ DataDir = ?config(data_dir, Config),
+ TestDir = filename:join(DataDir, "groups_1"),
+ TestSpec = [{groups,TestDir,groups_11_SUITE, [{test_group_2, [parallel], [{test_group_3, [sequence]}]},
+ {test_group_9, [sequence], [{test_group_8, [parallel]}]}]}],
+ setup_and_execute(multi_group_config, TestSpec, Config).
+
skip_group(Config) when is_list(Config) ->
DataDir = ?config(data_dir, Config),
@@ -224,7 +255,6 @@ skip_group(Config) when is_list(Config) ->
setup_and_execute(skip_group, TestSpec, Config).
-
%%%-----------------------------------------------------------------
%%%
@@ -753,7 +783,7 @@ test_events(all_groups) ->
[
{?eh,start_logging,'_'},
{?eh,tc_start,{groups_11_SUITE,init_per_suite}},
- {?eh,test_stats,{12,0,{0,0}}},
+ {?eh,test_stats,{16,0,{0,0}}},
{?eh,tc_done,{groups_11_SUITE,end_per_suite,'_'}},
{negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}}
];
@@ -761,7 +791,7 @@ test_events(all_groups) ->
test_events(skip_all_groups) ->
[
{?eh,start_logging,'_'},
- {?eh,start_info,{1,1,12}},
+ {?eh,start_info,{1,1,16}},
{?eh,tc_start,{groups_11_SUITE,init_per_suite}},
{?eh,tc_user_skip,{groups_11_SUITE,{init_per_group,test_group_1a},"SKIPPED!"}},
{?eh,tc_user_skip,{groups_11_SUITE,{testcase_1a,test_group_1a},"SKIPPED!"}},
@@ -775,26 +805,38 @@ test_events(skip_all_groups) ->
{?eh,tc_user_skip,{groups_11_SUITE,{testcase_1b,test_group_1b},"SKIPPED!"}},
{?eh,test_stats,{0,0,{4,0}}},
{?eh,tc_user_skip,{groups_11_SUITE,{end_per_group,test_group_1b},"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_11_SUITE,{init_per_group,test_group_1c},"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_11_SUITE,{testcase_1a,test_group_1c},"SKIPPED!"}},
+ {?eh,test_stats,{0,0,{5,0}}},
+ {?eh,tc_user_skip,{groups_11_SUITE,{testcase_1b,test_group_1c},"SKIPPED!"}},
+ {?eh,test_stats,{0,0,{6,0}}},
+ {?eh,tc_user_skip,{groups_11_SUITE,{end_per_group,test_group_1c},"SKIPPED!"}},
{?eh,tc_user_skip,{groups_11_SUITE,{init_per_group,test_group_2},"SKIPPED!"}},
{?eh,tc_user_skip,{groups_11_SUITE,{testcase_2a,test_group_2},"SKIPPED!"}},
- {?eh,test_stats,{0,0,{5,0}}},
+ {?eh,test_stats,{0,0,{7,0}}},
{?eh,tc_user_skip,{groups_11_SUITE,{testcase_3a,test_group_3},"SKIPPED!"}},
- {?eh,test_stats,{0,0,{6,0}}},
+ {?eh,test_stats,{0,0,{8,0}}},
{?eh,tc_user_skip,{groups_11_SUITE,{testcase_3b,test_group_3},"SKIPPED!"}},
- {?eh,test_stats,{0,0,{7,0}}},
+ {?eh,test_stats,{0,0,{9,0}}},
{?eh,tc_user_skip,{groups_11_SUITE,{testcase_2b,test_group_2},"SKIPPED!"}},
- {?eh,test_stats,{0,0,{8,0}}},
+ {?eh,test_stats,{0,0,{10,0}}},
{?eh,tc_user_skip,{groups_11_SUITE,{end_per_group,test_group_2},"SKIPPED!"}},
{?eh,tc_user_skip,{groups_11_SUITE,{init_per_group,test_group_4},"SKIPPED!"}},
{?eh,tc_user_skip,{groups_11_SUITE,{testcase_5a,test_group_5},"SKIPPED!"}},
- {?eh,test_stats,{0,0,{9,0}}},
+ {?eh,test_stats,{0,0,{11,0}}},
{?eh,tc_user_skip,{groups_11_SUITE,{testcase_7a,test_group_7},"SKIPPED!"}},
- {?eh,test_stats,{0,0,{10,0}}},
+ {?eh,test_stats,{0,0,{12,0}}},
{?eh,tc_user_skip,{groups_11_SUITE,{testcase_7b,test_group_7},"SKIPPED!"}},
- {?eh,test_stats,{0,0,{11,0}}},
+ {?eh,test_stats,{0,0,{13,0}}},
{?eh,tc_user_skip,{groups_11_SUITE,{testcase_5b,test_group_5},"SKIPPED!"}},
- {?eh,test_stats,{0,0,{12,0}}},
+ {?eh,test_stats,{0,0,{14,0}}},
{?eh,tc_user_skip,{groups_11_SUITE,{end_per_group,test_group_4},"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_11_SUITE,{init_per_group,test_group_9},"SKIPPED!"}},
+ {?eh,tc_user_skip,{groups_11_SUITE,{testcase_9,test_group_9},"SKIPPED!"}},
+ {?eh,test_stats,{0,0,{15,0}}},
+ {?eh,tc_user_skip,{groups_11_SUITE,{testcase_8,test_group_8},"SKIPPED!"}},
+ {?eh,test_stats,{0,0,{16,0}}},
+ {?eh,tc_user_skip,{groups_11_SUITE,{end_per_group,test_group_9},"SKIPPED!"}},
{?eh,tc_start,{groups_11_SUITE,end_per_suite}},
{?eh,tc_done,{groups_11_SUITE,end_per_suite,ok}},
{negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}}
@@ -813,6 +855,29 @@ test_events(group) ->
{negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}}
];
+test_events(group_path) ->
+ [
+ {?eh,start_logging,'_'},
+ {?eh,tc_start,{groups_11_SUITE,init_per_suite}},
+ {?eh,tc_start,{groups_11_SUITE,{init_per_group,test_group_2,[sequence]}}},
+ {?eh,tc_start,{groups_11_SUITE,{init_per_group,test_group_3,[parallel]}}},
+ {?eh,tc_done,{groups_11_SUITE,{end_per_group,test_group_3,[parallel]},'_'}},
+ {?eh,tc_done,{groups_11_SUITE,{end_per_group,test_group_2,[sequence]},'_'}},
+ {?eh,tc_done,{groups_11_SUITE,end_per_suite,'_'}},
+ {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}}
+ ];
+
+test_events(group_config) ->
+ [
+ {?eh,start_logging,'_'},
+ {?eh,tc_start,{groups_11_SUITE,init_per_suite}},
+ {?eh,tc_start,{groups_11_SUITE,{init_per_group,test_group_1c,[parallel]}}},
+ {?eh,tc_done,{groups_11_SUITE,{end_per_group,test_group_1c,[parallel]},'_'}},
+ {?eh,tc_done,{groups_11_SUITE,end_per_suite,'_'}},
+ {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}}
+ ];
+
+
test_events(skip_group) ->
[
{?eh,start_logging,'_'},
@@ -858,6 +923,104 @@ test_events(group_all_testcases) ->
{negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}}
];
+test_events(group_spec) ->
+ [
+ {?eh,start_logging,'_'},
+ {?eh,tc_start,{groups_11_SUITE,init_per_suite}},
+ {parallel, [
+ {?eh,tc_start,{groups_11_SUITE,{init_per_group,test_group_2,[parallel]}}},
+ {?eh,tc_done,{groups_11_SUITE,{init_per_group,test_group_2,[parallel]},ok}},
+ {?eh,tc_start,{groups_11_SUITE,testcase_2a}},
+ {?eh,tc_done,{groups_11_SUITE,testcase_2a,ok}},
+ [
+ {?eh,tc_start,{groups_11_SUITE,{init_per_group,test_group_3,[sequence]}}},
+ {?eh,tc_start,{groups_11_SUITE,testcase_3a}},
+ {?eh,tc_start,{groups_11_SUITE,testcase_3b}},
+ {?eh,test_stats,{3,0,{0,0}}},
+ {?eh,tc_done,{groups_11_SUITE,{end_per_group,test_group_3,[sequence]},'_'}}
+ ],
+ {?eh,tc_start,{groups_11_SUITE,testcase_2b}},
+ {?eh,tc_done,{groups_11_SUITE,testcase_2b,ok}},
+ {?eh,test_stats,{4,0,{0,0}}},
+ {?eh,tc_start,{groups_11_SUITE,{end_per_group,test_group_2,[parallel]}}},
+ {?eh,tc_done,{groups_11_SUITE,{end_per_group,test_group_2,[parallel]},ok}}
+ ]},
+ {?eh,tc_done,{groups_11_SUITE,end_per_suite,'_'}},
+ {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}}
+ ];
+
+test_events(multi_group) ->
+ [
+ {?eh,start_logging,'_'},
+ {?eh,tc_start,{groups_11_SUITE,init_per_suite}},
+ {?eh,tc_start,{groups_11_SUITE,{init_per_group,test_group_1a,[]}}},
+ {?eh,tc_start,{groups_11_SUITE,testcase_1a}},
+ {?eh,tc_start,{groups_11_SUITE,testcase_1b}},
+ {?eh,test_stats,{2,0,{0,0}}},
+ {?eh,tc_done,{groups_11_SUITE,{end_per_group,test_group_1a,[]},'_'}},
+ {?eh,tc_start,{groups_11_SUITE,{init_per_group,test_group_2,[sequence]}}},
+ {parallel, [
+ {?eh,tc_start,{groups_11_SUITE,{init_per_group,test_group_3,[parallel]}}},
+ {?eh,tc_done,{groups_11_SUITE,{init_per_group,test_group_3,[parallel]}, ok}},
+ {?eh,tc_start,{groups_11_SUITE,testcase_3a}},
+ {?eh,tc_done,{groups_11_SUITE,testcase_3a,ok}},
+ {?eh,tc_start,{groups_11_SUITE,testcase_3b}},
+ {?eh,tc_done,{groups_11_SUITE,testcase_3b,ok}},
+ {?eh,test_stats,{4,0,{0,0}}},
+ {?eh,tc_start,{groups_11_SUITE,{end_per_group,test_group_3,[parallel]}}},
+ {?eh,tc_done,{groups_11_SUITE,{end_per_group,test_group_3,[parallel]}, ok}}
+ ]},
+ {?eh,tc_done,{groups_11_SUITE,{end_per_group,test_group_2,[sequence]},'_'}},
+ {?eh,tc_done,{groups_11_SUITE,end_per_suite,'_'}},
+ {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}}
+ ];
+
+test_events(multi_group_config) ->
+ [
+ {?eh,start_logging,'_'},
+ {?eh,tc_start,{groups_11_SUITE,init_per_suite}},
+ {parallel, [
+ {?eh,tc_start,{groups_11_SUITE,{init_per_group,test_group_2,[parallel]}}},
+ {?eh,tc_done,{groups_11_SUITE,{init_per_group,test_group_2,[parallel]},ok}},
+ {?eh,tc_start,{groups_11_SUITE,testcase_2a}},
+ {?eh,tc_done,{groups_11_SUITE,testcase_2a,ok}},
+ [
+ {?eh,tc_start,{groups_11_SUITE,{init_per_group,test_group_3,[sequence]}}},
+ {?eh,tc_start,{groups_11_SUITE,testcase_3a}},
+ {?eh,tc_start,{groups_11_SUITE,testcase_3b}},
+ {?eh,test_stats,{3,0,{0,0}}},
+ {?eh,tc_done,{groups_11_SUITE,{end_per_group,test_group_3,[sequence]},'_'}}
+ ],
+ {?eh,tc_start,{groups_11_SUITE,testcase_2b}},
+ {?eh,tc_done,{groups_11_SUITE,testcase_2b,ok}},
+ {?eh,test_stats,{4,0,{0,0}}},
+ {?eh,tc_start,{groups_11_SUITE,{end_per_group,test_group_2,[parallel]}}},
+ {?eh,tc_done,{groups_11_SUITE,{end_per_group,test_group_2,[parallel]},ok}}
+ ]},
+ [
+ {?eh,tc_start,{groups_11_SUITE,{init_per_group,test_group_9,[sequence]}}},
+ {?eh,tc_done,{groups_11_SUITE,{init_per_group,test_group_9,[sequence]},ok}},
+ {?eh,tc_start,{groups_11_SUITE,testcase_9}},
+ {?eh,tc_done,{groups_11_SUITE,testcase_9,ok}},
+ {?eh,test_stats,{5,0,{0,0}}},
+ {parallel,[
+ {?eh,tc_start,
+ {groups_11_SUITE,{init_per_group,test_group_8,[parallel]}}},
+ {?eh,tc_done,{groups_11_SUITE,{init_per_group,test_group_8,[parallel]},ok}},
+ {?eh,tc_start,{groups_11_SUITE,testcase_8}},
+ {?eh,tc_done,{groups_11_SUITE,testcase_8,ok}},
+ {?eh,test_stats,{6,0,{0,0}}},
+ {?eh,tc_start,{groups_11_SUITE,{end_per_group,test_group_8,[parallel]}}},
+ {?eh,tc_done,{groups_11_SUITE,{end_per_group,test_group_8,[parallel]},ok}}
+ ]},
+ {?eh,tc_start,{groups_11_SUITE,{end_per_group,test_group_9,[sequence]}}},
+ {?eh,tc_done,{groups_11_SUITE,{end_per_group,test_group_9,[sequence]},ok}}
+ ],
+ {?eh,tc_done,{groups_11_SUITE,end_per_suite,'_'}},
+ {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}}
+ ];
+
+
test_events(skip_group_all_testcases) ->
[
{?eh,start_logging,'_'},
diff --git a/lib/common_test/test/ct_testspec_1_SUITE_data/groups_1/groups_11_SUITE.erl b/lib/common_test/test/ct_testspec_1_SUITE_data/groups_1/groups_11_SUITE.erl
index 4d481fe3b8..56c97d9782 100644
--- a/lib/common_test/test/ct_testspec_1_SUITE_data/groups_1/groups_11_SUITE.erl
+++ b/lib/common_test/test/ct_testspec_1_SUITE_data/groups_1/groups_11_SUITE.erl
@@ -36,9 +36,11 @@ groups() ->
{test_group_1b, [], [testcase_1a,testcase_1b]},
- {test_group_2, [], [testcase_2a,
+ {test_group_1c, [sequence], [testcase_1a,testcase_1b]},
- {test_group_3, [], [testcase_3a,
+ {test_group_2, [sequence], [testcase_2a,
+
+ {test_group_3, [parallel], [testcase_3a,
testcase_3b]},
testcase_2b]},
@@ -49,7 +51,12 @@ groups() ->
testcase_5b]}]},
{test_group_6, [{group, test_group_7}]},
- {test_group_7, [testcase_7a,testcase_7b]}
+ {test_group_7, [testcase_7a,testcase_7b]},
+
+ {test_group_8, [sequence], [testcase_8]},
+
+ {test_group_9, [parallel], [testcase_9, {group, test_group_8}]}
+
].
all() ->
@@ -64,10 +71,11 @@ all() ->
%% this func only for internal test purposes
grs_and_tcs() ->
{[
- test_group_1a, test_group_1b,
+ test_group_1a, test_group_1b, test_group_1c,
test_group_2, test_group_3,
test_group_4, test_group_5,
- test_group_6, test_group_7
+ test_group_6, test_group_7,
+ test_group_8, test_group_9
],
[
testcase_1,
@@ -77,7 +85,9 @@ grs_and_tcs() ->
testcase_3a, testcase_3b,
testcase_3,
testcase_5a, testcase_5b,
- testcase_7a, testcase_7b
+ testcase_7a, testcase_7b,
+ testcase_8,
+ testcase_9
]}.
%%--------------------------------------------------------------------
@@ -95,7 +105,10 @@ end_per_suite(Config) ->
%%--------------------------------------------------------------------
init_per_group(Group, Config) ->
- [{name,Group}] = ?config(tc_group_properties,Config),
+ Group = case ?config(tc_group_properties,Config) of
+ [{name, Group0}] -> Group0;
+ [{name, Group0}, _Props] -> Group0
+ end,
{Grs,_} = grs_and_tcs(),
case lists:member(Group, Grs) of
true ->
@@ -164,8 +177,11 @@ testcase_1a(Config) ->
_ ->
case ?config(test_group_1b,Config) of
test_group_1b -> ok;
+ _ -> case ?config(test_group_1c,Config) of
+ test_group_1c -> ok;
_ -> ct:fail(no_group_data)
end
+ end
end,
testcase_1a = ?config(testcase_1a,Config),
ok.
@@ -178,8 +194,11 @@ testcase_1b(Config) ->
_ ->
case ?config(test_group_1b,Config) of
test_group_1b -> ok;
+ _ -> case ?config(test_group_1c,Config) of
+ test_group_1c -> ok;
_ -> ct:fail(no_group_data)
end
+ end
end,
undefined = ?config(testcase_1a,Config),
testcase_1b = ?config(testcase_1b,Config),
@@ -191,6 +210,7 @@ testcase_2(Config) ->
init = ?config(suite,Config),
undefined = ?config(test_group_1a,Config),
undefined = ?config(test_group_1b,Config),
+ undefined = ?config(test_group_1c, Config),
testcase_2 = ?config(testcase_2,Config),
ok.
@@ -280,3 +300,19 @@ testcase_7b(Config) ->
undefined = ?config(testcase_7a,Config),
testcase_7b = ?config(testcase_7b,Config),
ok.
+testcase_8() ->
+ [].
+testcase_8(Config) ->
+ init = ?config(suite,Config),
+ test_group_9 = ?config(test_group_9,Config),
+ test_group_8 = ?config(test_group_8,Config),
+ testcase_8 = ?config(testcase_8,Config),
+ ok.
+testcase_9() ->
+ [].
+testcase_9(Config) ->
+ init = ?config(suite,Config),
+ test_group_9 = ?config(test_group_9,Config),
+ undefined = ?config(test_group_8,Config),
+ testcase_9 = ?config(testcase_9,Config),
+ ok.
--
2.31.1
