Sign Up
Log In
Log In
or
Sign Up
Places
All Projects
Status Monitor
Collapse sidebar
home:Ledest:erlang:19
erlang
1285-Update-the-doc-CT-test_spec-grammar-to-mat...
Overview
Repositories
Revisions
Requests
Users
Attributes
Meta
File 1285-Update-the-doc-CT-test_spec-grammar-to-match-actual-.patch of Package erlang
From da1224651c206e4ef33a1575fb37ff0877ab4847 Mon Sep 17 00:00:00 2001 From: acw224 <thomasdcuvillier@gmail.com> Date: Tue, 3 Aug 2021 08:26:13 -0700 Subject: [PATCH 1/2] Update the doc CT test_spec grammar to match actual behavior The grammar in the doc doesn't match actual behavior Two problems: Groups = GroupPath, GroupPath = [GroupName] leads to think that [group1, group2] is the group path group1-group2 whereas it is understood as an array of GroupSpec. GroupSpec = [group, properties, GroupSpec] was not recognised, only [group, properties, [GroupSpec] An updated grammar is proposed, together with additional tests to validate the pertinence of the new grammar. --- lib/common_test/doc/src/run_test_chapter.xml | 8 +- lib/common_test/test/ct_testspec_1_SUITE.erl | 189 ++++++++++++++++-- .../groups_1/groups_11_SUITE.erl | 50 ++++- 3 files changed, 222 insertions(+), 25 deletions(-) diff --git a/lib/common_test/doc/src/run_test_chapter.xml b/lib/common_test/doc/src/run_test_chapter.xml index 0b8657ced3..1d88abc27e 100644 --- a/lib/common_test/doc/src/run_test_chapter.xml +++ b/lib/common_test/doc/src/run_test_chapter.xml @@ -955,9 +955,9 @@ Dir = string() Suites = atom() | [atom()] | all Suite = atom() - Groups = GroupPath | [GroupPath] | GroupSpec | [GroupSpec] | all - GroupPath = [GroupName] - GroupSpec = GroupName | {GroupName,Properties} | {GroupName,Properties,GroupSpec} + Groups = GroupPath | GroupSpec | [GroupSpec] | all + GroupPath = [[GroupName]] + GroupSpec = GroupName | {GroupName,Properties} | {GroupName,Properties,[GroupSpec]} GroupName = atom() GroupNames = GroupName | [GroupName] Cases = atom() | [atom()] | all @@ -1608,5 +1608,3 @@ div.error pre { color:white }</pre> </section> </chapter> - - diff --git a/lib/common_test/test/ct_testspec_1_SUITE.erl b/lib/common_test/test/ct_testspec_1_SUITE.erl index 2d2c42999f..31ae19a033 100644 --- a/lib/common_test/test/ct_testspec_1_SUITE.erl +++ b/lib/common_test/test/ct_testspec_1_SUITE.erl @@ -63,6 +63,8 @@ all() -> [all_suites, skip_all_suites, suite, skip_suite, all_testcases, skip_all_testcases, testcase, skip_testcase, all_groups, skip_all_groups, group, + group_path, group_config, + group_spec, multi_group, multi_group_config, skip_group, group_all_testcases, skip_group_all_testcases, group_testcase, skip_group_testcase, topgroup, subgroup, skip_subgroup, @@ -207,12 +209,41 @@ skip_all_groups(Config) when is_list(Config) -> group(Config) when is_list(Config) -> DataDir = ?config(data_dir, Config), - TestDir = filename:join(DataDir, "groups_1"), TestSpec = [{groups,TestDir,groups_11_SUITE,test_group_1a}], - setup_and_execute(group, TestSpec, Config). +group_path(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + TestDir = filename:join(DataDir, "groups_1"), + TestSpec = [{groups,TestDir,groups_11_SUITE,[[test_group_2, test_group_3]]}], + setup_and_execute(group_path, TestSpec, Config). + +group_config(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + TestDir = filename:join(DataDir, "groups_1"), + TestSpec = [{groups,TestDir,groups_11_SUITE, {test_group_1c, [parallel]}}], + setup_and_execute(group_config, TestSpec, Config). + +group_spec(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + TestDir = filename:join(DataDir, "groups_1"), + TestSpec = [{groups,TestDir,groups_11_SUITE, {test_group_2, [parallel], [{test_group_3, [sequence]}]}}], + setup_and_execute(group_spec, TestSpec, Config). + +multi_group(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + TestDir = filename:join(DataDir, "groups_1"), + TestSpec = [{groups,TestDir,groups_11_SUITE, [test_group_1a, test_group_3]}], + setup_and_execute(multi_group, TestSpec, Config). + +multi_group_config(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + TestDir = filename:join(DataDir, "groups_1"), + TestSpec = [{groups,TestDir,groups_11_SUITE, [{test_group_2, [parallel], [{test_group_3, [sequence]}]}, + {test_group_9, [sequence], [{test_group_8, [parallel]}]}]}], + setup_and_execute(multi_group_config, TestSpec, Config). + skip_group(Config) when is_list(Config) -> DataDir = ?config(data_dir, Config), @@ -224,7 +255,6 @@ skip_group(Config) when is_list(Config) -> setup_and_execute(skip_group, TestSpec, Config). - %%%----------------------------------------------------------------- %%% @@ -753,7 +783,7 @@ test_events(all_groups) -> [ {?eh,start_logging,'_'}, {?eh,tc_start,{groups_11_SUITE,init_per_suite}}, - {?eh,test_stats,{12,0,{0,0}}}, + {?eh,test_stats,{16,0,{0,0}}}, {?eh,tc_done,{groups_11_SUITE,end_per_suite,'_'}}, {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} ]; @@ -761,7 +791,7 @@ test_events(all_groups) -> test_events(skip_all_groups) -> [ {?eh,start_logging,'_'}, - {?eh,start_info,{1,1,12}}, + {?eh,start_info,{1,1,16}}, {?eh,tc_start,{groups_11_SUITE,init_per_suite}}, {?eh,tc_user_skip,{groups_11_SUITE,{init_per_group,test_group_1a},"SKIPPED!"}}, {?eh,tc_user_skip,{groups_11_SUITE,{testcase_1a,test_group_1a},"SKIPPED!"}}, @@ -775,26 +805,38 @@ test_events(skip_all_groups) -> {?eh,tc_user_skip,{groups_11_SUITE,{testcase_1b,test_group_1b},"SKIPPED!"}}, {?eh,test_stats,{0,0,{4,0}}}, {?eh,tc_user_skip,{groups_11_SUITE,{end_per_group,test_group_1b},"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_11_SUITE,{init_per_group,test_group_1c},"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_11_SUITE,{testcase_1a,test_group_1c},"SKIPPED!"}}, + {?eh,test_stats,{0,0,{5,0}}}, + {?eh,tc_user_skip,{groups_11_SUITE,{testcase_1b,test_group_1c},"SKIPPED!"}}, + {?eh,test_stats,{0,0,{6,0}}}, + {?eh,tc_user_skip,{groups_11_SUITE,{end_per_group,test_group_1c},"SKIPPED!"}}, {?eh,tc_user_skip,{groups_11_SUITE,{init_per_group,test_group_2},"SKIPPED!"}}, {?eh,tc_user_skip,{groups_11_SUITE,{testcase_2a,test_group_2},"SKIPPED!"}}, - {?eh,test_stats,{0,0,{5,0}}}, + {?eh,test_stats,{0,0,{7,0}}}, {?eh,tc_user_skip,{groups_11_SUITE,{testcase_3a,test_group_3},"SKIPPED!"}}, - {?eh,test_stats,{0,0,{6,0}}}, + {?eh,test_stats,{0,0,{8,0}}}, {?eh,tc_user_skip,{groups_11_SUITE,{testcase_3b,test_group_3},"SKIPPED!"}}, - {?eh,test_stats,{0,0,{7,0}}}, + {?eh,test_stats,{0,0,{9,0}}}, {?eh,tc_user_skip,{groups_11_SUITE,{testcase_2b,test_group_2},"SKIPPED!"}}, - {?eh,test_stats,{0,0,{8,0}}}, + {?eh,test_stats,{0,0,{10,0}}}, {?eh,tc_user_skip,{groups_11_SUITE,{end_per_group,test_group_2},"SKIPPED!"}}, {?eh,tc_user_skip,{groups_11_SUITE,{init_per_group,test_group_4},"SKIPPED!"}}, {?eh,tc_user_skip,{groups_11_SUITE,{testcase_5a,test_group_5},"SKIPPED!"}}, - {?eh,test_stats,{0,0,{9,0}}}, + {?eh,test_stats,{0,0,{11,0}}}, {?eh,tc_user_skip,{groups_11_SUITE,{testcase_7a,test_group_7},"SKIPPED!"}}, - {?eh,test_stats,{0,0,{10,0}}}, + {?eh,test_stats,{0,0,{12,0}}}, {?eh,tc_user_skip,{groups_11_SUITE,{testcase_7b,test_group_7},"SKIPPED!"}}, - {?eh,test_stats,{0,0,{11,0}}}, + {?eh,test_stats,{0,0,{13,0}}}, {?eh,tc_user_skip,{groups_11_SUITE,{testcase_5b,test_group_5},"SKIPPED!"}}, - {?eh,test_stats,{0,0,{12,0}}}, + {?eh,test_stats,{0,0,{14,0}}}, {?eh,tc_user_skip,{groups_11_SUITE,{end_per_group,test_group_4},"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_11_SUITE,{init_per_group,test_group_9},"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_11_SUITE,{testcase_9,test_group_9},"SKIPPED!"}}, + {?eh,test_stats,{0,0,{15,0}}}, + {?eh,tc_user_skip,{groups_11_SUITE,{testcase_8,test_group_8},"SKIPPED!"}}, + {?eh,test_stats,{0,0,{16,0}}}, + {?eh,tc_user_skip,{groups_11_SUITE,{end_per_group,test_group_9},"SKIPPED!"}}, {?eh,tc_start,{groups_11_SUITE,end_per_suite}}, {?eh,tc_done,{groups_11_SUITE,end_per_suite,ok}}, {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} @@ -813,6 +855,29 @@ test_events(group) -> {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} ]; +test_events(group_path) -> + [ + {?eh,start_logging,'_'}, + {?eh,tc_start,{groups_11_SUITE,init_per_suite}}, + {?eh,tc_start,{groups_11_SUITE,{init_per_group,test_group_2,[sequence]}}}, + {?eh,tc_start,{groups_11_SUITE,{init_per_group,test_group_3,[parallel]}}}, + {?eh,tc_done,{groups_11_SUITE,{end_per_group,test_group_3,[parallel]},'_'}}, + {?eh,tc_done,{groups_11_SUITE,{end_per_group,test_group_2,[sequence]},'_'}}, + {?eh,tc_done,{groups_11_SUITE,end_per_suite,'_'}}, + {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} + ]; + +test_events(group_config) -> + [ + {?eh,start_logging,'_'}, + {?eh,tc_start,{groups_11_SUITE,init_per_suite}}, + {?eh,tc_start,{groups_11_SUITE,{init_per_group,test_group_1c,[parallel]}}}, + {?eh,tc_done,{groups_11_SUITE,{end_per_group,test_group_1c,[parallel]},'_'}}, + {?eh,tc_done,{groups_11_SUITE,end_per_suite,'_'}}, + {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} + ]; + + test_events(skip_group) -> [ {?eh,start_logging,'_'}, @@ -858,6 +923,104 @@ test_events(group_all_testcases) -> {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} ]; +test_events(group_spec) -> + [ + {?eh,start_logging,'_'}, + {?eh,tc_start,{groups_11_SUITE,init_per_suite}}, + {parallel, [ + {?eh,tc_start,{groups_11_SUITE,{init_per_group,test_group_2,[parallel]}}}, + {?eh,tc_done,{groups_11_SUITE,{init_per_group,test_group_2,[parallel]},ok}}, + {?eh,tc_start,{groups_11_SUITE,testcase_2a}}, + {?eh,tc_done,{groups_11_SUITE,testcase_2a,ok}}, + [ + {?eh,tc_start,{groups_11_SUITE,{init_per_group,test_group_3,[sequence]}}}, + {?eh,tc_start,{groups_11_SUITE,testcase_3a}}, + {?eh,tc_start,{groups_11_SUITE,testcase_3b}}, + {?eh,test_stats,{3,0,{0,0}}}, + {?eh,tc_done,{groups_11_SUITE,{end_per_group,test_group_3,[sequence]},'_'}} + ], + {?eh,tc_start,{groups_11_SUITE,testcase_2b}}, + {?eh,tc_done,{groups_11_SUITE,testcase_2b,ok}}, + {?eh,test_stats,{4,0,{0,0}}}, + {?eh,tc_start,{groups_11_SUITE,{end_per_group,test_group_2,[parallel]}}}, + {?eh,tc_done,{groups_11_SUITE,{end_per_group,test_group_2,[parallel]},ok}} + ]}, + {?eh,tc_done,{groups_11_SUITE,end_per_suite,'_'}}, + {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} + ]; + +test_events(multi_group) -> + [ + {?eh,start_logging,'_'}, + {?eh,tc_start,{groups_11_SUITE,init_per_suite}}, + {?eh,tc_start,{groups_11_SUITE,{init_per_group,test_group_1a,[]}}}, + {?eh,tc_start,{groups_11_SUITE,testcase_1a}}, + {?eh,tc_start,{groups_11_SUITE,testcase_1b}}, + {?eh,test_stats,{2,0,{0,0}}}, + {?eh,tc_done,{groups_11_SUITE,{end_per_group,test_group_1a,[]},'_'}}, + {?eh,tc_start,{groups_11_SUITE,{init_per_group,test_group_2,[sequence]}}}, + {parallel, [ + {?eh,tc_start,{groups_11_SUITE,{init_per_group,test_group_3,[parallel]}}}, + {?eh,tc_done,{groups_11_SUITE,{init_per_group,test_group_3,[parallel]}, ok}}, + {?eh,tc_start,{groups_11_SUITE,testcase_3a}}, + {?eh,tc_done,{groups_11_SUITE,testcase_3a,ok}}, + {?eh,tc_start,{groups_11_SUITE,testcase_3b}}, + {?eh,tc_done,{groups_11_SUITE,testcase_3b,ok}}, + {?eh,test_stats,{4,0,{0,0}}}, + {?eh,tc_start,{groups_11_SUITE,{end_per_group,test_group_3,[parallel]}}}, + {?eh,tc_done,{groups_11_SUITE,{end_per_group,test_group_3,[parallel]}, ok}} + ]}, + {?eh,tc_done,{groups_11_SUITE,{end_per_group,test_group_2,[sequence]},'_'}}, + {?eh,tc_done,{groups_11_SUITE,end_per_suite,'_'}}, + {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} + ]; + +test_events(multi_group_config) -> + [ + {?eh,start_logging,'_'}, + {?eh,tc_start,{groups_11_SUITE,init_per_suite}}, + {parallel, [ + {?eh,tc_start,{groups_11_SUITE,{init_per_group,test_group_2,[parallel]}}}, + {?eh,tc_done,{groups_11_SUITE,{init_per_group,test_group_2,[parallel]},ok}}, + {?eh,tc_start,{groups_11_SUITE,testcase_2a}}, + {?eh,tc_done,{groups_11_SUITE,testcase_2a,ok}}, + [ + {?eh,tc_start,{groups_11_SUITE,{init_per_group,test_group_3,[sequence]}}}, + {?eh,tc_start,{groups_11_SUITE,testcase_3a}}, + {?eh,tc_start,{groups_11_SUITE,testcase_3b}}, + {?eh,test_stats,{3,0,{0,0}}}, + {?eh,tc_done,{groups_11_SUITE,{end_per_group,test_group_3,[sequence]},'_'}} + ], + {?eh,tc_start,{groups_11_SUITE,testcase_2b}}, + {?eh,tc_done,{groups_11_SUITE,testcase_2b,ok}}, + {?eh,test_stats,{4,0,{0,0}}}, + {?eh,tc_start,{groups_11_SUITE,{end_per_group,test_group_2,[parallel]}}}, + {?eh,tc_done,{groups_11_SUITE,{end_per_group,test_group_2,[parallel]},ok}} + ]}, + [ + {?eh,tc_start,{groups_11_SUITE,{init_per_group,test_group_9,[sequence]}}}, + {?eh,tc_done,{groups_11_SUITE,{init_per_group,test_group_9,[sequence]},ok}}, + {?eh,tc_start,{groups_11_SUITE,testcase_9}}, + {?eh,tc_done,{groups_11_SUITE,testcase_9,ok}}, + {?eh,test_stats,{5,0,{0,0}}}, + {parallel,[ + {?eh,tc_start, + {groups_11_SUITE,{init_per_group,test_group_8,[parallel]}}}, + {?eh,tc_done,{groups_11_SUITE,{init_per_group,test_group_8,[parallel]},ok}}, + {?eh,tc_start,{groups_11_SUITE,testcase_8}}, + {?eh,tc_done,{groups_11_SUITE,testcase_8,ok}}, + {?eh,test_stats,{6,0,{0,0}}}, + {?eh,tc_start,{groups_11_SUITE,{end_per_group,test_group_8,[parallel]}}}, + {?eh,tc_done,{groups_11_SUITE,{end_per_group,test_group_8,[parallel]},ok}} + ]}, + {?eh,tc_start,{groups_11_SUITE,{end_per_group,test_group_9,[sequence]}}}, + {?eh,tc_done,{groups_11_SUITE,{end_per_group,test_group_9,[sequence]},ok}} + ], + {?eh,tc_done,{groups_11_SUITE,end_per_suite,'_'}}, + {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} + ]; + + test_events(skip_group_all_testcases) -> [ {?eh,start_logging,'_'}, diff --git a/lib/common_test/test/ct_testspec_1_SUITE_data/groups_1/groups_11_SUITE.erl b/lib/common_test/test/ct_testspec_1_SUITE_data/groups_1/groups_11_SUITE.erl index 4d481fe3b8..56c97d9782 100644 --- a/lib/common_test/test/ct_testspec_1_SUITE_data/groups_1/groups_11_SUITE.erl +++ b/lib/common_test/test/ct_testspec_1_SUITE_data/groups_1/groups_11_SUITE.erl @@ -36,9 +36,11 @@ groups() -> {test_group_1b, [], [testcase_1a,testcase_1b]}, - {test_group_2, [], [testcase_2a, + {test_group_1c, [sequence], [testcase_1a,testcase_1b]}, - {test_group_3, [], [testcase_3a, + {test_group_2, [sequence], [testcase_2a, + + {test_group_3, [parallel], [testcase_3a, testcase_3b]}, testcase_2b]}, @@ -49,7 +51,12 @@ groups() -> testcase_5b]}]}, {test_group_6, [{group, test_group_7}]}, - {test_group_7, [testcase_7a,testcase_7b]} + {test_group_7, [testcase_7a,testcase_7b]}, + + {test_group_8, [sequence], [testcase_8]}, + + {test_group_9, [parallel], [testcase_9, {group, test_group_8}]} + ]. all() -> @@ -64,10 +71,11 @@ all() -> %% this func only for internal test purposes grs_and_tcs() -> {[ - test_group_1a, test_group_1b, + test_group_1a, test_group_1b, test_group_1c, test_group_2, test_group_3, test_group_4, test_group_5, - test_group_6, test_group_7 + test_group_6, test_group_7, + test_group_8, test_group_9 ], [ testcase_1, @@ -77,7 +85,9 @@ grs_and_tcs() -> testcase_3a, testcase_3b, testcase_3, testcase_5a, testcase_5b, - testcase_7a, testcase_7b + testcase_7a, testcase_7b, + testcase_8, + testcase_9 ]}. %%-------------------------------------------------------------------- @@ -95,7 +105,10 @@ end_per_suite(Config) -> %%-------------------------------------------------------------------- init_per_group(Group, Config) -> - [{name,Group}] = ?config(tc_group_properties,Config), + Group = case ?config(tc_group_properties,Config) of + [{name, Group0}] -> Group0; + [{name, Group0}, _Props] -> Group0 + end, {Grs,_} = grs_and_tcs(), case lists:member(Group, Grs) of true -> @@ -164,8 +177,11 @@ testcase_1a(Config) -> _ -> case ?config(test_group_1b,Config) of test_group_1b -> ok; + _ -> case ?config(test_group_1c,Config) of + test_group_1c -> ok; _ -> ct:fail(no_group_data) end + end end, testcase_1a = ?config(testcase_1a,Config), ok. @@ -178,8 +194,11 @@ testcase_1b(Config) -> _ -> case ?config(test_group_1b,Config) of test_group_1b -> ok; + _ -> case ?config(test_group_1c,Config) of + test_group_1c -> ok; _ -> ct:fail(no_group_data) end + end end, undefined = ?config(testcase_1a,Config), testcase_1b = ?config(testcase_1b,Config), @@ -191,6 +210,7 @@ testcase_2(Config) -> init = ?config(suite,Config), undefined = ?config(test_group_1a,Config), undefined = ?config(test_group_1b,Config), + undefined = ?config(test_group_1c, Config), testcase_2 = ?config(testcase_2,Config), ok. @@ -280,3 +300,19 @@ testcase_7b(Config) -> undefined = ?config(testcase_7a,Config), testcase_7b = ?config(testcase_7b,Config), ok. +testcase_8() -> + []. +testcase_8(Config) -> + init = ?config(suite,Config), + test_group_9 = ?config(test_group_9,Config), + test_group_8 = ?config(test_group_8,Config), + testcase_8 = ?config(testcase_8,Config), + ok. +testcase_9() -> + []. +testcase_9(Config) -> + init = ?config(suite,Config), + test_group_9 = ?config(test_group_9,Config), + undefined = ?config(test_group_8,Config), + testcase_9 = ?config(testcase_9,Config), + ok. -- 2.31.1
Locations
Projects
Search
Status Monitor
Help
OpenBuildService.org
Documentation
API Documentation
Code of Conduct
Contact
Support
@OBShq
Terms
openSUSE Build Service is sponsored by
The Open Build Service is an
openSUSE project
.
Sign Up
Log In
Places
Places
All Projects
Status Monitor
