From 4e5194cacae7f568dac25229bffd0b7a9434f4a3 Mon Sep 17 00:00:00 2001 From: Jon Anderson Date: Thu, 7 Aug 2014 23:18:44 -0400 Subject: [PATCH 1/3] Add support for test group annotations to riak_tests. - Tests can be annotated with one or more '-test_type(name)' attributes. - Tests can be declared to be in multiple groups - These types can be run in groups via the riak_test command line --groups (-g) option - Multiple groups can be specified by comma delimiting. --- src/riak_test_escript.erl | 63 +++++++++++++++++++++++++++++++-------- tests/bucket_types.erl | 2 ++ tests/ensemble_basic.erl | 2 ++ 3 files changed, 55 insertions(+), 12 deletions(-) diff --git a/src/riak_test_escript.erl b/src/riak_test_escript.erl index 9e92ff81c..7bc4d3a6f 100644 --- a/src/riak_test_escript.erl +++ b/src/riak_test_escript.erl @@ -25,17 +25,26 @@ -export([add_deps/1]). main(Args) -> - {ParsedArgs, HarnessArgs, Tests} = prepare(Args), - OutDir = proplists:get_value(outdir, ParsedArgs), - Results = execute(Tests, OutDir, report(ParsedArgs), HarnessArgs), - finalize(Results, ParsedArgs). + case prepare(Args) of + ok -> + ok; + {ParsedArgs, HarnessArgs, Tests} -> + OutDir = proplists:get_value(outdir, ParsedArgs), + Results = execute(Tests, OutDir, report(ParsedArgs), HarnessArgs), + finalize(Results, ParsedArgs) + end. prepare(Args) -> - {ParsedArgs, _, Tests} = ParseResults = parse_args(Args), - io:format("Tests to run: ~p~n", [Tests]), - ok = erlang_setup(ParsedArgs), - ok = test_setup(ParsedArgs), - ParseResults. + case parse_args(Args) of + {ParsedArgs, _, Tests} = ParseResults -> + io:format("Tests to run: ~p~n", [Tests]), + ok = erlang_setup(ParsedArgs), + ok = test_setup(ParsedArgs), + ParseResults; + ok -> + io:format("No tests to run.~n"), + ok + end. execute(Tests, Outdir, Report, HarnessArgs) -> TestCount = length(Tests), @@ -66,6 +75,7 @@ cli_options() -> {config, $c, "conf", string, "specifies the project configuration"}, {tests, $t, "tests", string, "specifies which tests to run"}, {suites, $s, "suites", string, "which suites to run"}, + {groups, $g, "groups", string, "specifiy a list of test groups to run"}, {dir, $d, "dir", string, "run all tests in the specified directory"}, {skip, $x, "skip", string, "list of tests to skip in a directory"}, {verbose, $v, "verbose", undefined, "verbose output"}, @@ -130,7 +140,9 @@ help_or_parse_tests(ParsedArgs, HarnessArgs, false) -> %% test metadata load_initial_config(ParsedArgs), - TestData = compose_test_data(ParsedArgs), + Groups = proplists:get_all_values(groups, ParsedArgs), + TestData = load_tests(Groups, ParsedArgs), + Tests = which_tests_to_run(report(ParsedArgs), TestData), Offset = rt_config:get(offset, undefined), Workers = rt_config:get(workers, undefined), @@ -231,7 +243,7 @@ maybe_teardown(true, TestResults, Coverage, Verbose) -> end, ok. -compose_test_data(ParsedArgs) -> +load_tests([], ParsedArgs) -> RawTestList = proplists:get_all_values(tests, ParsedArgs), TestList = lists:foldl(fun(X, Acc) -> string:tokens(X, ", ") ++ Acc end, [], RawTestList), %% Parse Command Line Tests @@ -246,8 +258,33 @@ compose_test_data(ParsedArgs) -> Dirs = proplists:get_all_values(dir, ParsedArgs), SkipTests = string:tokens(proplists:get_value(skip, ParsedArgs, []), [$,]), DirTests = lists:append([load_tests_in_dir(Dir, SkipTests) || Dir <- Dirs]), - Project = list_to_binary(rt_config:get(rt_project, "undefined")), + compose_test_data(DirTests, SpecificTests, ParsedArgs); +load_tests(RawGroupList, ParsedArgs) -> + Groups = lists:foldl(fun(X, Acc) -> string:tokens(X, ", ") ++ Acc end, [], RawGroupList), + Dirs = proplists:get_value(dir, ParsedArgs, ["./ebin"]), + AllDirTests = lists:append([load_tests_in_dir(Dir, []) || Dir <- Dirs]), + DirTests = get_group_tests(AllDirTests, Groups), + compose_test_data(DirTests, [], ParsedArgs). + +get_group_tests(Tests, Groups) -> + lists:filter(fun(Test) -> + Mod = list_to_atom(Test), + Attrs = Mod:module_info(attributes), + match_group_attributes(Attrs, Groups) + end, Tests). + +match_group_attributes(Attributes, Groups) -> + case proplists:get_all_values(test_type, Attributes) of + undefined -> + false; + TestTypes -> + lists:member(true, + [ hd(TestType) == list_to_atom(Group) + || Group <- Groups, TestType <- TestTypes ]) + end. +compose_test_data(DirTests, SpecificTests, ParsedArgs) -> + Project = list_to_binary(rt_config:get(rt_project, "undefined")), Backends = case proplists:get_all_values(backend, ParsedArgs) of [] -> [undefined]; Other -> Other @@ -259,6 +296,7 @@ compose_test_data(ParsedArgs) -> TestFoldFun = test_data_fun(rt:get_version(), Project, Backends, Upgrades), lists:foldl(TestFoldFun, [], lists:usort(DirTests ++ SpecificTests)). + test_data_fun(Version, Project, Backends, Upgrades) -> fun(Test, Tests) -> [{list_to_atom(Test), @@ -492,3 +530,4 @@ so_kill_riak_maybe() -> io:format("Leaving Riak Up... "), rt:whats_up() end. + diff --git a/tests/bucket_types.erl b/tests/bucket_types.erl index 4f05ea19c..54af6f416 100644 --- a/tests/bucket_types.erl +++ b/tests/bucket_types.erl @@ -5,6 +5,8 @@ -include_lib("eunit/include/eunit.hrl"). -include("rt.hrl"). +-test_type(bucket_types). + properties() -> CustomConfig = rt_cluster:augment_config(riak_core, {default_bucket_props, [{n_val, 2}]}, diff --git a/tests/ensemble_basic.erl b/tests/ensemble_basic.erl index 5e2b308d2..6c7a91be4 100644 --- a/tests/ensemble_basic.erl +++ b/tests/ensemble_basic.erl @@ -22,6 +22,8 @@ -export([confirm/0]). -include_lib("eunit/include/eunit.hrl"). +-test_type(ensemble). + confirm() -> NumNodes = 5, NVal = 5, From 846fa6e45ee2b60b082b5d9f92a427dad5eeeb79 Mon Sep 17 00:00:00 2001 From: Jon Anderson Date: Fri, 8 Aug 2014 11:51:32 -0400 Subject: [PATCH 2/3] Add support for multiple test_type attributes by changing format to take lists. - For example, in a riak_test: -test_type([bucket_types, http]). --- src/riak_test_escript.erl | 4 ++-- tests/bucket_types.erl | 2 +- tests/ensemble_basic.erl | 2 +- tests/http_bucket_types.erl | 2 ++ 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/riak_test_escript.erl b/src/riak_test_escript.erl index 7bc4d3a6f..d2851edb7 100644 --- a/src/riak_test_escript.erl +++ b/src/riak_test_escript.erl @@ -274,12 +274,12 @@ get_group_tests(Tests, Groups) -> end, Tests). match_group_attributes(Attributes, Groups) -> - case proplists:get_all_values(test_type, Attributes) of + case proplists:get_value(test_type, Attributes) of undefined -> false; TestTypes -> lists:member(true, - [ hd(TestType) == list_to_atom(Group) + [ TestType == list_to_atom(Group) || Group <- Groups, TestType <- TestTypes ]) end. diff --git a/tests/bucket_types.erl b/tests/bucket_types.erl index 54af6f416..95290cfd5 100644 --- a/tests/bucket_types.erl +++ b/tests/bucket_types.erl @@ -5,7 +5,7 @@ -include_lib("eunit/include/eunit.hrl"). -include("rt.hrl"). --test_type(bucket_types). +-test_type([bucket_types]). properties() -> CustomConfig = rt_cluster:augment_config(riak_core, diff --git a/tests/ensemble_basic.erl b/tests/ensemble_basic.erl index 6c7a91be4..1f2d71e04 100644 --- a/tests/ensemble_basic.erl +++ b/tests/ensemble_basic.erl @@ -22,7 +22,7 @@ -export([confirm/0]). -include_lib("eunit/include/eunit.hrl"). --test_type(ensemble). +-test_type([ensemble]). confirm() -> NumNodes = 5, diff --git a/tests/http_bucket_types.erl b/tests/http_bucket_types.erl index 290e40d74..809e018c9 100644 --- a/tests/http_bucket_types.erl +++ b/tests/http_bucket_types.erl @@ -6,6 +6,8 @@ -include_lib("riakc/include/riakc.hrl"). -include("rt.hrl"). +-test_type([bucket_types, http]). + properties() -> CustomConfig = rt_cluster:augment_config(riak_core, {default_bucket_props, [{n_val, 2}]}, From 5ea57871ad705c519fb1d107b690829d8f8c46a5 Mon Sep 17 00:00:00 2001 From: Jon Anderson Date: Fri, 8 Aug 2014 17:02:39 -0400 Subject: [PATCH 3/3] Address review feedback. - Options --groups and --tests are mutually exclusive for now. - fix indentation for overly long lines. --- src/riak_test_escript.erl | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/src/riak_test_escript.erl b/src/riak_test_escript.erl index d2851edb7..f48ab8acf 100644 --- a/src/riak_test_escript.erl +++ b/src/riak_test_escript.erl @@ -129,13 +129,18 @@ parse_args(Args) -> help_or_parse_args({ok, {[], _}}) -> print_help(); help_or_parse_args({ok, {ParsedArgs, HarnessArgs}}) -> - help_or_parse_tests(ParsedArgs, HarnessArgs, lists:member(help, ParsedArgs)); + help_or_parse_tests(ParsedArgs, + HarnessArgs, + lists:member(help, ParsedArgs), + args_invalid(ParsedArgs)); help_or_parse_args(_) -> print_help(). -help_or_parse_tests(_, _, true) -> +help_or_parse_tests(_, _, true, _) -> print_help(); -help_or_parse_tests(ParsedArgs, HarnessArgs, false) -> +help_or_parse_tests(_, _, false, true) -> + print_help(); +help_or_parse_tests(ParsedArgs, HarnessArgs, false, false) -> %% Have to load the `riak_test' config prior to assembling the %% test metadata load_initial_config(ParsedArgs), @@ -148,6 +153,16 @@ help_or_parse_tests(ParsedArgs, HarnessArgs, false) -> Workers = rt_config:get(workers, undefined), shuffle_tests(ParsedArgs, HarnessArgs, Tests, Offset, Workers). +args_invalid(ParsedArgs) -> + case { proplists:is_defined(groups, ParsedArgs), + proplists:is_defined(tests, ParsedArgs) } of + {true, true} -> + io:format("--groups and --tests are currently mutually exclusive.~n~n"), + true; + {_, _} -> + false + end. + load_initial_config(ParsedArgs) -> %% Loads application defaults application:load(riak_test), @@ -245,7 +260,9 @@ maybe_teardown(true, TestResults, Coverage, Verbose) -> load_tests([], ParsedArgs) -> RawTestList = proplists:get_all_values(tests, ParsedArgs), - TestList = lists:foldl(fun(X, Acc) -> string:tokens(X, ", ") ++ Acc end, [], RawTestList), + TestList = lists:foldl(fun(X, Acc) -> + string:tokens(X, ", ") ++ Acc + end, [], RawTestList), %% Parse Command Line Tests {CodePaths, SpecificTests} = lists:foldl(fun extract_test_names/2, @@ -260,7 +277,9 @@ load_tests([], ParsedArgs) -> DirTests = lists:append([load_tests_in_dir(Dir, SkipTests) || Dir <- Dirs]), compose_test_data(DirTests, SpecificTests, ParsedArgs); load_tests(RawGroupList, ParsedArgs) -> - Groups = lists:foldl(fun(X, Acc) -> string:tokens(X, ", ") ++ Acc end, [], RawGroupList), + Groups = lists:foldl(fun(X, Acc) -> + string:tokens(X, ", ") ++ Acc + end, [], RawGroupList), Dirs = proplists:get_value(dir, ParsedArgs, ["./ebin"]), AllDirTests = lists:append([load_tests_in_dir(Dir, []) || Dir <- Dirs]), DirTests = get_group_tests(AllDirTests, Groups),