diff --git a/include/rt.hrl b/include/rt.hrl index a40996d61..78de7e028 100644 --- a/include/rt.hrl +++ b/include/rt.hrl @@ -23,18 +23,3 @@ url :: string(), headers=[] :: [{atom(), string()}] }). - --record(rt_properties, { - nodes :: [node()], - node_count=6 :: non_neg_integer(), - metadata=[] :: proplists:proplist(), - properties=[] :: proplists:proplist(), - rolling_upgrade=false :: boolean(), - start_version=current :: atom(), - upgrade_version=current :: atom(), - wait_for_transfers=false :: boolean(), - valid_backends=all :: all | [atom()], - make_cluster=true :: boolean(), - config :: term() - }). --type rt_properties() :: #rt_properties{}. diff --git a/src/riak_test_runner.erl b/src/riak_test_runner.erl index 54d39007f..d181177ae 100644 --- a/src/riak_test_runner.erl +++ b/src/riak_test_runner.erl @@ -139,22 +139,22 @@ test_fun({PropsMod, PropsFun}, {SetupMod, SetupFun}, ConfirmModFun, MetaData) -> Properties = PropsMod:PropsFun(), case SetupMod:SetupFun(Properties, MetaData) of {ok, SetupData} -> - lager:info("Wait for transfers? ~p", [SetupData#rt_properties.wait_for_transfers]), + RollingUpgradeTest = rt_properties:get(rolling_upgrade, SetupData), ConfirmFun = compose_confirm_fun(ConfirmModFun, SetupData, - MetaData), + MetaData, + RollingUpgradeTest), + ConfirmFun(); _ -> fail end end. -compose_confirm_fun({ConfirmMod, ConfirmFun}, - SetupData=#rt_properties{rolling_upgrade=true}, - MetaData) -> - Nodes = SetupData#rt_properties.nodes, - WaitForTransfers = SetupData#rt_properties.wait_for_transfers, - UpgradeVersion = SetupData#rt_properties.upgrade_version, +compose_confirm_fun({ConfirmMod, ConfirmFun}, SetupData, MetaData, true) -> + Nodes = rt_properties:get(nodes, SetupData), + WaitForTransfers = rt_properties:get(wait_for_transfers, SetupData), + UpgradeVersion = rt_properties:get(upgrade_version, SetupData), fun() -> InitialResult = ConfirmMod:ConfirmFun(SetupData, MetaData), OtherResults = [begin @@ -165,9 +165,7 @@ compose_confirm_fun({ConfirmMod, ConfirmFun}, end || Node <- Nodes], lists:all(fun(R) -> R =:= pass end, [InitialResult | OtherResults]) end; -compose_confirm_fun({ConfirmMod, ConfirmFun}, - SetupData=#rt_properties{rolling_upgrade=false}, - MetaData) -> +compose_confirm_fun({ConfirmMod, ConfirmFun}, SetupData, MetaData, false) -> fun() -> ConfirmMod:ConfirmFun(SetupData, MetaData) end. diff --git a/src/rt_cluster.erl b/src/rt_cluster.erl index 26470e122..cdd507c3e 100644 --- a/src/rt_cluster.erl +++ b/src/rt_cluster.erl @@ -39,6 +39,7 @@ try_nodes_ready/3, versions/0, teardown/0]). + -export([maybe_wait_for_transfers/2]). -include("rt.hrl"). @@ -47,27 +48,29 @@ %% @doc Default properties used if a riak_test module does not specify %% a custom properties function. --spec properties() -> rt_properties(). +-spec properties() -> rt_properties:properties(). properties() -> - #rt_properties{config=config()}. + rt_properties:new([{config, config()}]). --spec setup(rt_properties(), proplists:proplist()) -> - {ok, rt_properties()} | {error, term()}. +-spec setup(rt_properties:properties(), proplists:proplist()) -> + {ok, rt_properties:properties()} | {error, term()}. setup(Properties, MetaData) -> rt_config:set_conf(all, [{"buckets.default.allow_mult", "false"}]), - RollingUpgrade = proplists:get_value(rolling_upgrade, - MetaData, - Properties#rt_properties.rolling_upgrade), - Version = Properties#rt_properties.start_version, - Versions = [{Version, Properties#rt_properties.config} || - _ <- lists:seq(1, Properties#rt_properties.node_count)], - Nodes = deploy_or_build_cluster(Versions, Properties#rt_properties.make_cluster), - - maybe_wait_for_transfers(Nodes, Properties#rt_properties.wait_for_transfers), - UpdProperties = Properties#rt_properties{nodes=Nodes, - rolling_upgrade=RollingUpgrade}, - {ok, UpdProperties}. + RollingUpgrade = + proplists:get_value(rolling_upgrade, + MetaData, + rt_properties:get(rolling_upgrade, Properties)), + Version = rt_properties:get(start_version, Properties), + Config = rt_properties:get(config, Properties), + Versions = [{Version, Config} || + _ <- lists:seq(1, rt_properties:get(node_count, Properties))], + Nodes = deploy_or_build_cluster(Versions, + rt_properties:get(make_cluster, Properties)), + maybe_wait_for_transfers(Nodes, + rt_properties:get(wait_for_transfers, Properties)), + UpdPropertyList = [{nodes, Nodes}, {rolling_upgrade, RollingUpgrade}], + rt_properties:set(UpdPropertyList, Properties). deploy_or_build_cluster(Versions, true) -> build_cluster(Versions); @@ -218,7 +221,9 @@ config() -> {riak_pipe, [{worker_limit, 200}]}]. augment_config(Section, Property, Config) -> - UpdSectionConfig = update_section(Section, Property, lists:keyfind(Section, 1, Config)), + UpdSectionConfig = update_section(Section, + Property, + lists:keyfind(Section, 1, Config)), lists:keyreplace(Section, 1, Config, UpdSectionConfig). update_section(Section, Property, false) -> diff --git a/src/rt_properties.erl b/src/rt_properties.erl new file mode 100644 index 000000000..a484d0224 --- /dev/null +++ b/src/rt_properties.erl @@ -0,0 +1,195 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2013 Basho Technologies, Inc. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% ------------------------------------------------------------------- + +%% @doc Implements a set of functions for accessing and manipulating +%% an `rt_properties' record.-module(rt_properties). + +-module(rt_properties). + +-include("rt.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-record(rt_properties_v1, { + nodes :: [node()], + node_count=6 :: non_neg_integer(), + metadata=[] :: proplists:proplist(), + properties=[] :: proplists:proplist(), + rolling_upgrade=false :: boolean(), + start_version=current :: atom(), + upgrade_version=current :: atom(), + wait_for_transfers=false :: boolean(), + valid_backends=all :: all | [atom()], + make_cluster=true :: boolean(), + config :: term() + }). +-type properties() :: #rt_properties_v1{}. + +-export_type([properties/0]). + +-define(RT_PROPERTIES, #rt_properties_v1). +-define(RECORD_FIELDS, record_info(fields, rt_properties_v1)). + +-export([new/0, + new/1, + get/2, + set/2, + set/3]). + +%% @doc Create a new properties record with all fields initialized to +%% the default values. +-spec new() -> properties(). +new() -> + ?RT_PROPERTIES{}. + +%% @doc Create a new properties record with the fields initialized to +%% non-default value. Each field to be initialized should be +%% specified as an entry in a property list (i.e. a list of +%% pairs). Invalid property fields are ignored by this function. +-spec new(proplists:proplist()) -> properties(). +new(PropertyDefaults) -> + {Properties, _} = + lists:foldl(fun set_property/2, {?RT_PROPERTIES{}, []}, PropertyDefaults), + Properties. + +%% @doc Get the value of a property from a properties record. An error +%% is returned if `Properties' is not a valid `rt_properties' record +%% or if the property requested is not a valid property. +-spec get(atom(), properties()) -> term() | {error, atom()}. +get(Property, Properties) -> + get(Property, Properties, validate_request(Property, Properties)). + +%% @doc Set the value for a property in a properties record. An error +%% is returned if `Properties' is not a valid `rt_properties' record +%% or if any of the properties to be set are not a valid property. In +%% the case that invalid properties are specified the error returned +%% contains a list of erroneous properties. +-spec set([{atom(), term()}], properties()) -> {ok, properties()} | {error, atom()}. +set(PropertyList, Properties) when is_list(PropertyList) -> + set_properties(PropertyList, Properties, validate_record(Properties)). + +%% @doc Set the value for a property in a properties record. An error +%% is returned if `Properties' is not a valid `rt_properties' record +%% or if the property to be set is not a valid property. +-spec set(atom(), term(), properties()) -> {ok, properties()} | {error, atom()}. +set(Property, Value, Properties) -> + set_property(Property, Value, Properties, validate_request(Property, Properties)). + + +-spec get(atom(), properties(), ok | {error, atom()}) -> + term() | {error, atom()}. +get(Property, Properties, ok) -> + element(field_index(Property), Properties); +get(_Property, _Properties, {error, _}=Error) -> + Error. + +%% This function is used by `new/1' to set properties at record +%% creation time and by `set/2' to set multiple properties at once. +%% Node properties record validation is done by this function. It is +%% strictly used as a fold function which is the reason for the odd +%% structure of the input parameters. It accumulates any invalid +%% properties that are encountered and the caller may use that +%% information or ignore it. +-spec set_property({atom(), term()}, {properties(), [atom()]}) -> + {properties(), [atom()]}. +set_property({Property, Value}, {Properties, Invalid}) -> + case is_valid_property(Property) of + true -> + {setelement(field_index(Property), Properties, Value), Invalid}; + false -> + {Properties, [Property | Invalid]} + end. + +-spec set_property(atom(), term(), properties(), ok | {error, atom()}) -> + {ok, properties()} | {error, atom()}. +set_property(Property, Value, Properties, ok) -> + {ok, setelement(field_index(Property), Properties, Value)}; +set_property(_Property, _Value, _Properties, {error, _}=Error) -> + Error. + +-spec set_properties([{atom(), term()}], + properties(), + ok | {error, {atom(), [atom()]}}) -> + {properties(), [atom()]}. +set_properties(PropertyList, Properties, ok) -> + case lists:foldl(fun set_property/2, {Properties, []}, PropertyList) of + {UpdProperties, []} -> + UpdProperties; + {_, InvalidProperties} -> + {error, {invalid_properties, InvalidProperties}} + end; +set_properties(_, _, {error, _}=Error) -> + Error. + +-spec validate_request(atom(), term()) -> ok | {error, atom()}. +validate_request(Property, Properties) -> + validate_property(Property, validate_record(Properties)). + +-spec validate_record(term()) -> ok | {error, invalid_properties}. +validate_record(Record) -> + case is_valid_record(Record) of + true -> + ok; + false -> + {error, invalid_properties} + end. + +-spec validate_property(atom(), ok | {error, atom()}) -> ok | {error, invalid_property}. +validate_property(Property, ok) -> + case is_valid_property(Property) of + true -> + ok; + false -> + {error, invalid_property} + end; +validate_property(_Property, {error, _}=Error) -> + Error. + +-spec is_valid_record(term()) -> boolean(). +is_valid_record(Record) -> + is_record(Record, rt_properties_v1). + +-spec is_valid_property(atom()) -> boolean(). +is_valid_property(Property) -> + Fields = ?RECORD_FIELDS, + lists:member(Property, Fields). + +-spec field_index(atom()) -> non_neg_integer(). +field_index(nodes) -> + ?RT_PROPERTIES.nodes; +field_index(node_count) -> + ?RT_PROPERTIES.node_count; +field_index(metadata) -> + ?RT_PROPERTIES.metadata; +field_index(properties) -> + ?RT_PROPERTIES.properties; +field_index(rolling_upgrade) -> + ?RT_PROPERTIES.rolling_upgrade; +field_index(start_version) -> + ?RT_PROPERTIES.start_version; +field_index(upgrade_version) -> + ?RT_PROPERTIES.upgrade_version; +field_index(wait_for_transfers) -> + ?RT_PROPERTIES.wait_for_transfers; +field_index(valid_backends) -> + ?RT_PROPERTIES.valid_backends; +field_index(make_cluster) -> + ?RT_PROPERTIES.make_cluster; +field_index(config) -> + ?RT_PROPERTIES.config. diff --git a/tests/basic_command_line.erl b/tests/basic_command_line.erl index d71b6e833..e2754e4e8 100644 --- a/tests/basic_command_line.erl +++ b/tests/basic_command_line.erl @@ -24,12 +24,10 @@ -export([properties/0, confirm/2]). properties() -> - DefaultProps = rt_cluster:properties(), - DefaultProps#rt_properties{node_count=1, - rolling_upgrade=false, - make_cluster=true}. + rt_properties:new([{node_count, 1}]). -confirm(#rt_properties{nodes=Nodes}, _MD) -> +confirm(Properties, _MD) -> + Nodes = rt_properties:get(nodes, Properties), Node = hd(Nodes), %% Deploy a node to test against lager:info("Deploy node to test command line"), diff --git a/tests/bucket_props_roundtrip.erl b/tests/bucket_props_roundtrip.erl index 90bd2fbd4..a2d7876c9 100644 --- a/tests/bucket_props_roundtrip.erl +++ b/tests/bucket_props_roundtrip.erl @@ -54,13 +54,10 @@ ]). properties() -> - DefaultProps = rt_cluster:properties(), - DefaultProps#rt_properties{node_count=1, - rolling_upgrade=false, - make_cluster=true}. + rt_properties:new([{node_count, 1}]). -confirm(#rt_properties{nodes=Nodes}, _MD) -> - [Node] = Nodes, +confirm(Properties, _MD) -> + [Node] = rt_properties:get(nodes, Properties), [ check_prop_set_and_get(Node, Prop, FirstVal, SecondVal) || {Prop, FirstVal, SecondVal} <- ?PROPS ], diff --git a/tests/bucket_types.erl b/tests/bucket_types.erl index 118c96323..fea36b1ec 100644 --- a/tests/bucket_types.erl +++ b/tests/bucket_types.erl @@ -6,16 +6,14 @@ -include("rt.hrl"). properties() -> - DefaultProps = rt_cluster:properties(), - CustomConfig = rt_cluster:augment_config(riak_core, + CustomConfig = rt_cluster:augment_config(riak_core, {default_bucket_props, [{n_val, 2}]}, - DefaultProps#rt_properties.config), - DefaultProps#rt_properties{node_count=4, - rolling_upgrade=false, - make_cluster=true, - config=CustomConfig}. + rt_cluster:config()), + rt_properties:new([{node_count, 4}, + {config, CustomConfig}]). -confirm(#rt_properties{nodes=Nodes}, _MD) -> +confirm(Properties, _MD) -> + Nodes = rt_properties:get(nodes, Properties), Node = hd(Nodes), application:start(inets), @@ -51,7 +49,7 @@ confirm(#rt_properties{nodes=Nodes}, _MD) -> %% write implicitly to the default bucket riakc_pb_socket:put(PB, riakc_obj:update_value(O1, <<"newvalue">>)), - + %% read from the default bucket explicitly {ok, O3} = riakc_pb_socket:get(PB, {<<"default">>, <<"bucket">>}, <<"key">>), diff --git a/tests/cluster_meta_basic.erl b/tests/cluster_meta_basic.erl index cb4cf0935..f4012a0b4 100644 --- a/tests/cluster_meta_basic.erl +++ b/tests/cluster_meta_basic.erl @@ -31,12 +31,10 @@ -define(VAL2, val2). properties() -> - DefaultProps = rt_cluster:properties(), - DefaultProps#rt_properties{node_count=5, - rolling_upgrade=false, - make_cluster=true}. + rt_properties:new([{node_count, 5}]). -confirm(#rt_properties{nodes=Nodes}, _MD) -> +confirm(Properties, _MD) -> + Nodes = rt_properties:get(nodes, Properties), ok = test_fold_full_prefix(Nodes), ok = test_metadata_conflicts(Nodes), ok = test_writes_after_partial_cluster_failure(Nodes), diff --git a/tests/http_bucket_types.erl b/tests/http_bucket_types.erl index 8b3d147c1..a51935a3a 100644 --- a/tests/http_bucket_types.erl +++ b/tests/http_bucket_types.erl @@ -7,16 +7,15 @@ -include("rt.hrl"). properties() -> - DefaultProps = rt_cluster:properties(), - CustomConfig = rt_cluster:augment_config(riak_core, + CustomConfig = rt_cluster:augment_config(riak_core, {default_bucket_props, [{n_val, 2}]}, - DefaultProps#rt_properties.config), - DefaultProps#rt_properties{node_count=1, - rolling_upgrade=false, - make_cluster=true, - config=CustomConfig}. + rt_cluster:config()), + rt_properties:new([{node_count, 1}, + {config, CustomConfig}]). -confirm(#rt_properties{nodes=Nodes}, _MD) -> + +confirm(Properties, _MD) -> + Nodes = rt_properties:get(nodes, Properties), Node = hd(Nodes), application:start(ibrowse), diff --git a/tests/secondary_index_tests.erl b/tests/secondary_index_tests.erl index 4ad75f1c5..df58b8f41 100644 --- a/tests/secondary_index_tests.erl +++ b/tests/secondary_index_tests.erl @@ -34,19 +34,17 @@ -define(KEYS(A,B,G1,G2), [int_to_key(N) || N <- lists:seq(A,B), G1, G2]). properties() -> - DefaultProps = rt_cluster:properties(), - DefaultProps#rt_properties{node_count=3, - wait_for_transfers=true, - rolling_upgrade=false, - start_version=previous, - config=config()}. + rt_properties:new([{node_count, 3}, + {wait_for_transfers, true}, + {start_version, previous}, + {config, config()}]). config() -> - [ - {riak_kv, [{secondary_index_sort_default, false}]} - ]. + [{riak_kv, [{secondary_index_sort_default, false}]}, + {riak_core, [{handoff_concurrency, 11}]}]. -confirm(#rt_properties{nodes=Nodes}, _MD) -> +confirm(Properties, _MD) -> + Nodes = rt_properties:get(nodes, Properties), Bucket = druuid:v4_str(), lager:info("Bucket: ~p", [Bucket]), PBC = rt_pb:pbc(hd(Nodes)), diff --git a/tests/verify_build_cluster.erl b/tests/verify_build_cluster.erl index c502e6d90..fac763314 100644 --- a/tests/verify_build_cluster.erl +++ b/tests/verify_build_cluster.erl @@ -24,18 +24,17 @@ -include_lib("eunit/include/eunit.hrl"). properties() -> - DefaultProps = rt_cluster:properties(), UpdConfig = rt_cluster:augment_config(riak_core, {default_bucket_props, [{allow_mult, false}]}, - DefaultProps#rt_properties.config), - DefaultProps#rt_properties{config=UpdConfig, - node_count=4, - rolling_upgrade=true, - make_cluster=false, - start_version=previous}. + rt_cluster:config()), + rt_properties:new([{config, UpdConfig}, + {node_count, 4}, + {rolling_upgrade, true}, + {make_cluster, false}, + {start_version, previous}]). -confirm(#rt_properties{nodes=Nodes}, _MD) -> - [Node1, Node2, Node3, Node4] = Nodes, +confirm(Properties, _MD) -> + [Node1, Node2, Node3, Node4] = Nodes = rt_properties:get(nodes, Properties), lager:info("Loading some data up in this cluster."), ?assertEqual([], rt_systest:write(Node1, 0, 1000, <<"verify_build_cluster">>, 2)),