From 742bd5676d471ec448b2c944c286514f43e79dee Mon Sep 17 00:00:00 2001 From: Brett Hazen Date: Mon, 6 Feb 2017 11:54:55 -0700 Subject: [PATCH 01/35] Add devrel_matrix.yml --- devrel_matrix.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 devrel_matrix.yml diff --git a/devrel_matrix.yml b/devrel_matrix.yml new file mode 100644 index 000000000..70113c99a --- /dev/null +++ b/devrel_matrix.yml @@ -0,0 +1,19 @@ +docker_rt_riak_test_config: + - version: current + product_version: current + product: "riak_ts{% if ee == 'true' %}_ee{% endif %}" + - version: previous + product_version: 1.5.1 + product: "riak_ts{% if ee == 'true' %}_ee{% endif %}" + - version: legacy + product_version: 1.4.1 + product: "riak_ts{% if ee == 'true' %}_ee{% endif %}" + - version: 2.0.2 + product_version: 2.0.2 + product: "riak{% if ee == 'true' %}_ee{% endif %}" + - version: 2.0.4 + product_version: 2.0.4 + product: "riak{% if ee == 'true' %}_ee{% endif %}" + - version: 2.0.5 + product_version: 2.0.5 + product: "riak{% if ee == 'true' %}_ee{% endif %}" From 7b56bd483b76fa29fa2e8c647bbe1f5b19c44fb4 Mon Sep 17 00:00:00 2001 From: Jenkins Date: Tue, 7 Feb 2017 00:19:23 -0800 Subject: [PATCH 02/35] Jenkins - update locked-deps for riak_ts_ee-1.6.0nightly20170207 --- rebar.config.lock | 1 - 1 file changed, 1 deletion(-) diff --git a/rebar.config.lock b/rebar.config.lock index cf45213e3..940c54ea3 100644 --- a/rebar.config.lock +++ b/rebar.config.lock @@ -32,7 +32,6 @@ {riakc,".*", {git,"https://github.com/basho/riak-erlang-client", "82ae32bd28090ab911b5c25d2be690c7d2a3e027"}}, - "f3e64014d90c261880f5b06113bb604a6d5df495"}}, {ibrowse,".*", {git,"https://github.com/basho/ibrowse.git", "b28542d1e326ba44bcfaf7fd6d3c7f8761d20f08"}}, From f06173db35a999b0e30f36c2190e51228c317dc1 Mon Sep 17 00:00:00 2001 From: Andrei Zavada Date: Sun, 5 Feb 2017 18:22:14 +0200 Subject: [PATCH 03/35] ts_simple_percentile: new module --- tests/ts_qbuf_util.erl | 26 ++- tests/ts_qbuf_util.hrl | 1 + tests/ts_simple_invdist_funs_SUITE.erl | 228 +++++++++++++++++++++++++ 3 files changed, 248 insertions(+), 7 deletions(-) create mode 100644 tests/ts_simple_invdist_funs_SUITE.erl diff --git a/tests/ts_qbuf_util.erl b/tests/ts_qbuf_util.erl index 0b268012f..7b839f55b 100644 --- a/tests/ts_qbuf_util.erl +++ b/tests/ts_qbuf_util.erl @@ -23,7 +23,9 @@ -export([ ack_query_error/3, + ack_query_error/4, base_query/1, + base_query/2, create_table/2, full_query/2, insert_data/3, @@ -80,8 +82,10 @@ insert_data(C, Table, Data) -> %% Form queries base_query(Table) -> - fmt("select * from ~s where a = '~s' and b = '~s' and c >= ~b and c <= ~b", - [Table, + base_query(Table, "*"). +base_query(Table, Select) -> + fmt("select ~s from ~s where a = '~s' and b = '~s' and c >= ~b and c <= ~b", + [Select, Table, binary_to_list(?WHERE_FILTER_A), binary_to_list(?WHERE_FILTER_B), ?TIMEBASE, ?TIMEBASE + ?LIFESPAN_EXTRA * 1000]). @@ -108,18 +112,26 @@ make_orderby_with_qualifiers(F) -> ack_query_error(Cfg, Query, ErrCode) -> + ack_query_error(Cfg, Query, ErrCode, <<".*">>). + +ack_query_error(Cfg, Query, ExpErrCode, ExpErrMsgPat) -> Node = hd(proplists:get_value(cluster, Cfg)), C = rt:pbc(Node), Res = riakc_ts:query(C, Query, [], undefined, []), case Res of - {error, {ErrCode, ErrMsg}} -> - io:format("reported error ~s", [ErrMsg]), - ok; + {error, {ExpErrCode, GotErrMsg}} -> + case re:run(GotErrMsg, ExpErrMsgPat, []) of + {match, _} -> + ok; + nomatch -> + ct:pal("Query: ~s\nWrong reported error: ~s\n", [Query, GotErrMsg]), + fail + end; {error, OtherReason} -> - io:format("error not correctly reported: got ~p instead", [OtherReason]), + ct:pal("Query: ~s\nError not correctly reported: got ~p instead\n", [Query, OtherReason]), fail; NonError -> - io:format("error not detected: got ~p instead", [NonError]), + ct:pal("Query: ~s\nError not detected: got ~p instead\n", [Query, NonError]), fail end. diff --git a/tests/ts_qbuf_util.hrl b/tests/ts_qbuf_util.hrl index 3f17b6e0f..3fd4593e8 100644 --- a/tests/ts_qbuf_util.hrl +++ b/tests/ts_qbuf_util.hrl @@ -27,6 +27,7 @@ -define(ORDBY_COLS, ["a", "b", "c", "d", "e", undefined]). %% error codes as defined in riak_kv_ts_svc.erl +-define(E_SUBMIT, 1001). -define(E_SELECT_RESULT_TOO_BIG, 1022). -define(E_QBUF_CREATE_ERROR, 1023). -define(E_QBUF_LDB_ERROR, 1024). diff --git a/tests/ts_simple_invdist_funs_SUITE.erl b/tests/ts_simple_invdist_funs_SUITE.erl new file mode 100644 index 000000000..e49279922 --- /dev/null +++ b/tests/ts_simple_invdist_funs_SUITE.erl @@ -0,0 +1,228 @@ +% ------------------------------------------------------------------- +%% +%% Copyright (c) 2017 Basho Technologies, Inc. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% ------------------------------------------------------------------- +%% @doc Inverse distribution functions (PERCENTILE, MEDIAN) + +-module(ts_simple_invdist_funs_SUITE). + +-export([suite/0, init_per_suite/1, groups/0, all/0]). +-export([query_invdist_percentile/1, + query_invdist_percentile_multiple/1, + query_invdist_median/1, + query_invdist_errors/1]). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include("ts_qbuf_util.hrl"). + + +suite() -> + [{timetrap, {minutes, 5}}]. + +init_per_suite(Cfg) -> + Cluster = ts_setup:start_cluster(1), + C = rt:pbc(hd(Cluster)), + Data = make_data(), + ok = create_table(C, ?TABLE), + ok = insert_data(C, ?TABLE, Data), + [{cluster, Cluster}, + {data, Data} | Cfg]. + +groups() -> + []. + +all() -> + [ + query_invdist_percentile, + query_invdist_percentile_multiple, + query_invdist_median, + query_invdist_errors + ]. + + +%% test cases +%% ------------------------------- + +query_invdist_percentile(Cfg) -> + C = rt:pbc(hd(proplists:get_value(cluster, Cfg))), + Data = proplists:get_value(data, Cfg), + lists:foreach( + fun(Col) -> + check_column(C, Col, Data) orelse + ct:fail("") + end, + ["a", "b", "c", "d", "e"]). + + +query_invdist_percentile_multiple(Cfg) -> + C = rt:pbc(hd(proplists:get_value(cluster, Cfg))), + Data = proplists:get_value(data, Cfg), + {Col1, Col2, Pc1, Pc2} = {"b", "b", 0.22, 0.77}, + Query = make_query(fmt("percentile(~s, ~g), percentile(~s, ~g)", [Col1, Pc1, Col2, Pc2])), + {ok, {_Cols, [{Got1, Got2}]} = _Returned} = + riakc_ts:query(C, Query, [], undefined, []), + {Need1, Need2} = {get_percentile(Col1, Pc1, order_by(Col1, Data)), + get_percentile(Col2, Pc2, order_by(Col2, Data))}, + ct:log("Query \"~s\"", [Query]), + case {Got1 == Need1, Got2 == Need2} of + {true, true} -> + ok; + _ -> + ct:fail("Got {~p, ~p}, Need {~p, ~p}\n", [Got1, Got2, Need1, Need2]) + end. + +query_invdist_median(Cfg) -> + C = rt:pbc(hd(proplists:get_value(cluster, Cfg))), + lists:foreach( + fun(Col) -> + Query = make_query(fmt("percentile(~s, 0.5), median(~s)", [Col, Col])), + {ok, {_Cols, [{Got1, Got2}]} = _Returned} = + riakc_ts:query(C, Query, [], undefined, []), + ct:log("Query \"~s\"", [Query]), + case Got1 == Got2 of + true -> + ok; + _ -> + ct:fail("Got {~p, ~p}, Need equal\n", [Got1, Got2]) + end + end, + ["a", "b", "c", "d", "e"]). + +query_invdist_errors(Cfg) -> + lists:foreach( + fun({Select, Extra, ErrPat}) -> + Qry = make_query(Select, Extra), + ?assertEqual( + ok, + ts_qbuf_util:ack_query_error(Cfg, Qry, ?E_SUBMIT, ErrPat)) + end, + [{"percentile(nxcol, 0.2)", [], + "Unknown column \"nxcol\""}, + {"median(a), b", [], + "Inverse distribution functions cannot be used with other columns in SELECT clause"}, + {"percentile(1)", [], + "Function PERCENTILE/1 called with 1 argument"}, + {"percentile(a, 1.2)", [], + "Invalid argument 2 in call to function PERCENTILE"}, + {"percentile(a, 0.1), percentile(b, 0.2)", [], + "Multiple inverse distribution functions must all have the same column argument"}, + {"percentile(a, 1+c)", [], + "Function 'PERCENTILE' called with arguments of the wrong type"}, + {"percentile(a, 1.1+c)", [], + "Non-const expression passed as parameter for inverse distribution function"}, + {"percentile(a, 1.1/0)", [], + "Invalid expression passed as parameter for inverse distribution function"}, + {"percentile(a, 0.3), avg(b)", [], + "Inverse distribution functions cannot be used with GROUP BY clause or aggregating window functions"}, + {"percentile(a, 0.1)", "group by a", + "Inverse distribution functions cannot be used with GROUP BY clause or aggregating window functions"}, + {"percentile(a, 0.1)", "order by a", + "Inverse distribution functions cannot be used with any of ORDER BY, LIMIT or OFFSET clauses"}, + {"percentile(a, 0.1)", "limit 1", + "Inverse distribution functions cannot be used with any of ORDER BY, LIMIT or OFFSET clauses"}]). + + +%% supporting functions +%% ------------------------------- + +create_table(Client, Table) -> + DDL = " +create table " ++ Table ++ " +(a timestamp not null, + b double, + c sint64, + d sint64, + e sint64, + primary key ((quantum(a, 10, h)), a))", + {ok, {[], []}} = riakc_ts:query(Client, DDL), + ok. + +data_generator(I) -> + {?TIMEBASE + (I + 1) * 1000, + 100 * math:cos(float(I) / 10 * math:pi()), + case I rem 5 of 0 -> []; _ -> I end, %% sprinkle some NULLs + I+1, + -I + }. + +make_data() -> + [data_generator(I) || I <- lists:seq(0, ?LIFESPAN)]. + +insert_data(_C, _Table, []) -> + ok; +insert_data(C, Table, Data) -> + Batch = lists:sublist(Data, 50), + ok = riakc_ts:put(C, Table, Batch), + case catch lists:nthtail(50, Data) of + Rest when is_list(Rest) -> + insert_data(C, Table, Rest); + _ -> + ok + end. + +make_query(Select) -> + make_query(Select, ""). +make_query(Select, Extra) -> + fmt("select ~s from ~s where a >= ~b and a <= ~b~s", + [Select, ?TABLE, + ?TIMEBASE, ?TIMEBASE * 1000, + [" " ++ Extra || Extra /= []]]). + + + +check_column(C, Col, Data) -> + SortedData = order_by(Col, Data), + ok == lists:foreach( + fun({Pc, Pc_s}) -> + Query = make_query(fmt("percentile(~s, ~s)", [Col, Pc_s])), + {ok, {_Cols, [{Got}]} = _Returned} = + riakc_ts:query(C, Query, [], undefined, []), + Need = get_percentile(Col, Pc, SortedData), + ct:log("Query \"~s\"", [Query]), + case Got == Need of + true -> + ok; + false -> + ct:fail("Got ~p, Need ~p\n", [Got, Need]) + end + end, + [{0.24, "0.24"}, + {0.11, "0.11"}, + {0.0, "0.0"}, + {0.8, "0.8*1"}, + {1.0, "1/1.0"}, + {0.35, "(3.5/10)"}, + {0.3, "0.4-(1*0.1)"}]). + +order_by([FChar|_], Data) -> + FNo = 1 + (FChar - $a), + lists:sort( + fun(Row1, Row2) -> + element(FNo, Row1) =< element(FNo, Row2) + end, + Data). + +get_percentile([FChar|_], Pc, Data) -> + FNo = 1 + (FChar - $a), + Pos = lists:min([1 + round(Pc * length(Data)), length(Data)]), + element(FNo, lists:nth(Pos, Data)). + + +fmt(F, A) -> + lists:flatten(io_lib:format(F, A)). From 8af32125ce75d5ad5070a01657050cc2d1813f17 Mon Sep 17 00:00:00 2001 From: Andrei Zavada Date: Sat, 11 Feb 2017 16:34:50 +0200 Subject: [PATCH 04/35] `git --local config` doesn't work as expected --- bin/rtdev-setup-releases.sh | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/bin/rtdev-setup-releases.sh b/bin/rtdev-setup-releases.sh index 4b958b417..d67759b3e 100755 --- a/bin/rtdev-setup-releases.sh +++ b/bin/rtdev-setup-releases.sh @@ -39,14 +39,11 @@ cd $RT_DEST_DIR git init ## Some versions of git and/or OS require these fields -HAS_LOCAL=$(git config --local 2>&1 | grep unknown) -if [ -z "$HAS_LOCAL" ]; then - git config --local user.name "Riak Test" - git config --local user.email "dev@basho.com" - git config --local core.autocrlf input - git config --local core.safecrlf false - git config --local core.filemode true -fi +git config --local user.name "Riak Test" +git config --local user.email "dev@basho.com" +git config --local core.autocrlf input +git config --local core.safecrlf false +git config --local core.filemode true ## this prevents priv/*.so files from being deleted by git clean -fd ## (the latter is executed in rtdev-current.sh): @@ -54,5 +51,4 @@ echo "priv/" >.gitignore git add --all --force . git commit -a -m "riak_test init" > /dev/null - echo " - Successfully completed initial git commit of $RT_DEST_DIR" From 2c147fedfe6a3d7386da7fa10e38b6dd6788f638 Mon Sep 17 00:00:00 2001 From: Andrei Zavada Date: Sat, 11 Feb 2017 16:38:19 +0200 Subject: [PATCH 05/35] whitespace cleanup --- tests/ts_cluster_group_by_SUITE.erl | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/tests/ts_cluster_group_by_SUITE.erl b/tests/ts_cluster_group_by_SUITE.erl index e9b1923e0..f0ddcccea 100644 --- a/tests/ts_cluster_group_by_SUITE.erl +++ b/tests/ts_cluster_group_by_SUITE.erl @@ -55,7 +55,7 @@ end_per_testcase(_TestCase, _Config) -> groups() -> []. -all() -> +all() -> rt:grep_test_functions(?MODULE). client_pid(Ctx) -> @@ -115,11 +115,3 @@ group_by_2_test(Ctx) -> {rt_ignore_columns, [{1,500.5},{2,500.5},{3,500.5}]}, {ok,{Cols, lists:sort(Rows)}} ). - - - - - - - - From 89a3e4bc792ec2a2fa928c0fe1af6d9ae00fd2b2 Mon Sep 17 00:00:00 2001 From: Andrei Zavada Date: Sun, 12 Feb 2017 00:43:29 +0200 Subject: [PATCH 06/35] ta_simple_invdist_funs: test both backends --- tests/ts_simple_invdist_funs_SUITE.erl | 54 ++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/tests/ts_simple_invdist_funs_SUITE.erl b/tests/ts_simple_invdist_funs_SUITE.erl index e49279922..8b92df70f 100644 --- a/tests/ts_simple_invdist_funs_SUITE.erl +++ b/tests/ts_simple_invdist_funs_SUITE.erl @@ -23,6 +23,7 @@ -export([suite/0, init_per_suite/1, groups/0, all/0]). -export([query_invdist_percentile/1, + query_invdist_percentile_backends/1, query_invdist_percentile_multiple/1, query_invdist_median/1, query_invdist_errors/1]). @@ -51,6 +52,7 @@ all() -> [ query_invdist_percentile, query_invdist_percentile_multiple, + query_invdist_percentile_backends, query_invdist_median, query_invdist_errors ]. @@ -70,6 +72,37 @@ query_invdist_percentile(Cfg) -> ["a", "b", "c", "d", "e"]). +query_invdist_percentile_backends(Cfg) -> + Node = hd(proplists:get_value(cluster, Cfg)), + C = rt:pbc(Node), + Data = proplists:get_value(data, Cfg), + + WorkF = + fun() -> + lists:foreach( + fun(Col) -> + check_column(C, Col, Data) orelse + ct:fail("") + end, + ["a", "b", "c", "d", "e"]) + end, + + rpc:call(Node, code, add_patha, [filename:join([rt_local:home_dir(), "../../ebin"])]), + rt_intercept:load_code(Node), + + load_intercept(Node, C, {riak_kv_qry_buffers, [{{can_afford_inmem, 1}, can_afford_inmem_yes}]}), + ct:log("all inmem", []), + WorkF(), + load_intercept(Node, C, {riak_kv_qry_buffers, [{{can_afford_inmem, 1}, can_afford_inmem_no}]}), + ct:log("all ldb", []), + WorkF(), + load_intercept(Node, C, {riak_kv_qry_buffers, [{{can_afford_inmem, 1}, can_afford_inmem_random}]}), + ct:log("random", []), + WorkF(), + rt_intercept:clean(Node, riak_kv_qry_buffers), + ok. + + query_invdist_percentile_multiple(Cfg) -> C = rt:pbc(hd(proplists:get_value(cluster, Cfg))), Data = proplists:get_value(data, Cfg), @@ -224,5 +257,26 @@ get_percentile([FChar|_], Pc, Data) -> element(FNo, lists:nth(Pos, Data)). +load_intercept(Node, C, Intercept) -> + ok = rt_intercept:add(Node, Intercept), + %% when code changes underneath the riak_kv_qry_buffers + %% gen_server, it gets reinitialized. We need to probe it with a + %% dummy query until it becomes ready. + rt_intercept:wait_until_loaded(Node), + wait_until_qbuf_mgr_reinit(C). + +wait_until_qbuf_mgr_reinit(C) -> + ProbingQuery = make_query("*", ""), + case riakc_ts:query(C, ProbingQuery, [], undefined, []) of + {ok, _Data} -> + ok; + {error, {ErrCode, _NotReadyMessage}} + when ErrCode == ?E_QBUF_CREATE_ERROR; + ErrCode == ?E_QBUF_INTERNAL_ERROR -> + timer:sleep(100), + wait_until_qbuf_mgr_reinit(C) + end. + + fmt(F, A) -> lists:flatten(io_lib:format(F, A)). From 07f2288ea0aec1d155e2e5aad3d66d605f2246bd Mon Sep 17 00:00:00 2001 From: Andrei Zavada Date: Thu, 16 Feb 2017 06:37:44 +0200 Subject: [PATCH 07/35] support PERCENTILE_DISC, _CONT --- intercepts/riak_kv_qry_buffers_intercepts.erl | 2 +- tests/ts_simple_invdist_funs_SUITE.erl | 92 ++++++++++++------- 2 files changed, 61 insertions(+), 33 deletions(-) diff --git a/intercepts/riak_kv_qry_buffers_intercepts.erl b/intercepts/riak_kv_qry_buffers_intercepts.erl index f54c2e994..7bd01d75b 100644 --- a/intercepts/riak_kv_qry_buffers_intercepts.erl +++ b/intercepts/riak_kv_qry_buffers_intercepts.erl @@ -1,6 +1,6 @@ %% ------------------------------------------------------------------- %% -%% Copyright (c) 2016 Basho Technologies, Inc. +%% Copyright (c) 2016, 2017 Basho Technologies, Inc. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file diff --git a/tests/ts_simple_invdist_funs_SUITE.erl b/tests/ts_simple_invdist_funs_SUITE.erl index 8b92df70f..73ebaed5b 100644 --- a/tests/ts_simple_invdist_funs_SUITE.erl +++ b/tests/ts_simple_invdist_funs_SUITE.erl @@ -28,6 +28,8 @@ query_invdist_median/1, query_invdist_errors/1]). +-export([percentile_disc/3, percentile_cont/3]). %% make them 'used' for erlc + -include_lib("common_test/include/ct.hrl"). -include_lib("eunit/include/eunit.hrl"). -include("ts_qbuf_util.hrl"). @@ -107,11 +109,11 @@ query_invdist_percentile_multiple(Cfg) -> C = rt:pbc(hd(proplists:get_value(cluster, Cfg))), Data = proplists:get_value(data, Cfg), {Col1, Col2, Pc1, Pc2} = {"b", "b", 0.22, 0.77}, - Query = make_query(fmt("percentile(~s, ~g), percentile(~s, ~g)", [Col1, Pc1, Col2, Pc2])), + Query = make_query(fmt("percentile_disc(~s, ~g), percentile_cont(~s, ~g)", [Col1, Pc1, Col2, Pc2])), {ok, {_Cols, [{Got1, Got2}]} = _Returned} = riakc_ts:query(C, Query, [], undefined, []), - {Need1, Need2} = {get_percentile(Col1, Pc1, order_by(Col1, Data)), - get_percentile(Col2, Pc2, order_by(Col2, Data))}, + {Need1, Need2} = {percentile_disc(Col1, Pc1, order_by(Col1, Data)), + percentile_cont(Col2, Pc2, order_by(Col2, Data))}, ct:log("Query \"~s\"", [Query]), case {Got1 == Need1, Got2 == Need2} of {true, true} -> @@ -124,7 +126,7 @@ query_invdist_median(Cfg) -> C = rt:pbc(hd(proplists:get_value(cluster, Cfg))), lists:foreach( fun(Col) -> - Query = make_query(fmt("percentile(~s, 0.5), median(~s)", [Col, Col])), + Query = make_query(fmt("percentile_disc(~s, 0.5), median(~s)", [Col, Col])), {ok, {_Cols, [{Got1, Got2}]} = _Returned} = riakc_ts:query(C, Query, [], undefined, []), ct:log("Query \"~s\"", [Query]), @@ -145,29 +147,29 @@ query_invdist_errors(Cfg) -> ok, ts_qbuf_util:ack_query_error(Cfg, Qry, ?E_SUBMIT, ErrPat)) end, - [{"percentile(nxcol, 0.2)", [], + [{"percentile_disc(nxcol, 0.2)", [], "Unknown column \"nxcol\""}, {"median(a), b", [], "Inverse distribution functions cannot be used with other columns in SELECT clause"}, - {"percentile(1)", [], - "Function PERCENTILE/1 called with 1 argument"}, - {"percentile(a, 1.2)", [], - "Invalid argument 2 in call to function PERCENTILE"}, - {"percentile(a, 0.1), percentile(b, 0.2)", [], + {"percentile_disc(1)", [], + "Function PERCENTILE_DISC/1 called with 1 argument"}, + {"percentile_disc(a, 1.2)", [], + "Invalid argument 2 in call to function PERCENTILE_DISC"}, + {"percentile_disc(a, 0.1), percentile_disc(b, 0.2)", [], "Multiple inverse distribution functions must all have the same column argument"}, - {"percentile(a, 1+c)", [], - "Function 'PERCENTILE' called with arguments of the wrong type"}, - {"percentile(a, 1.1+c)", [], + {"percentile_disc(a, 1+c)", [], + "Function 'PERCENTILE_DISC' called with arguments of the wrong type"}, + {"percentile_disc(a, 1.1+c)", [], "Non-const expression passed as parameter for inverse distribution function"}, - {"percentile(a, 1.1/0)", [], + {"percentile_disc(a, 1.1/0)", [], "Invalid expression passed as parameter for inverse distribution function"}, - {"percentile(a, 0.3), avg(b)", [], + {"percentile_disc(a, 0.3), avg(b)", [], "Inverse distribution functions cannot be used with GROUP BY clause or aggregating window functions"}, - {"percentile(a, 0.1)", "group by a", + {"percentile_disc(a, 0.1)", "group by a", "Inverse distribution functions cannot be used with GROUP BY clause or aggregating window functions"}, - {"percentile(a, 0.1)", "order by a", + {"percentile_disc(a, 0.1)", "order by a", "Inverse distribution functions cannot be used with any of ORDER BY, LIMIT or OFFSET clauses"}, - {"percentile(a, 0.1)", "limit 1", + {"percentile_disc(a, 0.1)", "limit 1", "Inverse distribution functions cannot be used with any of ORDER BY, LIMIT or OFFSET clauses"}]). @@ -221,13 +223,23 @@ make_query(Select, Extra) -> check_column(C, Col, Data) -> SortedData = order_by(Col, Data), + Combos = + [{PercentileVariety, Parm, Parm_s} || + {Parm, Parm_s} <- [{0.24, "0.24"}, + {0.11, "0.11"}, + {0.0, "0.0"}, + {0.8, "0.8*1"}, + {1.0, "1/1.0"}, + {0.36, "(3.6/10)"}, + {0.40, "0.5 - 1 * 0.1"}], + PercentileVariety <- [percentile_cont, percentile_disc]], ok == lists:foreach( - fun({Pc, Pc_s}) -> - Query = make_query(fmt("percentile(~s, ~s)", [Col, Pc_s])), + fun({PercentileFun, Pc, Pc_s}) -> + Query = make_query(fmt("~s(~s, ~s)", [PercentileFun, Col, Pc_s])), + ct:log("Query \"~s\"", [Query]), {ok, {_Cols, [{Got}]} = _Returned} = riakc_ts:query(C, Query, [], undefined, []), - Need = get_percentile(Col, Pc, SortedData), - ct:log("Query \"~s\"", [Query]), + Need = apply(?MODULE, PercentileFun, [Col, Pc, SortedData]), case Got == Need of true -> ok; @@ -235,13 +247,7 @@ check_column(C, Col, Data) -> ct:fail("Got ~p, Need ~p\n", [Got, Need]) end end, - [{0.24, "0.24"}, - {0.11, "0.11"}, - {0.0, "0.0"}, - {0.8, "0.8*1"}, - {1.0, "1/1.0"}, - {0.35, "(3.5/10)"}, - {0.3, "0.4-(1*0.1)"}]). + Combos). order_by([FChar|_], Data) -> FNo = 1 + (FChar - $a), @@ -251,11 +257,33 @@ order_by([FChar|_], Data) -> end, Data). -get_percentile([FChar|_], Pc, Data) -> +percentile_disc([FChar|_], Pc, Data_) -> FNo = 1 + (FChar - $a), - Pos = lists:min([1 + round(Pc * length(Data)), length(Data)]), - element(FNo, lists:nth(Pos, Data)). + Data = [D || D <- Data_, element(FNo, D) /= []], + RN = (1 + (Pc * (length(Data) - 1))), + Row = lists:nth(trunc(RN), Data), + element(FNo, Row). +percentile_cont([FChar|_], Pc, Data_) -> + FNo = 1 + (FChar - $a), + Data = [D || D <- Data_, element(FNo, D) /= []], + RN = (1 + (Pc * (length(Data) - 1))), + {LoRN, HiRN} = {trunc(RN), ceil(RN)}, + case LoRN == HiRN of + true -> + element(FNo, lists:nth(LoRN, Data)); + false -> + LoVal = element(FNo, lists:nth(LoRN, Data)), + HiVal = element(FNo, lists:nth(HiRN, Data)), + (HiRN - RN) * LoVal + (RN - LoRN) * HiVal + end. + +ceil(X) -> + T = trunc(X), + case X - T == 0 of + true -> T; + false -> T + 1 + end. load_intercept(Node, C, Intercept) -> ok = rt_intercept:add(Node, Intercept), From 4ead64e41652456faa79a681cb597c86fcc264cb Mon Sep 17 00:00:00 2001 From: Andrei Zavada Date: Thu, 16 Feb 2017 17:01:21 +0200 Subject: [PATCH 08/35] tests for MODE --- tests/ts_simple_invdist_funs_SUITE.erl | 126 ++++++++++++++++++------- 1 file changed, 92 insertions(+), 34 deletions(-) diff --git a/tests/ts_simple_invdist_funs_SUITE.erl b/tests/ts_simple_invdist_funs_SUITE.erl index 73ebaed5b..f20c5ba5b 100644 --- a/tests/ts_simple_invdist_funs_SUITE.erl +++ b/tests/ts_simple_invdist_funs_SUITE.erl @@ -26,6 +26,7 @@ query_invdist_percentile_backends/1, query_invdist_percentile_multiple/1, query_invdist_median/1, + query_invdist_mode/1, query_invdist_errors/1]). -export([percentile_disc/3, percentile_cont/3]). %% make them 'used' for erlc @@ -56,6 +57,7 @@ all() -> query_invdist_percentile_multiple, query_invdist_percentile_backends, query_invdist_median, + query_invdist_mode, query_invdist_errors ]. @@ -93,13 +95,13 @@ query_invdist_percentile_backends(Cfg) -> rt_intercept:load_code(Node), load_intercept(Node, C, {riak_kv_qry_buffers, [{{can_afford_inmem, 1}, can_afford_inmem_yes}]}), - ct:log("all inmem", []), + ct:log("----------- all inmem", []), WorkF(), load_intercept(Node, C, {riak_kv_qry_buffers, [{{can_afford_inmem, 1}, can_afford_inmem_no}]}), - ct:log("all ldb", []), + ct:log("----------- all ldb", []), WorkF(), load_intercept(Node, C, {riak_kv_qry_buffers, [{{can_afford_inmem, 1}, can_afford_inmem_random}]}), - ct:log("random", []), + ct:log("----------- random", []), WorkF(), rt_intercept:clean(Node, riak_kv_qry_buffers), ok. @@ -110,7 +112,7 @@ query_invdist_percentile_multiple(Cfg) -> Data = proplists:get_value(data, Cfg), {Col1, Col2, Pc1, Pc2} = {"b", "b", 0.22, 0.77}, Query = make_query(fmt("percentile_disc(~s, ~g), percentile_cont(~s, ~g)", [Col1, Pc1, Col2, Pc2])), - {ok, {_Cols, [{Got1, Got2}]} = _Returned} = + {ok, {_Cols, [{Got1, Got2}]}} = riakc_ts:query(C, Query, [], undefined, []), {Need1, Need2} = {percentile_disc(Col1, Pc1, order_by(Col1, Data)), percentile_cont(Col2, Pc2, order_by(Col2, Data))}, @@ -119,26 +121,49 @@ query_invdist_percentile_multiple(Cfg) -> {true, true} -> ok; _ -> - ct:fail("Got {~p, ~p}, Need {~p, ~p}\n", [Got1, Got2, Need1, Need2]) + ct:fail("PERCENTILE_DISC vs _CONT: Got {~p, ~p}, Need {~p, ~p}\n", [Got1, Got2, Need1, Need2]) end. + query_invdist_median(Cfg) -> C = rt:pbc(hd(proplists:get_value(cluster, Cfg))), lists:foreach( fun(Col) -> Query = make_query(fmt("percentile_disc(~s, 0.5), median(~s)", [Col, Col])), - {ok, {_Cols, [{Got1, Got2}]} = _Returned} = + {ok, {_Cols, [{Got1, Got2}]}} = riakc_ts:query(C, Query, [], undefined, []), ct:log("Query \"~s\"", [Query]), case Got1 == Got2 of true -> ok; _ -> - ct:fail("Got {~p, ~p}, Need equal\n", [Got1, Got2]) + ct:fail("MEDIAN: Got {~p, ~p}, Need equal\n", [Got1, Got2]) end end, ["a", "b", "c", "d", "e"]). + +query_invdist_mode(Cfg) -> + C = rt:pbc(hd(proplists:get_value(cluster, Cfg))), + Data = proplists:get_value(data, Cfg), + lists:foreach( + fun(Col) -> + SortedData = order_by(Col, Data), + Query = make_query(fmt("mode(~s)", [Col])), + {ok, {_Cols, [{Got}]}} = + riakc_ts:query(C, Query, [], undefined, []), + ct:log("Query \"~s\"", [Query]), + Need = mode(Col, SortedData), + case Got == Need of + true -> + ok; + _ -> + ct:fail("MODE: Got ~p, Need ~p\n", [Got, Need]) + end + end, + ["a", "b", "c", "d", "e"]). + + query_invdist_errors(Cfg) -> lists:foreach( fun({Select, Extra, ErrPat}) -> @@ -173,6 +198,64 @@ query_invdist_errors(Cfg) -> "Inverse distribution functions cannot be used with any of ORDER BY, LIMIT or OFFSET clauses"}]). +%% tested functions implememnted independently +%% ------------------------------- + +percentile_disc(F, Pc, Data_) -> + Col = remove_nulls(F, Data_), + RN = (1 + (Pc * (length(Col) - 1))), + lists:nth(trunc(RN), Col). + +percentile_cont(F, Pc, Data_) -> + Col = remove_nulls(F, Data_), + RN = (1 + (Pc * (length(Col) - 1))), + {LoRN, HiRN} = {trunc(RN), ceil(RN)}, + case LoRN == HiRN of + true -> + lists:nth(LoRN, Col); + false -> + LoVal = lists:nth(LoRN, Col), + HiVal = lists:nth(HiRN, Col), + (HiRN - RN) * LoVal + (RN - LoRN) * HiVal + end. + +ceil(X) -> + T = trunc(X), + case X - T == 0 of + true -> T; + false -> T + 1 + end. + +mode(F, Data_) -> + Col = remove_nulls(F, Data_), + Min = lists:nth(1, Col), + largest_bin(Min, Col). + +remove_nulls([FChar|_], Data_) -> + FNo = 1 + (FChar - $a), + [element(FNo, D) || D <- Data_, element(FNo, D) /= []]. + +largest_bin(X1, Data) -> + largest_bin_({X1, 1, X1, 1}, 2, Data). + +largest_bin_({LargestV, _, _, _}, Pos, Data) when Pos > length(Data) -> + LargestV; +largest_bin_({LargestV, LargestC, CurrentV, CurrentC}, Pos, Data) -> + case lists:nth(Pos, Data) of + V when V == CurrentV -> + largest_bin_({LargestV, LargestC, + CurrentV, CurrentC + 1}, Pos + 1, Data); + V when V > CurrentV, + CurrentC > LargestC -> + largest_bin_({CurrentV, CurrentC, %% now these be largest + V, 1}, Pos + 1, Data); + V when V > CurrentV, + CurrentC =< LargestC -> + largest_bin_({LargestV, LargestC, %% keep largest, reset current + V, 1}, Pos + 1, Data) + end. + + %% supporting functions %% ------------------------------- @@ -237,7 +320,7 @@ check_column(C, Col, Data) -> fun({PercentileFun, Pc, Pc_s}) -> Query = make_query(fmt("~s(~s, ~s)", [PercentileFun, Col, Pc_s])), ct:log("Query \"~s\"", [Query]), - {ok, {_Cols, [{Got}]} = _Returned} = + {ok, {_Cols, [{Got}]}} = riakc_ts:query(C, Query, [], undefined, []), Need = apply(?MODULE, PercentileFun, [Col, Pc, SortedData]), case Got == Need of @@ -257,33 +340,8 @@ order_by([FChar|_], Data) -> end, Data). -percentile_disc([FChar|_], Pc, Data_) -> - FNo = 1 + (FChar - $a), - Data = [D || D <- Data_, element(FNo, D) /= []], - RN = (1 + (Pc * (length(Data) - 1))), - Row = lists:nth(trunc(RN), Data), - element(FNo, Row). - -percentile_cont([FChar|_], Pc, Data_) -> - FNo = 1 + (FChar - $a), - Data = [D || D <- Data_, element(FNo, D) /= []], - RN = (1 + (Pc * (length(Data) - 1))), - {LoRN, HiRN} = {trunc(RN), ceil(RN)}, - case LoRN == HiRN of - true -> - element(FNo, lists:nth(LoRN, Data)); - false -> - LoVal = element(FNo, lists:nth(LoRN, Data)), - HiVal = element(FNo, lists:nth(HiRN, Data)), - (HiRN - RN) * LoVal + (RN - LoRN) * HiVal - end. -ceil(X) -> - T = trunc(X), - case X - T == 0 of - true -> T; - false -> T + 1 - end. +%% intercepts load_intercept(Node, C, Intercept) -> ok = rt_intercept:add(Node, Intercept), From 868d85dc931d231214ff85a36dc1a90c9626dcb4 Mon Sep 17 00:00:00 2001 From: Jenkins Date: Fri, 17 Feb 2017 00:19:22 -0800 Subject: [PATCH 09/35] Jenkins - update locked-deps for riak_ts_ee-1.6.0nightly20170217 --- rebar.config.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rebar.config.lock b/rebar.config.lock index 940c54ea3..147ec1798 100644 --- a/rebar.config.lock +++ b/rebar.config.lock @@ -31,7 +31,7 @@ "6c67dc58251e1a1c29b2e0c444b1e9c9aa2d4a24"}}, {riakc,".*", {git,"https://github.com/basho/riak-erlang-client", - "82ae32bd28090ab911b5c25d2be690c7d2a3e027"}}, + "8ae580b98446b0e0a2303c2101ab43830ead7717"}}, {ibrowse,".*", {git,"https://github.com/basho/ibrowse.git", "b28542d1e326ba44bcfaf7fd6d3c7f8761d20f08"}}, From e689e42d706dae824013225b2eca75baed79042c Mon Sep 17 00:00:00 2001 From: Andrei Zavada Date: Sat, 18 Feb 2017 01:10:51 +0200 Subject: [PATCH 10/35] ts_simple_invdist_funs: externally verify percentile calculations --- tests/ts_simple_invdist_funs_SUITE.erl | 132 ++++++++++++++++++++++++- 1 file changed, 131 insertions(+), 1 deletion(-) diff --git a/tests/ts_simple_invdist_funs_SUITE.erl b/tests/ts_simple_invdist_funs_SUITE.erl index f20c5ba5b..3bffc7812 100644 --- a/tests/ts_simple_invdist_funs_SUITE.erl +++ b/tests/ts_simple_invdist_funs_SUITE.erl @@ -22,7 +22,8 @@ -module(ts_simple_invdist_funs_SUITE). -export([suite/0, init_per_suite/1, groups/0, all/0]). --export([query_invdist_percentile/1, +-export([query_invdist_selftest/1, + query_invdist_percentile/1, query_invdist_percentile_backends/1, query_invdist_percentile_multiple/1, query_invdist_median/1, @@ -53,6 +54,7 @@ groups() -> all() -> [ + query_invdist_selftest, query_invdist_percentile, query_invdist_percentile_multiple, query_invdist_percentile_backends, @@ -65,6 +67,22 @@ all() -> %% test cases %% ------------------------------- +query_invdist_selftest(_Cfg) -> + Data = + [{X} || X <- lists:sort(pseudo_random_data())], + {GotDisc, GotCont} = + lists:unzip( + [{round(100 * percentile_disc("a", Pc/100, Data)), + round(100 * percentile_cont("a", Pc/100, Data))} || Pc <- lists:seq(1, 97)]), + {NeedDisc, NeedCont} = + lists:unzip( + [{round(X * 100), round(Y * 100)} || {X, Y} <- externally_verified_percentiles()]), + io:format("~p ~p\n", [length(GotDisc), length(NeedDisc)]), + ?assertEqual(GotDisc, NeedDisc), + ?assertEqual(GotCont, NeedCont). + + + query_invdist_percentile(Cfg) -> C = rt:pbc(hd(proplists:get_value(cluster, Cfg))), Data = proplists:get_value(data, Cfg), @@ -366,3 +384,115 @@ wait_until_qbuf_mgr_reinit(C) -> fmt(F, A) -> lists:flatten(io_lib:format(F, A)). + +pseudo_random_data() -> + [float(X) || X <- [1,2,3,4,5,7,0,11,-4,2,2,19,3,11,17,4,9]]. + +externally_verified_percentiles() -> + %% these are generated with this script: + %% #!/bin/env python + %% import numpy as np + %% a = np.array([1,2,3,4,5,7,0,11,-4,2,2,19,3,11,17,4,9]) + %% for p in range(1,99): + %% disc = np.percentile(a, p, interpolation='lower') + %% cont = np.percentile(a, p, interpolation='linear') + %% print("{0:.3f}, {1:.3f},".format(disc, cont)) + + [{-4.000, -3.360}, + {-4.000, -2.720}, + {-4.000, -2.080}, + {-4.000, -1.440}, + {-4.000, -0.800}, + {-4.000, -0.160}, + {0.000, 0.120}, + {0.000, 0.280}, + {0.000, 0.440}, + {0.000, 0.600}, + {0.000, 0.760}, + {0.000, 0.920}, + {1.000, 1.080}, + {1.000, 1.240}, + {1.000, 1.400}, + {1.000, 1.560}, + {1.000, 1.720}, + {1.000, 1.880}, + {2.000, 2.000}, + {2.000, 2.000}, + {2.000, 2.000}, + {2.000, 2.000}, + {2.000, 2.000}, + {2.000, 2.000}, + {2.000, 2.000}, + {2.000, 2.000}, + {2.000, 2.000}, + {2.000, 2.000}, + {2.000, 2.000}, + {2.000, 2.000}, + {2.000, 2.000}, + {2.000, 2.120}, + {2.000, 2.280}, + {2.000, 2.440}, + {2.000, 2.600}, + {2.000, 2.760}, + {2.000, 2.920}, + {3.000, 3.000}, + {3.000, 3.000}, + {3.000, 3.000}, + {3.000, 3.000}, + {3.000, 3.000}, + {3.000, 3.000}, + {3.000, 3.040}, + {3.000, 3.200}, + {3.000, 3.360}, + {3.000, 3.520}, + {3.000, 3.680}, + {3.000, 3.840}, + {4.000, 4.000}, + {4.000, 4.000}, + {4.000, 4.000}, + {4.000, 4.000}, + {4.000, 4.000}, + {4.000, 4.000}, + {4.000, 4.000}, + {4.000, 4.120}, + {4.000, 4.280}, + {4.000, 4.440}, + {4.000, 4.600}, + {4.000, 4.760}, + {4.000, 4.920}, + {5.000, 5.160}, + {5.000, 5.480}, + {5.000, 5.800}, + {5.000, 6.120}, + {5.000, 6.440}, + {5.000, 6.760}, + {7.000, 7.080}, + {7.000, 7.400}, + {7.000, 7.720}, + {7.000, 8.040}, + {7.000, 8.360}, + {7.000, 8.680}, + {9.000, 9.000}, + {9.000, 9.320}, + {9.000, 9.640}, + {9.000, 9.960}, + {9.000, 10.280}, + {9.000, 10.600}, + {9.000, 10.920}, + {11.000, 11.000}, + {11.000, 11.000}, + {11.000, 11.000}, + {11.000, 11.000}, + {11.000, 11.000}, + {11.000, 11.000}, + {11.000, 11.480}, + {11.000, 12.440}, + {11.000, 13.400}, + {11.000, 14.360}, + {11.000, 15.320}, + {11.000, 16.280}, + {17.000, 17.080}, + {17.000, 17.400}, + {17.000, 17.720}, + {17.000, 18.040} + ]. From 94bbcc5d87463837806840270d40e83b3e20aec7 Mon Sep 17 00:00:00 2001 From: Andrei Zavada Date: Mon, 20 Feb 2017 22:39:49 +0200 Subject: [PATCH 11/35] rtdev-setup-releases: fix compatibility issues with older git versions --- bin/rtdev-setup-releases.sh | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/bin/rtdev-setup-releases.sh b/bin/rtdev-setup-releases.sh index d67759b3e..d78f0c83a 100755 --- a/bin/rtdev-setup-releases.sh +++ b/bin/rtdev-setup-releases.sh @@ -38,12 +38,13 @@ fi cd $RT_DEST_DIR git init -## Some versions of git and/or OS require these fields -git config --local user.name "Riak Test" -git config --local user.email "dev@basho.com" -git config --local core.autocrlf input -git config --local core.safecrlf false -git config --local core.filemode true +if [ -z "$(git config --list 2>&1 | grep unknown)" ]; then + git config --local user.name "Riak Test" + git config --local user.email "dev@basho.com" + git config --local core.autocrlf input + git config --local core.safecrlf false + git config --local core.filemode true +fi ## this prevents priv/*.so files from being deleted by git clean -fd ## (the latter is executed in rtdev-current.sh): From 659f26f8a381636b23dd7c8b63a7ebd7cc4ab629 Mon Sep 17 00:00:00 2001 From: Andrei Zavada Date: Tue, 21 Feb 2017 18:13:35 +0200 Subject: [PATCH 12/35] further simplify and explain what we do git config for --- bin/rtdev-setup-releases.sh | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/bin/rtdev-setup-releases.sh b/bin/rtdev-setup-releases.sh index d78f0c83a..ff3654213 100755 --- a/bin/rtdev-setup-releases.sh +++ b/bin/rtdev-setup-releases.sh @@ -35,16 +35,19 @@ else echo "No devdirs found. Not copying any releases." fi -cd $RT_DEST_DIR -git init - -if [ -z "$(git config --list 2>&1 | grep unknown)" ]; then - git config --local user.name "Riak Test" - git config --local user.email "dev@basho.com" - git config --local core.autocrlf input - git config --local core.safecrlf false - git config --local core.filemode true -fi +cd "$RT_DEST_DIR" +git init + +# Set the path to the root of the working tree. This prevents git to +# ascend the directory tree (and possibly mess with other +# repositories). That is, changes we are going to make with `git +# config` will only affect the rt repo we have just created. +GIT_WORK_TREE="$RT_DEST_DIR" +git config user.name "Riak Test" +git config user.email "dev@basho.com" +git config core.autocrlf input +git config core.safecrlf false +git config core.filemode true ## this prevents priv/*.so files from being deleted by git clean -fd ## (the latter is executed in rtdev-current.sh): From 0927f834be45e235d30235027b8716c4c567c517 Mon Sep 17 00:00:00 2001 From: Jenkins Date: Thu, 23 Feb 2017 00:19:19 -0800 Subject: [PATCH 13/35] Jenkins - update locked-deps for riak_ts_ee-1.6.0nightly20170223 --- rebar.config.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rebar.config.lock b/rebar.config.lock index 147ec1798..8916829ad 100644 --- a/rebar.config.lock +++ b/rebar.config.lock @@ -31,7 +31,7 @@ "6c67dc58251e1a1c29b2e0c444b1e9c9aa2d4a24"}}, {riakc,".*", {git,"https://github.com/basho/riak-erlang-client", - "8ae580b98446b0e0a2303c2101ab43830ead7717"}}, + "87958bdf1926d752fb1319b6788330176f6dd591"}}, {ibrowse,".*", {git,"https://github.com/basho/ibrowse.git", "b28542d1e326ba44bcfaf7fd6d3c7f8761d20f08"}}, From eb3cf628ccd05f8f6005ecd8912076454852ae36 Mon Sep 17 00:00:00 2001 From: Brett Hazen Date: Fri, 24 Feb 2017 10:00:46 -0700 Subject: [PATCH 14/35] Lock deps for riak_ts_ee-1.6.0d --- devrel_matrix.yml => basho_builds.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename devrel_matrix.yml => basho_builds.yml (95%) diff --git a/devrel_matrix.yml b/basho_builds.yml similarity index 95% rename from devrel_matrix.yml rename to basho_builds.yml index 70113c99a..1d2c4e505 100644 --- a/devrel_matrix.yml +++ b/basho_builds.yml @@ -3,7 +3,7 @@ docker_rt_riak_test_config: product_version: current product: "riak_ts{% if ee == 'true' %}_ee{% endif %}" - version: previous - product_version: 1.5.1 + product_version: 1.5.2 product: "riak_ts{% if ee == 'true' %}_ee{% endif %}" - version: legacy product_version: 1.4.1 From 958e6289fb66964beb34440607d457b1dbe718f4 Mon Sep 17 00:00:00 2001 From: Andrei Zavada Date: Mon, 27 Feb 2017 17:34:51 +0200 Subject: [PATCH 15/35] ts_simple_invdist_funs: fix error messages to match updates to riak_kv --- tests/ts_simple_invdist_funs_SUITE.erl | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/ts_simple_invdist_funs_SUITE.erl b/tests/ts_simple_invdist_funs_SUITE.erl index 3bffc7812..700fa2102 100644 --- a/tests/ts_simple_invdist_funs_SUITE.erl +++ b/tests/ts_simple_invdist_funs_SUITE.erl @@ -193,27 +193,27 @@ query_invdist_errors(Cfg) -> [{"percentile_disc(nxcol, 0.2)", [], "Unknown column \"nxcol\""}, {"median(a), b", [], - "Inverse distribution functions cannot be used with other columns in SELECT clause"}, + "Inverse distribution functions .* cannot be used with other columns in SELECT clause"}, {"percentile_disc(1)", [], "Function PERCENTILE_DISC/1 called with 1 argument"}, {"percentile_disc(a, 1.2)", [], "Invalid argument 2 in call to function PERCENTILE_DISC"}, {"percentile_disc(a, 0.1), percentile_disc(b, 0.2)", [], - "Multiple inverse distribution functions must all have the same column argument"}, + "Multiple inverse distribution functions .* must all have the same column argument"}, {"percentile_disc(a, 1+c)", [], "Function 'PERCENTILE_DISC' called with arguments of the wrong type"}, {"percentile_disc(a, 1.1+c)", [], - "Non-const expression passed as parameter for inverse distribution function"}, + "Inverse distribution functions .* must have a static const expression for its parameters"}, {"percentile_disc(a, 1.1/0)", [], "Invalid expression passed as parameter for inverse distribution function"}, {"percentile_disc(a, 0.3), avg(b)", [], - "Inverse distribution functions cannot be used with GROUP BY clause or aggregating window functions"}, + "Inverse distribution functions .* cannot be used with GROUP BY clause or other aggregating window functions"}, {"percentile_disc(a, 0.1)", "group by a", - "Inverse distribution functions cannot be used with GROUP BY clause or aggregating window functions"}, + "Inverse distribution functions .* cannot be used with GROUP BY clause or other aggregating window functions"}, {"percentile_disc(a, 0.1)", "order by a", - "Inverse distribution functions cannot be used with any of ORDER BY, LIMIT or OFFSET clauses"}, + "Inverse distribution functions .* cannot be used with any of ORDER BY, LIMIT or OFFSET clauses"}, {"percentile_disc(a, 0.1)", "limit 1", - "Inverse distribution functions cannot be used with any of ORDER BY, LIMIT or OFFSET clauses"}]). + "Inverse distribution functions .* cannot be used with any of ORDER BY, LIMIT or OFFSET clauses"}]). %% tested functions implememnted independently From aa4b67262071236fa95141c6733e1f4c508e77ce Mon Sep 17 00:00:00 2001 From: Andrei Zavada Date: Mon, 27 Feb 2017 18:06:28 +0200 Subject: [PATCH 16/35] ts_simple_invdist_funs: also test on tables with desc keys --- tests/ts_simple_invdist_funs_SUITE.erl | 56 ++++++++++++++++++-------- 1 file changed, 39 insertions(+), 17 deletions(-) diff --git a/tests/ts_simple_invdist_funs_SUITE.erl b/tests/ts_simple_invdist_funs_SUITE.erl index 700fa2102..5eae61680 100644 --- a/tests/ts_simple_invdist_funs_SUITE.erl +++ b/tests/ts_simple_invdist_funs_SUITE.erl @@ -36,6 +36,8 @@ -include_lib("eunit/include/eunit.hrl"). -include("ts_qbuf_util.hrl"). +-define(TABLE_A, "table_with_regular_keys"). +-define(TABLE_D, "table_with_descending_keys"). suite() -> [{timetrap, {minutes, 5}}]. @@ -44,8 +46,10 @@ init_per_suite(Cfg) -> Cluster = ts_setup:start_cluster(1), C = rt:pbc(hd(Cluster)), Data = make_data(), - ok = create_table(C, ?TABLE), - ok = insert_data(C, ?TABLE, Data), + ok = create_table(C, ?TABLE_A), + ok = create_table_desc(C, ?TABLE_D), + ok = insert_data(C, ?TABLE_A, Data), + ok = insert_data(C, ?TABLE_D, Data), [{cluster, Cluster}, {data, Data} | Cfg]. @@ -185,7 +189,7 @@ query_invdist_mode(Cfg) -> query_invdist_errors(Cfg) -> lists:foreach( fun({Select, Extra, ErrPat}) -> - Qry = make_query(Select, Extra), + Qry = make_query(?TABLE_A, Select, Extra), ?assertEqual( ok, ts_qbuf_util:ack_query_error(Cfg, Qry, ?E_SUBMIT, ErrPat)) @@ -289,6 +293,18 @@ create table " ++ Table ++ " {ok, {[], []}} = riakc_ts:query(Client, DDL), ok. +create_table_desc(Client, Table) -> + DDL = " +create table " ++ Table ++ " +(a timestamp not null, + b double, + c sint64, + d sint64, + e sint64, + primary key ((quantum(a, 10, h)), a desc))", + {ok, {[], []}} = riakc_ts:query(Client, DDL), + ok. + data_generator(I) -> {?TIMEBASE + (I + 1) * 1000, 100 * math:cos(float(I) / 10 * math:pi()), @@ -313,10 +329,12 @@ insert_data(C, Table, Data) -> end. make_query(Select) -> - make_query(Select, ""). -make_query(Select, Extra) -> + make_query(?TABLE_A, Select, ""). +make_query(Table, Select) -> + make_query(Table, Select, ""). +make_query(Table, Select, Extra) -> fmt("select ~s from ~s where a >= ~b and a <= ~b~s", - [Select, ?TABLE, + [Select, Table, ?TIMEBASE, ?TIMEBASE * 1000, [" " ++ Extra || Extra /= []]]). @@ -334,19 +352,23 @@ check_column(C, Col, Data) -> {0.36, "(3.6/10)"}, {0.40, "0.5 - 1 * 0.1"}], PercentileVariety <- [percentile_cont, percentile_disc]], + Checker = + fun(Q, Need) -> + ct:log("Query \"~s\"", [Q]), + {ok, {_Cols, [{Got}]}} = + riakc_ts:query(C, Q, [], undefined, []), + case Got == Need of + true -> + ok; + false -> + ct:fail("Got ~p, Need ~p\n", [Got, Need]) + end + end, ok == lists:foreach( fun({PercentileFun, Pc, Pc_s}) -> - Query = make_query(fmt("~s(~s, ~s)", [PercentileFun, Col, Pc_s])), - ct:log("Query \"~s\"", [Query]), - {ok, {_Cols, [{Got}]}} = - riakc_ts:query(C, Query, [], undefined, []), Need = apply(?MODULE, PercentileFun, [Col, Pc, SortedData]), - case Got == Need of - true -> - ok; - false -> - ct:fail("Got ~p, Need ~p\n", [Got, Need]) - end + ok = Checker(make_query(?TABLE_A, fmt("~s(~s, ~s)", [PercentileFun, Col, Pc_s])), Need), + ok = Checker(make_query(?TABLE_D, fmt("~s(~s, ~s)", [PercentileFun, Col, Pc_s])), Need) end, Combos). @@ -370,7 +392,7 @@ load_intercept(Node, C, Intercept) -> wait_until_qbuf_mgr_reinit(C). wait_until_qbuf_mgr_reinit(C) -> - ProbingQuery = make_query("*", ""), + ProbingQuery = make_query("*"), case riakc_ts:query(C, ProbingQuery, [], undefined, []) of {ok, _Data} -> ok; From 5482f670c29821f0f88204aa38644f9a88d4df71 Mon Sep 17 00:00:00 2001 From: Brett Hazen Date: Mon, 27 Feb 2017 11:34:49 -0700 Subject: [PATCH 17/35] Tweaks devrel scripts to work with different versions of git --- bin/rtdev-install.sh | 20 +++++++++++--------- bin/rtdev-setup-releases.sh | 25 +++++++++++++------------ 2 files changed, 24 insertions(+), 21 deletions(-) diff --git a/bin/rtdev-install.sh b/bin/rtdev-install.sh index b367636da..9df8e1329 100755 --- a/bin/rtdev-install.sh +++ b/bin/rtdev-install.sh @@ -46,15 +46,17 @@ cd $RT_DEST_DIR echo " - Reinitializing git state" -## Some versions of git and/or OS require these fields -HAS_LOCAL=$(git config --local 2>&1 | grep unknown) -if [ -z "$HAS_LOCAL" ]; then - git config --local user.name "Riak Test" - git config --local user.email "dev@basho.com" - git config --local core.autocrlf input - git config --local core.safecrlf false - git config --local core.filemode true -fi +# Set the path to the root of the working tree. This prevents git to +# ascend the directory tree (and possibly mess with other +# repositories). That is, changes we are going to make with `git +# config` will only affect the rt repo we have just created. +GIT_WORK_TREE="$RT_DEST_DIR" +git config user.name "Riak Test" +git config user.email "dev@basho.com" +git config core.autocrlf input +git config core.safecrlf false +git config core.filemode true + git add --all --force . git commit -a -m "riak_test init" --amend > /dev/null diff --git a/bin/rtdev-setup-releases.sh b/bin/rtdev-setup-releases.sh index 4b958b417..8f3af0bcd 100755 --- a/bin/rtdev-setup-releases.sh +++ b/bin/rtdev-setup-releases.sh @@ -35,18 +35,19 @@ else echo "No devdirs found. Not copying any releases." fi -cd $RT_DEST_DIR -git init - -## Some versions of git and/or OS require these fields -HAS_LOCAL=$(git config --local 2>&1 | grep unknown) -if [ -z "$HAS_LOCAL" ]; then - git config --local user.name "Riak Test" - git config --local user.email "dev@basho.com" - git config --local core.autocrlf input - git config --local core.safecrlf false - git config --local core.filemode true -fi +cd "$RT_DEST_DIR" +git init + +# Set the path to the root of the working tree. This prevents git to +# ascend the directory tree (and possibly mess with other +# repositories). That is, changes we are going to make with `git +# config` will only affect the rt repo we have just created. +GIT_WORK_TREE="$RT_DEST_DIR" +git config user.name "Riak Test" +git config user.email "dev@basho.com" +git config core.autocrlf input +git config core.safecrlf false +git config core.filemode true ## this prevents priv/*.so files from being deleted by git clean -fd ## (the latter is executed in rtdev-current.sh): From 5f2ef6f576248b805642556a3d86871612858661 Mon Sep 17 00:00:00 2001 From: Brett Hazen Date: Tue, 21 Feb 2017 21:01:13 -0700 Subject: [PATCH 18/35] Move YZ tests to yokozuna repo --- intercepts/yz_noop_extractor_intercepts.erl | 44 -- intercepts/yz_solrq_helper_intercepts.erl | 48 -- src/yokozuna_rt.erl | 563 ------------------- tests/yz_core_properties_create_unload.erl | 170 ------ tests/yz_crdt.erl | 564 -------------------- tests/yz_default_bucket_type_upgrade.erl | 98 ---- tests/yz_ensemble.erl | 117 ---- tests/yz_extractors.erl | 441 --------------- tests/yz_handoff.erl | 206 ------- tests/yz_schema_change_reset.erl | 305 ----------- 10 files changed, 2556 deletions(-) delete mode 100644 intercepts/yz_noop_extractor_intercepts.erl delete mode 100644 intercepts/yz_solrq_helper_intercepts.erl delete mode 100644 src/yokozuna_rt.erl delete mode 100644 tests/yz_core_properties_create_unload.erl delete mode 100644 tests/yz_crdt.erl delete mode 100644 tests/yz_default_bucket_type_upgrade.erl delete mode 100644 tests/yz_ensemble.erl delete mode 100644 tests/yz_extractors.erl delete mode 100644 tests/yz_handoff.erl delete mode 100644 tests/yz_schema_change_reset.erl diff --git a/intercepts/yz_noop_extractor_intercepts.erl b/intercepts/yz_noop_extractor_intercepts.erl deleted file mode 100644 index ac7146289..000000000 --- a/intercepts/yz_noop_extractor_intercepts.erl +++ /dev/null @@ -1,44 +0,0 @@ -%% ------------------------------------------------------------------- -%% -%% Copyright (c) 2015 Basho Technologies, Inc. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%%------------------------------------------------------------------- - -%% Example from: -%% http://docs.basho.com/riak/latest/dev/search/custom-extractors/#An-Example-Custom-Extractor - --module(yz_noop_extractor_intercepts). --compile(export_all). --include("intercept.hrl"). - -extract_httpheader(Value) -> - extract_httpheader(Value, []). - -extract_httpheader(Value, _Opts) -> - {ok, - {http_request, - Method, - {absoluteURI, http, Host, undefined, Uri}, - _Version}, - _Rest} = erlang:decode_packet(http, Value, []), - [{method, Method}, {host, list_to_binary(Host)}, {uri, list_to_binary(Uri)}]. - -extract_non_unicode_data(Value) -> - extract_non_unicode_data(Value, []). - -extract_non_unicode_data(_Value, _Opts) -> - [{blob, <<9147374713>>}]. diff --git a/intercepts/yz_solrq_helper_intercepts.erl b/intercepts/yz_solrq_helper_intercepts.erl deleted file mode 100644 index 926929d0f..000000000 --- a/intercepts/yz_solrq_helper_intercepts.erl +++ /dev/null @@ -1,48 +0,0 @@ -%% ------------------------------------------------------------------- -%% -%% Copyright (c) 2015 Basho Technologies, Inc. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%%------------------------------------------------------------------- --module(yz_solrq_helper_intercepts). --compile(export_all). - --include("intercept.hrl"). - --define(M, yz_solrq_helper_orig). - -handle_get_ops_for_no_sibling_deletes(LI, P, Obj) -> - Lookup = ets:lookup(intercepts_tab, del_put), - case Lookup of - [] -> original_get_ops_for_no_sibling_deletes(LI, P, Obj); - _ -> - case proplists:get_value(del_put, Lookup) of - 0 -> - error_logger:info_msg( - "Delete operation intercepted for BKey ~p", - [{riak_object:bucket(Obj), riak_object:key(Obj)}]), - ets:update_counter(intercepts_tab, del_put, 1), - []; - _ -> - original_get_ops_for_no_sibling_deletes(LI, P, Obj) - end - end. - -original_get_ops_for_no_sibling_deletes(LI, P, Obj) -> - error_logger:info_msg( - "Delete operation original for BKey ~p", - [{riak_object:bucket(Obj), riak_object:key(Obj)}]), - ?M:get_ops_for_no_sibling_deletes_orig(LI, P, Obj). diff --git a/src/yokozuna_rt.erl b/src/yokozuna_rt.erl deleted file mode 100644 index 21a492317..000000000 --- a/src/yokozuna_rt.erl +++ /dev/null @@ -1,563 +0,0 @@ -%% ------------------------------------------------------------------- -%% -%% Copyright (c) 2015 Basho Technologies, Inc. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%%------------------------------------------------------------------- --module(yokozuna_rt). - --include_lib("eunit/include/eunit.hrl"). --include("yokozuna_rt.hrl"). - --export([brutal_kill_remove_index_dirs/3, - check_exists/2, - clear_trees/1, - commit/2, - drain_solrqs/1, - expire_trees/1, - gen_keys/1, - host_entries/1, - create_indexed_bucket_type/3, - create_indexed_bucket_type/4, - override_schema/5, - remove_index_dirs/3, - rolling_upgrade/2, - search/4, - search/5, - search_expect/5, - search_expect/6, - search_expect/7, - assert_search/6, - verify_num_found_query/3, - wait_for_aae/1, - wait_for_full_exchange_round/2, - wait_for_index/2, - wait_for_schema/2, - wait_for_schema/3, - wait_until/2, - write_data/5, - write_data/6, - http/4, - http/5, - http/6]). - --type host() :: string(). --type portnum() :: integer(). --type count() :: non_neg_integer(). --type json_string() :: atom | string() | binary(). --type search_type() :: solr | yokozuna. --type method() :: get | post | head | options | put | delete | trace | - mkcol | propfind | proppatch | lock | unlock | move | copy. --type response() :: {ok, string(), [{string(), string()}], string()|binary()} | - {error, term()}. --type cluster() :: [node()]. - - --define(FMT(S, Args), lists:flatten(io_lib:format(S, Args))). --define(SOFTCOMMIT, 1000). - --spec host_entries(rt:conn_info()) -> [{host(), portnum()}]. -host_entries(ClusterConnInfo) -> - [riak_http(I) || {_,I} <- ClusterConnInfo]. - -%% @doc Generate `SeqMax' keys. Yokozuna supports only UTF-8 compatible keys. --spec gen_keys(pos_integer()) -> list(). -gen_keys(SeqMax) -> - [<> || N <- lists:seq(1, SeqMax), - not lists:any( - fun(E) -> E > 127 end, - binary_to_list(<>))]. - -%% @doc Write `Keys' via the PB inteface to a `Bucket' and have them -%% searchable in an `Index'. --spec write_data([node()], pid(), index_name(), bucket(), [binary()]) -> ok. -write_data(Cluster, Pid, Index, Bucket, Keys) -> - riakc_pb_socket:set_options(Pid, [queue_if_disconnected]), - - create_and_set_index(Cluster, Pid, Bucket, Index), - yokozuna_rt:commit(Cluster, Index), - - %% Write keys - lager:info("Writing ~p keys", [length(Keys)]), - [ok = rt:pbc_write(Pid, Bucket, Key, Key, "text/plain") || Key <- Keys], - ok. - --spec write_data([node()], pid(), index_name(), {schema_name(), raw_schema()}, - bucket(), [binary()]) -> ok. -write_data(Cluster, Pid, Index, {SchemaName, SchemaData}, - Bucket, Keys) -> - riakc_pb_socket:set_options(Pid, [queue_if_disconnected]), - - riakc_pb_socket:create_search_schema(Pid, SchemaName, SchemaData), - - create_and_set_index(Cluster, Pid, Bucket, Index, SchemaName), - yokozuna_rt:commit(Cluster, Index), - - %% Write keys - lager:info("Writing ~p keys", [length(Keys)]), - [ok = rt:pbc_write(Pid, Bucket, Key, Key, "text/plain") || Key <- Keys], - ok. - -%% @doc Peform a rolling upgrade of the `Cluster' to a different `Version' based -%% on current | previous | legacy. --spec rolling_upgrade([node()], current | previous | legacy) -> ok. -rolling_upgrade(Cluster, Version) -> - rolling_upgrade(Cluster, Version, same, [riak_kv, yokozuna]). --spec rolling_upgrade([node()], current | previous | legacy, [term()] | same, [atom()]) -> ok. -rolling_upgrade(Cluster, Version, UpgradeConfig, WaitForServices) when is_list(Cluster) -> - lager:info("Perform rolling upgrade on cluster ~p", [Cluster]), - [rolling_upgrade(Node, Version, UpgradeConfig, WaitForServices) || Node <- Cluster], - ok; -rolling_upgrade(Node, Version, UpgradeConfig, WaitForServices) -> - rt:upgrade(Node, Version, UpgradeConfig), - [rt:wait_for_service(Node, Service) || Service <- WaitForServices], - ok. - -%% @doc Use AAE status to verify that exchange has occurred for all -%% partitions since the time this function was invoked. --spec wait_for_aae([node()]) -> ok. -wait_for_aae(Cluster) -> - lager:info("Wait for AAE to migrate/repair indexes"), - wait_for_all_trees(Cluster), - wait_for_full_exchange_round(Cluster, erlang:now()), - ok. - -%% @doc Wait for all AAE trees to be built. --spec wait_for_all_trees([node()]) -> ok. -wait_for_all_trees(Cluster) -> - F = fun(Node) -> - lager:info("Check if all trees built for node ~p", [Node]), - Info = rpc:call(Node, yz_kv, compute_tree_info, []), - NotBuilt = [X || {_,undefined}=X <- Info], - NotBuilt == [] - end, - wait_until(Cluster, F), - ok. - -%% @doc Wait for a full exchange round since `Timestamp'. This means -%% that all `{Idx,N}' for all partitions must have exchanged after -%% `Timestamp'. --spec wait_for_full_exchange_round([node()], os:now()) -> ok. -wait_for_full_exchange_round(Cluster, Timestamp) -> - lager:info("wait for full AAE exchange round on cluster ~p", [Cluster]), - MoreRecent = - fun({_Idx, _, undefined, _RepairStats}) -> - false; - ({_Idx, _, AllExchangedTime, _RepairStats}) -> - AllExchangedTime > Timestamp - end, - AllExchanged = - fun(Node) -> - Exchanges = rpc:call(Node, yz_kv, compute_exchange_info, []), - {_Recent, WaitingFor1} = lists:partition(MoreRecent, Exchanges), - WaitingFor2 = [element(1,X) || X <- WaitingFor1], - lager:info("Still waiting for AAE of ~p ~p", [Node, WaitingFor2]), - [] == WaitingFor2 - end, - wait_until(Cluster, AllExchanged), - ok. - -%% @doc Wait for index creation. This is to handle *legacy* versions of yokozuna -%% in upgrade tests --spec wait_for_index(list(), index_name()) -> ok. -wait_for_index(Cluster, Index) -> - IsIndexUp = - fun(Node) -> - lager:info("Waiting for index ~s to be avaiable on node ~p", - [Index, Node]), - rpc:call(Node, yz_index, exists, [Index]) - end, - wait_until(Cluster, IsIndexUp), - ok. - -%% @see wait_for_schema/3 -wait_for_schema(Cluster, Name) -> - wait_for_schema(Cluster, Name, ignore). - -%% @doc Wait for the schema `Name' to be read by all nodes in -%% `Cluster' before returning. If `Content' is binary data when -%% verify the schema bytes exactly match `Content'. --spec wait_for_schema([node()], schema_name(), ignore | raw_schema()) -> ok. -wait_for_schema(Cluster, Name, Content) -> - F = fun(Node) -> - lager:info("Attempt to read schema ~s from node ~p", - [Name, Node]), - {Host, Port} = riak_pb(hd(rt:connection_info([Node]))), - {ok, PBConn} = riakc_pb_socket:start_link(Host, Port), - R = riakc_pb_socket:get_search_schema(PBConn, Name), - riakc_pb_socket:stop(PBConn), - case R of - {ok, PL} -> - case Content of - ignore -> - Name == proplists:get_value(name, PL); - _ -> - (Name == proplists:get_value(name, PL)) and - (Content == proplists:get_value(content, PL)) - end; - _ -> - false - end - end, - wait_until(Cluster, F), - ok. - -%% @doc Expire YZ trees --spec expire_trees([node()]) -> ok. -expire_trees(Cluster) -> - lager:info("Expire all trees"), - _ = [ok = rpc:call(Node, yz_entropy_mgr, expire_trees, []) - || Node <- Cluster], - - %% The expire is async so just give it a moment - timer:sleep(100), - ok. - -%% @doc Expire YZ trees --spec clear_trees([node()]) -> ok. -clear_trees(Cluster) -> - lager:info("Expire all trees"), - _ = [ok = rpc:call(Node, yz_entropy_mgr, clear_trees, []) - || Node <- Cluster], - ok. - -brutal_kill_remove_index_dirs(Nodes, IndexName, Services) -> - IndexDirs = get_index_dirs(IndexName, Nodes), - rt:brutal_kill(hd(Nodes)), - [rt:stop(ANode) || ANode <- tl(Nodes)], - remove_index_dirs2(Nodes, IndexDirs, Services), - ok. - -%% @doc Remove index directories, removing the index. --spec remove_index_dirs([node()], index_name(), [atom()]) -> ok. -remove_index_dirs(Nodes, IndexName, Services) -> - IndexDirs = get_index_dirs(IndexName, Nodes), - [rt:stop(ANode) || ANode <- Nodes], - remove_index_dirs2(Nodes, IndexDirs, Services), - ok. - -remove_index_dirs2(Nodes, IndexDirs, Services) -> - lager:info("Remove index dirs: ~p, on nodes: ~p~n", - [IndexDirs, Nodes]), - [rt:del_dir(binary_to_list(IndexDir)) || IndexDir <- IndexDirs], - [start_and_wait(ANode, Services) || ANode <- Nodes]. - -get_index_dirs(IndexName, Nodes) -> - IndexDirs = [rpc:call(Node, yz_index, index_dir, [IndexName]) || - Node <- Nodes], - IndexDirs. - -start_and_wait(Node, WaitForServices) -> - rt:start(Node), - [rt:wait_for_service(Node, Service) || Service <- WaitForServices]. - -%% @doc Check if index/core exists in metadata, disk via yz_index:exists. --spec check_exists([node()], index_name()) -> ok. -check_exists(Nodes, IndexName) -> - wait_until(Nodes, - fun(N) -> - rpc:call(N, yz_index, exists, [IndexName]) - end). - --spec verify_num_found_query([node()], index_name(), count()) -> ok. -verify_num_found_query(Cluster, Index, ExpectedCount) -> - F = fun(Node) -> - Pid = rt:pbc(Node), - {ok, {_, _, _, NumFound}} = riakc_pb_socket:search(Pid, Index, <<"*:*">>), - lager:info("Check Count, Expected: ~p | Actual: ~p~n", - [ExpectedCount, NumFound]), - ExpectedCount =:= NumFound - end, - wait_until(Cluster, F), - ok. - -%% @doc Brought over from yz_rt in the yokozuna repo - FORNOW. --spec search_expect(node()|[node()], index_name(), string(), string(), - non_neg_integer()) -> ok. -search_expect(NodeOrNodes, Index, Name, Term, Expect) -> - search_expect(NodeOrNodes, yokozuna, Index, Name, Term, Expect). - --spec search_expect(node()|[node()], search_type(), index_name(), - string(), string(), [string()], non_neg_integer()) -> ok. -search_expect(Nodes, solr, Index, Name0, Term0, Shards, Expect) - when is_list(Shards), length(Shards) > 0, is_list(Nodes) -> - Name = quote_unicode(Name0), - Term = quote_unicode(Term0), - Node = rt:select_random(Nodes), - {Host, Port} = solr_hp(Node, Nodes), - URL = internal_solr_url(Host, Port, Index, Name, Term, Shards), - lager:info("Run solr search ~s", [URL]), - Opts = [{response_format, binary}], - F = fun(_) -> - {ok, "200", _, R} = ibrowse:send_req(URL, [], get, [], Opts, - ?IBROWSE_TIMEOUT), - verify_count_http(Expect, R) - end, - wait_until(Nodes, F); -search_expect(Node, solr=Type, Index, Name, Term, Shards, Expect) - when is_list(Shards), length(Shards) > 0 -> - search_expect([Node], Type, Index, Name, Term, Shards, Expect). - --spec search_expect(node()|[node()], search_type(), index_name(), - string(), string(), non_neg_integer()) -> ok. -search_expect(Nodes, solr=Type, Index, Name, Term, Expect) when is_list(Nodes) -> - Node = rt:select_random(Nodes), - HP = solr_hp(Node, Nodes), - - %% F could actually be returned in a shared fun, but w/ so much arity, - %% just using it twice makes sense. - F = fun(_) -> - {ok, "200", _, R} = search(Type, HP, Index, Name, Term), - verify_count_http(Expect, R) - end, - - wait_until(Nodes, F); -search_expect(Nodes, yokozuna=Type, Index, Name, Term, Expect) - when is_list(Nodes) -> - HP = hd(host_entries(rt:connection_info(Nodes))), - - F = fun(_) -> - {ok, "200", _, R} = search(Type, HP, Index, Name, Term), - verify_count_http(Expect, R) - end, - - wait_until(Nodes, F); -search_expect(Node, Type, Index, Name, Term, Expect) -> - search_expect([Node], Type, Index, Name, Term, Expect). - -assert_search(Pid, Cluster, Index, Search, SearchExpect, Params) -> - F = fun(_) -> - lager:info("Searching ~p and asserting it exists", - [SearchExpect]), - case riakc_pb_socket:search(Pid, Index, Search, Params) of - {ok,{search_results,[{_Index,Fields}], _Score, Found}} -> - ?assert(lists:member(SearchExpect, Fields)), - case Found of - 1 -> true; - 0 -> false - end; - {ok, {search_results, [], _Score, 0}} -> - lager:info("Search has not yet yielded data"), - false - end - end, - wait_until(Cluster, F). - -search(HP, Index, Name, Term) -> - search(yokozuna, HP, Index, Name, Term). - -search(Type, {Host, Port}, Index, Name, Term) when is_integer(Port) -> - search(Type, {Host, integer_to_list(Port)}, Index, Name, Term); - -search(Type, {Host, Port}, Index, Name0, Term0) -> - Name = quote_unicode(Name0), - Term = quote_unicode(Term0), - FmtStr = case Type of - solr -> - "http://~s:~s/internal_solr/~s/select?q=~s:~s&wt=json"; - yokozuna -> - "http://~s:~s/search/query/~s?q=~s:~s&wt=json" - end, - URL = ?FMT(FmtStr, [Host, Port, Index, Name, Term]), - lager:info("Run search ~s", [URL]), - Opts = [{response_format, binary}], - ibrowse:send_req(URL, [], get, [], Opts, ?IBROWSE_TIMEOUT). - -%%%=================================================================== -%%% Private -%%%=================================================================== - --spec verify_count_http(count(), json_string()) -> boolean(). -verify_count_http(Expected, Resp) -> - Count = get_count_http(Resp), - lager:info("Expected: ~p, Actual: ~p", [Expected, Count]), - Expected =:= Count. - --spec get_count_http(json_string()) -> count(). -get_count_http(Resp) -> - Struct = mochijson2:decode(Resp), - kvc:path([<<"response">>, <<"numFound">>], Struct). - --spec riak_http({node(), rt:interfaces()} | rt:interfaces()) -> - {host(), portnum()}. -riak_http({_Node, ConnInfo}) -> - riak_http(ConnInfo); -riak_http(ConnInfo) -> - proplists:get_value(http, ConnInfo). - --spec riak_pb({node(), rt:interfaces()} | rt:interfaces()) -> - {host(), portnum()}. -riak_pb({_Node, ConnInfo}) -> - riak_pb(ConnInfo); -riak_pb(ConnInfo) -> - proplists:get_value(pb, ConnInfo). - --spec create_and_set_index([node()], pid(), bucket(), index_name()) -> ok. -create_and_set_index(Cluster, Pid, Bucket, Index) -> - %% Create a search index and associate with a bucket - lager:info("Create a search index ~s and associate it with bucket ~s", - [Index, Bucket]), - _ = riakc_pb_socket:create_search_index(Pid, Index), - %% For possible legacy upgrade reasons or general check around the cluster, - %% wrap create index in a wait - wait_for_index(Cluster, Index), - set_index(Pid, hd(Cluster), Bucket, Index). --spec create_and_set_index([node()], pid(), bucket(), index_name(), - schema_name()) -> ok. -create_and_set_index(Cluster, Pid, Bucket, Index, Schema) -> - %% Create a search index and associate with a bucket - lager:info("Create a search index ~s with a custom schema named ~s and " ++ - "associate it with bucket ~p", [Index, Schema, Bucket]), - _ = riakc_pb_socket:create_search_index(Pid, Index, Schema, []), - %% For possible legacy upgrade reasons or general check around the cluster, - %% wrap create index in a wait - wait_for_index(Cluster, Index), - set_index(Pid, hd(Cluster), Bucket, Index). - --spec set_index(pid(), node(), bucket(), index_name()) -> ok. -set_index(_Pid, Node, {BucketType, _Bucket}, Index) -> - lager:info("Create and activate map-based bucket type ~s and tie it to search_index ~s", - [BucketType, Index]), - rt:create_and_activate_bucket_type(Node, BucketType, [{search_index, Index}]); -set_index(Pid, _Node, Bucket, Index) -> - ok = riakc_pb_socket:set_search_index(Pid, Bucket, Index). - -internal_solr_url(Host, Port, Index) -> - ?FMT("http://~s:~B/internal_solr/~s", [Host, Port, Index]). -internal_solr_url(Host, Port, Index, Name, Term, Shards) -> - Ss = [internal_solr_url(Host, ShardPort, Index) - || {_, ShardPort} <- Shards], - ?FMT("http://~s:~B/internal_solr/~s/select?wt=json&q=~s:~s&shards=~s", - [Host, Port, Index, Name, Term, string:join(Ss, ",")]). - -quote_unicode(Value) -> - mochiweb_util:quote_plus(binary_to_list( - unicode:characters_to_binary(Value))). - --spec commit([node()], index_name()) -> ok. -commit(Nodes, Index) -> - %% Wait for yokozuna index to trigger, then force a commit - timer:sleep(?SOFTCOMMIT), - lager:info("Commit search writes to ~s at softcommit (default) ~p", - [Index, ?SOFTCOMMIT]), - rpc:multicall(Nodes, yz_solr, commit, [Index]), - ok. - --spec drain_solrqs(node() | cluster()) -> ok. -drain_solrqs(Cluster) when is_list(Cluster) -> - [drain_solrqs(Node) || Node <- Cluster]; -drain_solrqs(Node) -> - rpc:call(Node, yz_solrq_drain_mgr, drain, []), - ok. - --spec override_schema(pid(), [node()], index_name(), schema_name(), string()) -> - {ok, [node()]}. -override_schema(Pid, Cluster, Index, Schema, RawUpdate) -> - lager:info("Overwrite schema with updated schema"), - ok = riakc_pb_socket:create_search_schema(Pid, Schema, RawUpdate), - yokozuna_rt:wait_for_schema(Cluster, Schema, RawUpdate), - [Node|_] = Cluster, - {ok, _} = rpc:call(Node, yz_index, reload, [Index]). - -%% @doc Wrapper around `rt:wait_until' to verify `F' against multiple -%% nodes. The function `F' is passed one of the `Nodes' as -%% argument and must return a `boolean()' delcaring whether the -%% success condition has been met or not. --spec wait_until([node()], fun((node()) -> boolean())) -> ok. -wait_until(Nodes, F) -> - [?assertEqual(ok, rt:wait_until(Node, F)) || Node <- Nodes], - ok. - --spec solr_hp(node(), [node()]) -> {host(), portnum()}. -solr_hp(Node, Cluster) -> - CI = connection_info(Cluster), - solr_http(proplists:get_value(Node, CI)). - --spec connection_info(list()) -> orddict:orddict(). -connection_info(Cluster) -> - CI = orddict:from_list(rt:connection_info(Cluster)), - SolrInfo = orddict:from_list([{Node, [{solr_http, get_yz_conn_info(Node)}]} - || Node <- Cluster]), - orddict:merge(fun(_,V1,V2) -> V1 ++ V2 end, CI, SolrInfo). - --spec solr_http({node(), orddict:orddict()}) -> {host(), portnum()}. -solr_http({_Node, ConnInfo}) -> - solr_http(ConnInfo); -solr_http(ConnInfo) -> - proplists:get_value(solr_http, ConnInfo). - --spec get_yz_conn_info(node()) -> {string(), string()}. -get_yz_conn_info(Node) -> - {ok, SolrPort} = rpc:call(Node, application, get_env, [yokozuna, solr_port]), - %% Currently Yokozuna hardcodes listener to all interfaces - {"127.0.0.1", SolrPort}. - --spec http(method(), string(), list(), binary()|[]) -> response(). -http(Method, URL, Headers, Body) -> - Opts = [], - ibrowse:send_req(URL, Headers, Method, Body, Opts, ?IBROWSE_TIMEOUT). - --spec http(method(), string(), list(), binary()|[], list()|timeout()) - -> response(). -http(Method, URL, Headers, Body, Opts) when is_list(Opts) -> - ibrowse:send_req(URL, Headers, Method, Body, Opts, ?IBROWSE_TIMEOUT); -http(Method, URL, Headers, Body, Timeout) when is_integer(Timeout) -> - Opts = [], - ibrowse:send_req(URL, Headers, Method, Body, Opts, Timeout). - --spec http(method(), string(), list(), binary()|[], list(), timeout()) - -> response(). -http(Method, URL, Headers, Body, Opts, Timeout) when - is_list(Opts) andalso is_integer(Timeout) -> - ibrowse:send_req(URL, Headers, Method, Body, Opts, Timeout). - --spec create_indexed_bucket_type(cluster(), binary(), index_name()) -> ok. -create_indexed_bucket_type(Cluster, BucketType, IndexName) -> - ok = create_index(Cluster, IndexName), - ok = create_bucket_type(Cluster, BucketType, [{search_index, IndexName}]). - --spec create_indexed_bucket_type(cluster(), binary(), index_name(), - schema_name()) -> ok. -create_indexed_bucket_type(Cluster, BucketType, IndexName, SchemaName) -> - ok = create_index(Cluster, IndexName, SchemaName), - ok = create_bucket_type(Cluster, BucketType, [{search_index, IndexName}]). - --spec create_index(cluster(), index_name()) -> ok. -create_index(Cluster, Index) -> - Node = select_random(Cluster), - lager:info("Creating index ~s [~p]", [Index, Node]), - rpc:call(Node, yz_index, create, [Index]), - ok = wait_for_index(Cluster, Index). - --spec create_index(cluster(), index_name(), schema_name()) -> ok. -create_index(Cluster, Index, SchemaName) -> - Node = select_random(Cluster), - lager:info("Creating index ~s with schema ~s [~p]", - [Index, SchemaName, Node]), - rpc:call(Node, yz_index, create, [Index, SchemaName]), - ok = wait_for_index(Cluster, Index). - -select_random(List) -> - Length = length(List), - Idx = random:uniform(Length), - lists:nth(Idx, List). - --spec create_bucket_type(cluster(), binary(), [term()]) -> ok. -create_bucket_type(Cluster, BucketType, Props) -> - Node = select_random(Cluster), - rt:create_and_activate_bucket_type(Node, BucketType, Props), - rt:wait_until_bucket_type_status(BucketType, active, Node), - rt:wait_until_bucket_type_visible(Cluster, BucketType). diff --git a/tests/yz_core_properties_create_unload.erl b/tests/yz_core_properties_create_unload.erl deleted file mode 100644 index 186227e64..000000000 --- a/tests/yz_core_properties_create_unload.erl +++ /dev/null @@ -1,170 +0,0 @@ -%% ------------------------------------------------------------------- -%% -%% Copyright (c) 2014 Basho Technologies, Inc. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%%------------------------------------------------------------------- --module(yz_core_properties_create_unload). --compile(export_all). --include_lib("eunit/include/eunit.hrl"). - --define(CFG, [{riak_kv, - [ - %% allow AAE to build trees and exchange rapidly - {anti_entropy_build_limit, {1000, 1000}}, - {anti_entropy_concurrency, 64}, - {anti_entropy_tick, 1000} - ]}, - {yokozuna, - [ - {enabled, true}, - {anti_entropy_tick, 1000} - ]}]). --define(INDEX, <<"test_idx_core">>). --define(TYPE, <<"data">>). --define(BUCKET, {?TYPE, <<"test_bkt_core">>}). --define(SEQMAX, 100). - -confirm() -> - Cluster = rt:build_cluster(4, ?CFG), - rt:wait_for_cluster_service(Cluster, yokozuna), - - %% Generate keys, YZ only supports UTF-8 compatible keys - Keys = [<> || N <- lists:seq(1, ?SEQMAX), - not lists:any(fun(E) -> E > 127 end, - binary_to_list(<>))], - KeyCount = length(Keys), - - %% Randomly select a subset of the test nodes to remove - %% core.properties from - RandNodes = rt:random_sublist(Cluster, 3), - - %% Select one of the modified nodes as a client endpoint - Node = rt:select_random(RandNodes), - Pid = rt:pbc(Node), - riakc_pb_socket:set_options(Pid, [queue_if_disconnected]), - - %% Create a search index and associate with a bucket - lager:info("Create and set Index ~p for Bucket ~p~n", [?INDEX, ?BUCKET]), - _ = riakc_pb_socket:create_search_index(Pid, ?INDEX), - yokozuna_rt:wait_for_index(Cluster, ?INDEX), - - ok = rt:create_and_activate_bucket_type(Node, - ?TYPE, - [{search_index, ?INDEX}]), - - rt:wait_until_bucket_type_visible(Cluster, ?TYPE), - - %% Write keys and wait for soft commit - lager:info("Writing ~p keys", [KeyCount]), - [ok = rt:pbc_write(Pid, ?BUCKET, Key, Key, "text/plain") || Key <- Keys], - yokozuna_rt:commit(Cluster, ?INDEX), - - yokozuna_rt:verify_num_found_query(Cluster, ?INDEX, KeyCount), - - - test_core_props_removal(Cluster, RandNodes, KeyCount, Pid), - test_remove_index_dirs(Cluster, RandNodes, KeyCount, Pid), - test_remove_segment_infos_and_rebuild(Cluster, RandNodes, KeyCount, Pid), - test_brutal_kill_and_delete_index_dirs(Cluster, RandNodes, KeyCount, Pid), - - riakc_pb_socket:stop(Pid), - - pass. - -test_core_props_removal(Cluster, RandNodes, KeyCount, Pid) -> - lager:info("Remove core.properties file in each index data dir"), - remove_core_props(RandNodes, ?INDEX), - - yokozuna_rt:check_exists(Cluster, ?INDEX), - - lager:info("Write one more piece of data"), - ok = rt:pbc_write(Pid, ?BUCKET, <<"foo">>, <<"foo">>, "text/plain"), - yokozuna_rt:commit(Cluster, ?INDEX), - - yokozuna_rt:verify_num_found_query(Cluster, ?INDEX, KeyCount+1). - -test_remove_index_dirs(Cluster, RandNodes, KeyCount, Pid) -> - lager:info("Remove index directories on each node and let them recreate/reindex"), - yokozuna_rt:remove_index_dirs(RandNodes, ?INDEX, [riak_kv, yokozuna]), - - yokozuna_rt:check_exists(Cluster, ?INDEX), - - yokozuna_rt:expire_trees(Cluster), - yokozuna_rt:wait_for_aae(Cluster), - - lager:info("Write second piece of data"), - ok = rt:pbc_write(Pid, ?BUCKET, <<"food">>, <<"foody">>, "text/plain"), - yokozuna_rt:commit(Cluster, ?INDEX), - - yokozuna_rt:verify_num_found_query(Cluster, ?INDEX, KeyCount+2). - -test_remove_segment_infos_and_rebuild(Cluster, RandNodes, KeyCount, Pid) -> - lager:info("Remove segment info files in each index data dir"), - remove_segment_infos(RandNodes, ?INDEX), - - lager:info("To fix, we remove index directories on each node and let them recreate/reindex"), - - yokozuna_rt:remove_index_dirs(RandNodes, ?INDEX, [riak_kv, yokozuna]), - - yokozuna_rt:check_exists(Cluster, ?INDEX), - - yokozuna_rt:expire_trees(Cluster), - yokozuna_rt:wait_for_aae(Cluster), - - lager:info("Write third piece of data"), - ok = rt:pbc_write(Pid, ?BUCKET, <<"baz">>, <<"bar">>, "text/plain"), - yokozuna_rt:commit(Cluster, ?INDEX), - - yokozuna_rt:verify_num_found_query(Cluster, ?INDEX, KeyCount+3). - -test_brutal_kill_and_delete_index_dirs(Cluster, RandNodes, KeyCount, Pid) -> - lager:info("Remove index directories on each node and let them recreate/reindex"), - yokozuna_rt:brutal_kill_remove_index_dirs(RandNodes, ?INDEX, [riak_kv, yokozuna]), - - yokozuna_rt:check_exists(Cluster, ?INDEX), - - yokozuna_rt:expire_trees(Cluster), - yokozuna_rt:wait_for_aae(Cluster), - - lager:info("Write fourth piece of data"), - ok = rt:pbc_write(Pid, ?BUCKET, <<"food">>, <<"foody">>, "text/plain"), - yokozuna_rt:commit(Cluster, ?INDEX), - - yokozuna_rt:verify_num_found_query(Cluster, ?INDEX, KeyCount + 4). - -%% @doc Remove core properties file on nodes. -remove_core_props(Nodes, IndexName) -> - IndexDirs = [rpc:call(Node, yz_index, index_dir, [IndexName]) || - Node <- Nodes], - PropsFiles = [filename:join([IndexDir, "core.properties"]) || - IndexDir <- IndexDirs], - lager:info("Remove core.properties files: ~p, on nodes: ~p~n", - [PropsFiles, Nodes]), - [file:delete(PropsFile) || PropsFile <- PropsFiles], - ok. - -%% @doc Remove lucence segment info files to check if reindexing will occur -%% on re-creation/re-indexing. -remove_segment_infos(Nodes, IndexName) -> - IndexDirs = [rpc:call(Node, yz_index, index_dir, [IndexName]) || - Node <- Nodes], - SiPaths = [binary_to_list(filename:join([IndexDir, "data/index/*.si"])) || - IndexDir <- IndexDirs], - SiFiles = lists:append([filelib:wildcard(Path) || Path <- SiPaths]), - lager:info("Remove segment info files: ~p, on in dirs: ~p~n", - [SiFiles, IndexDirs]), - [file:delete(SiFile) || SiFile <- SiFiles]. diff --git a/tests/yz_crdt.erl b/tests/yz_crdt.erl deleted file mode 100644 index f2c114cb3..000000000 --- a/tests/yz_crdt.erl +++ /dev/null @@ -1,564 +0,0 @@ --module(yz_crdt). - --compile(export_all). --compile({parse_transform, rt_intercept_pt}). - --include_lib("eunit/include/eunit.hrl"). - --define(HARNESS, (rt_config:get(rt_harness))). --define(INDEX, <<"maps">>). --define(TYPE, <<"maps">>). --define(KEY, <<"Chris Meiklejohn">>). --define(BUCKET, {?TYPE, <<"testbucket">>}). --define(GET(K,L), proplists:get_value(K, L)). --define(N, 3). - --define(CONF, - [ - {riak_core, - [{ring_creation_size, 8}] - }, - {riak_kv, - [{delete_mode, keep}, - {anti_entropy_build_limit, {100, 1000}}, - {anti_entropy_concurrency, 8}, - {anti_entropy_tick, 1000}]}, - {yokozuna, - [{enabled, true}] - }]). - -confirm() -> - rt:set_advanced_conf(all, ?CONF), - - %% Configure cluster. - Nodes = rt:build_cluster(5, ?CONF), - - Node = rt:select_random(Nodes), - - %% Create PB connection. - Pid = rt:pbc(Node), - riakc_pb_socket:set_options(Pid, [queue_if_disconnected]), - - %% Create index. - riakc_pb_socket:create_search_index(Pid, ?INDEX, <<"_yz_default">>, []), - - %% Create bucket type for maps. - rt:create_and_activate_bucket_type(Node, - ?TYPE, - [{datatype, map}, - {n_val, ?N}, - {search_index, ?INDEX}]), - - lager:info("Write some sample data"), - test_sample_data(Pid, Nodes, ?BUCKET, ?KEY, ?INDEX), - lager:info("Search and Validate our CRDT writes/updates."), - ok = rt:wait_until(fun() -> validate_sample_data(Pid, ?KEY, ?INDEX) - end), - - lager:info("Test setting the register of a map twice to different values." - " (The # of results should still be 1)"), - test_repeat_sets(Pid, Nodes, ?BUCKET, ?INDEX, ?KEY), - ok = rt:wait_until(fun() -> validate_test_repeat_set(Pid, ?INDEX) - end), - - lager:info("FYI: delete_mode is on keep here to make sure YZ handles" - " deletes correctly throughout."), - lager:info("Test varying deletes operations"), - test_and_validate_delete(Pid, Nodes, ?BUCKET, ?INDEX, ?KEY), - - lager:info("Test to make sure yz AAE handles deletes/removes correctly"), - test_and_validate_delete_aae(Pid, Nodes, ?BUCKET, ?INDEX), - - lager:info("Test to make sure siblings don't exist after partition"), - test_siblings(Nodes, ?BUCKET, ?INDEX), - lager:info("Verify counts and operations after heal + transfers + commits"), - ok = rt:wait_until(fun() -> validate_test_siblings(Pid, ?BUCKET, ?INDEX) - end), - - %% Stop PB connection - riakc_pb_socket:stop(Pid), - - pass. - -test_sample_data(Pid, Cluster, Bucket, Key, Index) -> - Map1 = riakc_map:update( - {<<"name">>, register}, - fun(R) -> - riakc_register:set(Key, R) - end, riakc_map:new()), - Map2 = riakc_map:update( - {<<"interests">>, set}, - fun(S) -> - riakc_set:add_element(<<"thing">>, S) end, - Map1), - - ok = riakc_pb_socket:update_type( - Pid, - Bucket, - Key, - riakc_map:to_op(Map2)), - - drain_and_commit(Cluster, Index). - -%% @doc Test setting the register of a map twice to different values. -%% The # of results should still be 1. -test_repeat_sets(Pid, Cluster, Bucket, Index, Key) -> - {ok, M1} = riakc_pb_socket:fetch_type(Pid, Bucket, Key), - M2 = riakc_map:update( - {<<"update">>, register}, - fun(R) -> - riakc_register:set(<<"foo">>, R) - end, M1), - ok = riakc_pb_socket:update_type( - Pid, - Bucket, - Key, - riakc_map:to_op(M2)), - M3 = riakc_map:update( - {<<"update">>, register}, - fun(R) -> - riakc_register:set(<<"bar">>, R) - end, M1), - ok = riakc_pb_socket:update_type( - Pid, - Bucket, - Key, - riakc_map:to_op(M3)), - - drain_and_commit(Cluster, Index). - -%% @doc Tests varying deletes of within a CRDT map and checks for correct counts -%% - Remove registers, remove and add elements within a set -%% - Delete the map (associated w/ a key) -%% - Recreate objects in the map and delete the map again -test_and_validate_delete(Pid, Cluster, Bucket, Index, Key) -> - {ok, M1} = riakc_pb_socket:fetch_type(Pid, Bucket, Key), - - lager:info("Remove register from map"), - M2 = riakc_map:erase({<<"name">>, register}, M1), - - lager:info("Delete element from set (in map) & Add element to set"), - M3 = riakc_map:update( - {<<"interests">>, set}, - fun(S) -> - riakc_set:del_element(<<"thing">>, - riakc_set:add_element(<<"roses">>, S)) - end, M2), - - ok = riakc_pb_socket:update_type( - Pid, - Bucket, - Key, - riakc_map:to_op(M3)), - - drain_and_commit(Cluster, Index), - - lager:info("Search deleted/erased name_register:*"), - search_and_validate_found(Pid, Index, <<"name_register:*">>, 0), - - lager:info("Add another element to set (in map)"), - M4 = riakc_map:update( - {<<"interests">>, set}, - fun(S) -> - riakc_set:add_element(<<"pans">>, S) - end, M3), - - ok = riakc_pb_socket:update_type( - Pid, - Bucket, - Key, - riakc_map:to_op(M4)), - - drain_and_commit(Cluster, Index), - - lager:info("Search deleted interests_set:thing*"), - search_and_validate_found(Pid, Index, <<"interests_set:thing*">>, 0), - - lager:info("Delete key for map"), - ?assertEqual(ok, riakc_pb_socket:delete(Pid, Bucket, Key)), - - drain_and_commit(Cluster, Index), - - ?assertEqual({error, {notfound, map}}, - riakc_pb_socket:fetch_type(Pid, Bucket, Key)), - - lager:info("Search deleted map: *:*"), - search_and_validate_found(Pid, Index, <<"*:*">>, 0), - - lager:info("Recreate object and check counts..."), - - lager:info("Set a new register for map"), - M5 = riakc_map:update( - {<<"name">>, register}, - fun(R) -> - riakc_register:set(<<"hello">>, R) - end, riakc_map:new()), - - ok = riakc_pb_socket:update_type( - Pid, - Bucket, - Key, - riakc_map:to_op(M5)), - - drain_and_commit(Cluster, Index), - - {ok, M6} = riakc_pb_socket:fetch_type(Pid, Bucket, Key), - Keys = riakc_map:fetch_keys(M6), - ?assertEqual(1, length(Keys)), - ?assert(riakc_map:is_key({<<"name">>, register}, M6)), - - lager:info("Search recreated map: *:*"), - search_and_validate_found(Pid, Index, <<"*:*">>, 1), - - lager:info("Delete key for map again"), - ?assertEqual(ok, riakc_pb_socket:delete(Pid, Bucket, Key)), - - drain_and_commit(Cluster, Index), - - ?assertEqual({error, {notfound, map}}, - riakc_pb_socket:fetch_type(Pid, Bucket, Key)), - - lager:info("Search ~p deleted map: *:*", [Key]), - search_and_validate_found(Pid, Index, <<"*:*">>, 0). - -%% @doc Tests key/map delete and AAE -%% - Use intercept to trap yz_kv:delete_operation to skip over -%% - Makes sure that yz AAE handles tombstone on expire/exchange -%% - Recreate objects and check -test_and_validate_delete_aae(Pid, Cluster, Bucket, Index) -> - Key1 = <<"ohyokozuna">>, - M1 = riakc_map:update( - {<<"name">>, register}, - fun(R) -> - riakc_register:set(<<"jokes are">>, R) - end, riakc_map:new()), - ok = riakc_pb_socket:update_type( - Pid, - Bucket, - Key1, - riakc_map:to_op(M1)), - - Key2 = <<"ohriaksearch">>, - M2 = riakc_map:update( - {<<"name">>, register}, - fun(R) -> - riakc_register:set(<<"better explained">>, R) - end, riakc_map:new()), - ok = riakc_pb_socket:update_type( - Pid, - Bucket, - Key2, - riakc_map:to_op(M2)), - - drain_and_commit(Cluster, Index), - - lager:info("Add and load handle_delete_operation intercept"), - - [make_intercepts_tab(ANode) || ANode <- Cluster], - - [rt_intercept:add(ANode, {yz_solrq_helper, [{{get_ops_for_no_sibling_deletes, 3}, - handle_get_ops_for_no_sibling_deletes}]}) - || ANode <- Cluster], - [true = rpc:call(ANode, ets, insert, [intercepts_tab, {del_put, 0}]) || - ANode <- Cluster], - [rt_intercept:wait_until_loaded(ANode) || ANode <- Cluster], - - lager:info("Delete key ~p for map", [Key2]), - ?assertEqual(ok, riakc_pb_socket:delete(Pid, Bucket, Key2)), - ?assertEqual({error, {notfound, map}}, - riakc_pb_socket:fetch_type(Pid, Bucket, Key2)), - - drain_and_commit(Cluster, Index), - - lager:info("Search all results, expect extra b/c tombstone" - " and we've modified the delete op : *:*"), - search_and_validate_found(Pid, Index, <<"*:*">>, 2), - - lager:info("Expire and re-check"), - yokozuna_rt:expire_trees(Cluster), - yokozuna_rt:wait_for_full_exchange_round(Cluster, erlang:now()), - drain_and_commit(Cluster, Index), - - lager:info("Search all results, expect removed tombstone b/c AAE" - " should clean it up: *:*"), - search_and_validate_found(Pid, Index, <<"*:*">>, 1), - - lager:info("Recreate object and check counts"), - - M3 = riakc_map:update( - {<<"name">>, register}, - fun(R) -> - riakc_register:set(<<"hello again, is it me you're" - "looking for">>, R) - end, riakc_map:new()), - - ok = riakc_pb_socket:update_type( - Pid, - Bucket, - Key2, - riakc_map:to_op(M3)), - - drain_and_commit(Cluster, Index), - - {ok, M4} = riakc_pb_socket:fetch_type(Pid, Bucket, Key2), - Keys = riakc_map:fetch_keys(M4), - ?assertEqual(1, length(Keys)), - ?assert(riakc_map:is_key({<<"name">>, register}, M4)), - - lager:info("Search recreated map: *:*"), - search_and_validate_found(Pid, Index, <<"*:*">>, 2). - -%% @doc Tests sibling handling/merge when there's a partition -%% - Write/remove from separate partitions -%% - Verify counts and that CRDTs have no siblings, vtags, -%% after healing partitions. The CRDT map merges so the search -%% results be consistent. -test_siblings(Cluster, Bucket, Index) -> - Key1 = <<"Movies">>, - Key2 = <<"Games">>, - Set1 = <<"directors">>, - Set2 = <<"characters">>, - - %% make siblings - {P1, P2} = lists:split(1, Cluster), - - %% Create an object in Partition 1 and siblings in Partition 2 - lager:info("Create partition: ~p | ~p", [P1, P2]), - Partition = rt:partition(P1, P2), - - %% PB connections for accessing each side - Pid1 = rt:pbc(hd(P1)), - Pid2 = rt:pbc(hd(P2)), - - riakc_pb_socket:set_options(Pid1, [queue_if_disconnected, auto_reconnect]), - riakc_pb_socket:set_options(Pid2, [queue_if_disconnected, auto_reconnect]), - - %% P1 writes - lager:info("Writing to Partition 1 Set 1: Key ~p | Director ~p", - [Key1, <<"Kubrick">>]), - M1 = riakc_map:update( - {Set1, set}, - fun(S) -> - riakc_set:add_element(<<"Kubrick">>, S) - end, riakc_map:new()), - ok = riakc_pb_socket:update_type( - Pid1, - Bucket, - Key1, - riakc_map:to_op(M1)), - - lager:info("Writing to Partition 1 Set 1: Key ~p | Director ~p", - [Key1, <<"Demme">>]), - M2 = riakc_map:update( - {Set1, set}, - fun(S) -> - riakc_set:add_element(<<"Demme">>, S) - end, riakc_map:new()), - ok = riakc_pb_socket:update_type( - Pid1, - Bucket, - Key1, - riakc_map:to_op(M2)), - - %% P2 Siblings - lager:info("Writing to Partition 2 Set 2: Key ~p | Char ~p", - [Key2, <<"Sonic">>]), - M3 = riakc_map:update( - {Set2, set}, - fun(S) -> - riakc_set:add_element(<<"Sonic">>, S) - end, riakc_map:new()), - ok = riakc_pb_socket:update_type( - Pid2, - Bucket, - Key2, - riakc_map:to_op(M3)), - - lager:info("Delete key from Partition 2: Key ~p", [Key2]), - ok = riakc_pb_socket:delete(Pid2, Bucket, Key2), - - lager:info("Writing to Partition 2 Set 2: after delete: Key ~p | Char" - " ~p", [Key2, <<"Crash">>]), - M4 = riakc_map:update( - {Set2, set}, - fun(S) -> - riakc_set:add_element(<<"Crash">>, S) - end, riakc_map:new()), - ok = riakc_pb_socket:update_type( - Pid2, - Bucket, - Key2, - riakc_map:to_op(M4)), - - lager:info("Writing to Partition 2 Set 2: Key ~p | Char ~p", - [Key2, <<"Mario">>]), - M5 = riakc_map:update( - {Set2, set}, - fun(S) -> - riakc_set:add_element(<<"Mario">>, S) - end, riakc_map:new()), - ok = riakc_pb_socket:update_type( - Pid2, - Bucket, - Key2, - riakc_map:to_op(M5)), - - lager:info("Writing to Partition 2 Set 1: Key ~p | Director ~p", - [Key1, <<"Klimov">>]), - M6 = riakc_map:update( - {Set1, set}, - fun(S) -> - riakc_set:add_element(<<"Klimov">>, S) - end, riakc_map:new()), - ok = riakc_pb_socket:update_type( - Pid2, - Bucket, - Key1, - riakc_map:to_op(M6)), - - rt:heal(Partition), - rt:wait_until_transfers_complete(Cluster), - - drain_and_commit(Cluster, Index). - -validate_sample_data(Pid, Key, Index) -> - try - Thing = <<"thing">>, - - {ok, {search_results, Results1a, _, Found1}} = riakc_pb_socket:search( - Pid, Index, <<"name_register:Chris*">>), - ?assertEqual(1, Found1), - - ?assertEqual(?GET(<<"name_register">>, ?GET(Index, Results1a)), - Key), - ?assertEqual(?GET(<<"interests_set">>, ?GET(Index, Results1a)), - Thing), - - {ok, {search_results, Results2a, _, Found2}} = riakc_pb_socket:search( - Pid, Index, <<"interests_set:thing*">>), - ?assertEqual(1, Found2), - ?assertEqual(?GET(<<"name_register">>, ?GET(Index, Results2a)), - Key), - ?assertEqual(?GET(<<"interests_set">>, ?GET(Index, Results2a)), - Thing), - - {ok, {search_results, Results3a, _, Found3}} = riakc_pb_socket:search( - Pid, Index, <<"_yz_rb:testbucket">>), - ?assertEqual(1, Found3), - ?assertEqual(?GET(<<"name_register">>, ?GET(Index, Results3a)), - Key), - ?assertEqual(?GET(<<"interests_set">>, ?GET(Index, Results3a)), - Thing), - - %% Redo queries and check if results are equal - {ok, {search_results, Results1b, _, _}} = riakc_pb_socket:search( - Pid, Index, <<"name_register:Chris*">>), - ?assertEqual(number_of_fields(Results1a, Index), - number_of_fields(Results1b, Index)), - - {ok, {search_results, Results2b, _, _}} = riakc_pb_socket:search( - Pid, Index, <<"interests_set:thing*">>), - ?assertEqual(number_of_fields(Results2a, Index), - number_of_fields(Results2b, Index)), - - {ok, {search_results, Results3b, _, _}} = riakc_pb_socket:search( - Pid, Index, <<"_yz_rb:testbucket">>), - ?assertEqual(number_of_fields(Results3a, Index), - number_of_fields(Results3b, Index)), - - true - catch Err:Reason -> - lager:info("Waiting for CRDT search results to converge. Error" - " was ~p.", [{Err, Reason}]), - false - end. - -validate_test_repeat_set(Pid, Index) -> - try - {ok, {search_results, _R, _, Found}} = riakc_pb_socket:search( - Pid, Index, - <<"update_register:*">>), - ?assertEqual(1, Found), - - true - catch Err:Reason -> - lager:info("Waiting for CRDT search results to converge. Error" - " was ~p.", [{Err, Reason}]), - false - end. - -validate_test_siblings(Pid, Bucket, Index) -> - try - Key1 = <<"Movies">>, - Key2 = <<"Games">>, - - {ok, MF1} = riakc_pb_socket:fetch_type(Pid, Bucket, Key1), - Keys = riakc_map:fetch_keys(MF1), - ?assertEqual(1, length(Keys)), - ?assert(riakc_map:is_key({<<"directors">>, set}, MF1)), - {ok, {search_results, Results1, _, _}} = riakc_pb_socket:search( - Pid, Index, - <<"directors_set:*">>), - lager:info("Search movies map directors_set:*: ~p~n", [Results1]), - ?assertEqual(3, length(proplists:lookup_all(<<"directors_set">>, - ?GET(?INDEX, Results1)))), - - {ok, MF2} = riakc_pb_socket:fetch_type(Pid, Bucket, Key2), - Keys2 = riakc_map:fetch_keys(MF2), - ?assertEqual(1, length(Keys2)), - ?assert(riakc_map:is_key({<<"characters">>, set}, MF2)), - {ok, {search_results, Results2, _, _}} = riakc_pb_socket:search( - Pid, Index, - <<"characters_set:*">>), - lager:info("Search games map characters_set:*: ~p~n", [Results2]), - ?assertEqual(2, length(proplists:lookup_all(<<"characters_set">>, - ?GET(?INDEX, Results2)))), - - {ok, {search_results, Results3, _, Found}} = riakc_pb_socket:search( - Pid, Index, - <<"_yz_vtag:*">>), - lager:info("Search vtags in search *:*: ~p~n", [Results3]), - ?assertEqual(0, Found), - true - catch Err:Reason -> - lager:info("Waiting for CRDT search results to converge. Error" - " was ~p.", [{Err, Reason}]), - false - end. - -%% @private -drain_and_commit(Cluster, Index) -> - yokozuna_rt:drain_solrqs(Cluster), - yokozuna_rt:commit(Cluster, Index). - -%% @private -number_of_fields(Resp, Index) -> - length(?GET(Index, Resp)). - -%% @private -make_intercepts_tab(Node) -> - SupPid = rpc:call(Node, erlang, whereis, [sasl_safe_sup]), - intercepts_tab = rpc:call(Node, ets, new, [intercepts_tab, [named_table, - public, set, {heir, SupPid, {}}]]). - -%% @private -search_and_validate_found(Pid, Index, Search, ExpectedCount) -> - ok = rt:wait_until( - fun() -> - try - {ok, {search_results, Results2, _, F}} = - riakc_pb_socket:search(Pid, Index, Search), - ?assertEqual(ExpectedCount, F), - true - catch Err:Reason -> - lager:info( - "Waiting for CRDT search results to converge." - " Index: ~p" - " Search: ~p" - " Error: ~p", - [Index, Search, {Err, Reason}] - ), - false - end - end). - diff --git a/tests/yz_default_bucket_type_upgrade.erl b/tests/yz_default_bucket_type_upgrade.erl deleted file mode 100644 index 0152d692e..000000000 --- a/tests/yz_default_bucket_type_upgrade.erl +++ /dev/null @@ -1,98 +0,0 @@ -%% ------------------------------------------------------------------- -%% -%% Copyright (c) 2015 Basho Technologies, Inc. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%%-------------------------------------------------------------------- - -%% @doc Test that checks to make sure that default bucket_types -%% do not lose data when expiring/clearing AAE trees when -%% trees are rebuilt for comparison. -%% @end - - --module(yz_default_bucket_type_upgrade). --compile(export_all). --include_lib("eunit/include/eunit.hrl"). --include_lib("riakc/include/riakc.hrl"). - --define(N, 3). --define(YZ_CAP, {yokozuna, handle_legacy_default_bucket_type_aae}). --define(INDEX, <<"test_upgrade_idx">>). --define(BUCKET, <<"test_upgrade_bucket">>). --define(SEQMAX, 2000). --define(CFG, - [{riak_core, - [ - {ring_creation_size, 16}, - {default_bucket_props, - [ - {n_val, ?N}, - {allow_mult, true}, - {dvv_enabled, true} - ]} - ]}, - {riak_kv, - [ - {anti_entropy_build_limit, {100, 1000}}, - {anti_entropy_concurrency, 8} - ] - }, - {yokozuna, - [ - {anti_entropy_tick, 1000}, - {enabled, true} - ]} - ]). - -confirm() -> - %% This test explicitly requires an upgrade from 2.0.5 to test a - %% new capability - OldVsn = "2.0.5", - - [_, Node|_] = Cluster = rt:build_cluster(lists:duplicate(4, {OldVsn, ?CFG})), - rt:wait_for_cluster_service(Cluster, yokozuna), - - [rt:assert_capability(ANode, ?YZ_CAP, {unknown_capability, ?YZ_CAP}) || ANode <- Cluster], - - GenKeys = yokozuna_rt:gen_keys(?SEQMAX), - KeyCount = length(GenKeys), - lager:info("KeyCount ~p", [KeyCount]), - - OldPid = rt:pbc(Node), - - yokozuna_rt:write_data(Cluster, OldPid, ?INDEX, ?BUCKET, GenKeys), - yokozuna_rt:commit(Cluster, ?INDEX), - - yokozuna_rt:verify_num_found_query(Cluster, ?INDEX, KeyCount), - - %% Upgrade - yokozuna_rt:rolling_upgrade(Cluster, current), - - [rt:assert_capability(ANode, ?YZ_CAP, v1) || ANode <- Cluster], - [rt:assert_supported(rt:capability(ANode, all), ?YZ_CAP, [v1, v0]) || ANode <- Cluster], - - yokozuna_rt:verify_num_found_query(Cluster, ?INDEX, KeyCount), - - lager:info("Write one more piece of data"), - Pid = rt:pbc(Node), - ok = rt:pbc_write(Pid, ?BUCKET, <<"foo">>, <<"foo">>, "text/plain"), - yokozuna_rt:commit(Cluster, ?INDEX), - - yokozuna_rt:expire_trees(Cluster), - yokozuna_rt:verify_num_found_query(Cluster, ?INDEX, KeyCount + 1), - - pass. diff --git a/tests/yz_ensemble.erl b/tests/yz_ensemble.erl deleted file mode 100644 index f4a604592..000000000 --- a/tests/yz_ensemble.erl +++ /dev/null @@ -1,117 +0,0 @@ --module(yz_ensemble). --compile(export_all). --include_lib("eunit/include/eunit.hrl"). - --define(CFG, - [ - {riak_core, - [ - {ring_creation_size, 8} - ]}, - {yokozuna, - [ - {enabled, true} - ]} - ]). - -confirm() -> - NumNodes = 3, - NVal = 3, - ConfigB = ensemble_util:fast_config(NVal), - Config = ConfigB ++ [{yokozuna, [{enabled, true}]}], - lager:info("Building cluster and waiting for ensemble to stablize"), - Nodes = build_cluster_with_yz_support(NumNodes, Config, NVal), - rt:wait_for_cluster_service(Nodes, yokozuna), - vnode_util:load(Nodes), - Node = hd(Nodes), - - lager:info("Creating/activating 'strong' bucket type"), - rt:create_and_activate_bucket_type(Node, <<"strong">>, - [{consistent, true}, {n_val, NVal}]), - - Bucket = {<<"strong">>, <<"test">>}, - Index = <<"testi">>, - create_index(Node, Index), - set_bucket_props(Node, Bucket, Index), - - verify_ensemble_delete_support(Nodes, Bucket, Index), - - pass. - - -%% @private -%% @doc Populates then deletes from SC bucket -verify_ensemble_delete_support(Cluster, Bucket, Index) -> - %% Yz only supports UTF-8 compatible keys - Keys = [<> || N <- lists:seq(1,2000), - not lists:any(fun(E) -> E > 127 end,binary_to_list(<>))], - - PBC = rt:pbc(hd(Cluster)), - - lager:info("Writing ~p keys", [length(Keys)]), - [ok = rt:pbc_write(PBC, Bucket, Key, Key, "text/plain") || Key <- Keys], - yokozuna_rt:commit(Cluster, Index), - - %% soft commit wait, then check that last key is indexed - lager:info("Search for keys to verify they exist"), - LKey = lists:last(Keys), - rt:wait_until(fun() -> - {M, _} = riakc_pb_socket:search(PBC, Index, query_value(LKey)), - ok == M - end), - [{ok, _} = - riakc_pb_socket:search(PBC, Index, query_value(Key)) || Key <- Keys], - - lager:info("Deleting keys"), - [riakc_pb_socket:delete(PBC, Bucket, Key) || Key <- Keys], - yokozuna_rt:commit(Cluster, Index), - rt:wait_until(fun() -> - case riakc_pb_socket:search(PBC, Index, query_value(LKey)) of - {ok,{search_results,Res,_,_}} -> - lager:info("RES: ~p ~p~n", [Res, LKey]), - Res == []; - S -> - lager:info("OTHER: ~p ~p~n", [S, LKey]), - false - end - end), - [ {ok,{search_results,[],_,_}} = - riakc_pb_socket:search(PBC, Index, query_value(Key)) || Key <- Keys], - - ok. - - -%% @private -%% @doc build a cluster from ensemble_util + yz support -%% -%% NOTE: There's a timing issue that causes join_cluster to hang the r_t -%% node when adding yokozuna and ensemble support. Waiting for yokozuna -%% to load on each node allows join_cluster to complete consistently -build_cluster_with_yz_support(Num, Config, NVal) -> - Nodes = rt:deploy_nodes(Num, Config), - [rt:wait_for_cluster_service([N], yokozuna) || N <- Nodes], - Node = hd(Nodes), - rt:join_cluster(Nodes), - ensemble_util:wait_until_cluster(Nodes), - ensemble_util:wait_for_membership(Node), - ensemble_util:wait_until_stable(Node, NVal), - Nodes. - -%% @private -%% @doc Builds a simple riak key query -query_value(Value) -> - V2 = iolist_to_binary(re:replace(Value, "\"", "%22")), - V3 = iolist_to_binary(re:replace(V2, "\\\\", "%5C")), - <<"_yz_rk:\"",V3/binary,"\"">>. - -%% pulled from yz_rt - -%% @private -create_index(Node, Index) -> - lager:info("Creating index ~s [~p]", [Index, Node]), - ok = rpc:call(Node, yz_index, create, [Index]). - -%% @private -set_bucket_props(Node, Bucket, Index) -> - Props = [{search_index, Index}], - rpc:call(Node, riak_core_bucket, set_bucket, [Bucket, Props]). diff --git a/tests/yz_extractors.erl b/tests/yz_extractors.erl deleted file mode 100644 index 29d51c4bb..000000000 --- a/tests/yz_extractors.erl +++ /dev/null @@ -1,441 +0,0 @@ -%% ------------------------------------------------------------------- -%% -%% Copyright (c) 2015 Basho Technologies, Inc. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%%------------------------------------------------------------------- - -%% @doc Test that checks if we're caching the extractor map and that -%% creating custom extractors is doable via protobufs. -%% @end - --module(yz_extractors). --compile(export_all). --include_lib("eunit/include/eunit.hrl"). --include_lib("riakc/include/riakc.hrl"). - --define(FMT(S, Args), lists:flatten(io_lib:format(S, Args))). --define(TYPE1, <<"extractors_in_paradise">>). --define(TYPE2, <<"extractors_in_paradiso">>). --define(INDEX1, <<"test_idx1">>). --define(BUCKET1, {?TYPE1, <<"test_bkt1">>}). --define(INDEX2, <<"test_idx2">>). --define(BUCKET2, {?TYPE2, <<"test_bkt2">>}). --define(TYPE3, <<"type3">>). --define(BUCKET3, {?TYPE3, <<"test_bkt3">>}). --define(INDEX3, <<"test_idx3">>). --define(SCHEMANAME, <<"test">>). --define(TEST_SCHEMA, -<<" - - - - - - - - - - - - - - - - - - -_yz_id - - - - - - - - - - - - - - - - - - - - - -">>). --define(TEST_SCHEMA_UPGRADE, -<<" - - - - - - - - - - - - - - - - - - - -_yz_id - - - - - - - - - - - - - - - - - - - - - -">>). --define(YZ_CAP, {yokozuna, extractor_map_in_cmd}). --define(GET_MAP_RING_MFA, {yz_extractor, get_map, 1}). --define(GET_MAP_MFA, {yz_extractor, get_map, 0}). --define(GET_MAP_READTHROUGH_MFA, {yz_extractor, get_map_read_through, 0}). --define(YZ_META_EXTRACTORS, {yokozuna, extractors}). --define(YZ_EXTRACTOR_MAP, yokozuna_extractor_map). --define(NEW_EXTRACTOR, {"application/httpheader", yz_noop_extractor}). --define(EXTRACTOR_CT, element(1, ?NEW_EXTRACTOR)). --define(EXTRACTOR_MOD, element(2, ?NEW_EXTRACTOR)). --define(DEFAULT_MAP, [{default, yz_noop_extractor}, - {"application/json",yz_json_extractor}, - {"application/riak_counter", yz_dt_extractor}, - {"application/riak_map", yz_dt_extractor}, - {"application/riak_set", yz_dt_extractor}, - {"application/xml",yz_xml_extractor}, - {"text/plain",yz_text_extractor}, - {"text/xml",yz_xml_extractor} - ]). --define(EXTRACTMAPEXPECT, lists:sort(?DEFAULT_MAP ++ [?NEW_EXTRACTOR])). --define(SEQMAX, 20). --define(NVAL, 3). --define(CFG, - [ - {riak_kv, - [ - %% allow AAE to build trees and exchange rapidly - {anti_entropy_build_limit, {100, 1000}}, - {anti_entropy_concurrency, 8}, - {anti_entropy_tick, 1000}, - %% but start with AAE turned off so as not to interfere with earlier parts of the test - {anti_entropy, {off, []}} - ]}, - {yokozuna, - [ - {enabled, true} - ]} - ]). - -confirm() -> - %% This test explicitly requires an upgrade from 2.0.5 to test a - %% new capability - OldVsn = "2.0.5", - - [_, Node|_] = Cluster = rt:build_cluster(lists:duplicate(4, {OldVsn, ?CFG})), - rt:wait_for_cluster_service(Cluster, yokozuna), - - [rt:assert_capability(ANode, ?YZ_CAP, {unknown_capability, ?YZ_CAP}) || ANode <- Cluster], - - OldPid = rt:pbc(Node), - - GenKeys = yokozuna_rt:gen_keys(?SEQMAX), - KeyCount = length(GenKeys), - - rt:count_calls(Cluster, [?GET_MAP_RING_MFA, ?GET_MAP_MFA]), - - yokozuna_rt:write_data(Cluster, OldPid, ?INDEX1, - {?SCHEMANAME, ?TEST_SCHEMA}, ?BUCKET1, GenKeys), - - ok = rt:stop_tracing(), - - {ok, BProps} = riakc_pb_socket:get_bucket(OldPid, ?BUCKET1), - N = proplists:get_value(n_val, BProps), - - riakc_pb_socket:stop(OldPid), - - PrevGetMapRingCC = rt:get_call_count(Cluster, ?GET_MAP_RING_MFA), - PrevGetMapCC = rt:get_call_count(Cluster, ?GET_MAP_MFA), - ?assertEqual(KeyCount * N, PrevGetMapRingCC), - ?assertEqual(KeyCount * N, PrevGetMapCC), - - %% test query count - yokozuna_rt:verify_num_found_query(Cluster, ?INDEX1, KeyCount), - - {RingVal1, MDVal1} = get_ring_and_cmd_vals(Node, ?YZ_META_EXTRACTORS, - ?YZ_EXTRACTOR_MAP), - - ?assertEqual(undefined, MDVal1), - %% In previous version, Ring only gets map metadata if a non-default - %% extractor is registered - ?assertEqual(undefined, RingVal1), - - ?assertEqual(?DEFAULT_MAP, get_map(Node)), - - %% %% Custom Register - ExtractMap = register_extractor(Node, ?EXTRACTOR_CT, ?EXTRACTOR_MOD), - - ?assertEqual(?EXTRACTMAPEXPECT, ExtractMap), - - %% Upgrade - yokozuna_rt:rolling_upgrade(Cluster, current), - - [rt:wait_until_ready(ANode) || ANode <- Cluster], - - [rt:assert_capability(ANode, ?YZ_CAP, true) || ANode <- Cluster], - [rt:assert_supported(rt:capability(ANode, all), ?YZ_CAP, [true, false]) || - ANode <- Cluster], - - %% test query count again - yokozuna_rt:verify_num_found_query(Cluster, ?INDEX1, KeyCount), - - Pid = rt:pbc(Node), - - rt:count_calls(Cluster, [?GET_MAP_RING_MFA, ?GET_MAP_MFA, - ?GET_MAP_READTHROUGH_MFA]), - - yokozuna_rt:write_data(Cluster, Pid, ?INDEX2, {?SCHEMANAME, ?TEST_SCHEMA}, - ?BUCKET2, GenKeys), - yokozuna_rt:commit(Cluster, ?INDEX2), - - ok = rt:stop_tracing(), - - riakc_pb_socket:stop(Pid), - - CurrGetMapRingCC = rt:get_call_count(Cluster, ?GET_MAP_RING_MFA), - CurrGetMapCC = rt:get_call_count(Cluster, ?GET_MAP_MFA), - CurrGetMapRTCC = rt:get_call_count(Cluster, ?GET_MAP_READTHROUGH_MFA), - - lager:info("Number of calls to get the map from the ring - current: ~p~n, previous: ~p~n", - [CurrGetMapRingCC, PrevGetMapRingCC]), - ?assert(CurrGetMapRingCC < PrevGetMapRingCC), - lager:info("Number of calls to get the map - current: ~p~n, previous: ~p~n", - [CurrGetMapCC, PrevGetMapCC]), - ?assert(CurrGetMapCC =< PrevGetMapCC), - lager:info("Number of calls to get_map_read_through/0: ~p~n, Number of calls to get_map/0: ~p~n", - [CurrGetMapRTCC, CurrGetMapCC]), - ?assert(CurrGetMapRTCC =< CurrGetMapCC), - - {_RingVal2, MDVal2} = get_ring_and_cmd_vals(Node, ?YZ_META_EXTRACTORS, - ?YZ_EXTRACTOR_MAP), - - ?assertEqual(?EXTRACTMAPEXPECT, MDVal2), - ?assertEqual(?EXTRACTMAPEXPECT, get_map(Node)), - - Packet = <<"GET http://www.google.com HTTP/1.1\n">>, - test_extractor_works(Cluster, Packet), - test_extractor_with_aae_expire(Cluster, ?INDEX2, ?BUCKET2, Packet), - test_bad_extraction(Cluster), - - pass. - -%%%=================================================================== -%%% Private -%%%=================================================================== - -get_ring_and_cmd_vals(Node, Prefix, Key) -> - Ring = rpc:call(Node, yz_misc, get_ring, [transformed]), - MDVal = metadata_get(Node, Prefix, Key), - RingVal = ring_meta_get(Node, Key, Ring), - {RingVal, MDVal}. - -metadata_get(Node, Prefix, Key) -> - rpc:call(Node, riak_core_metadata, get, [Prefix, Key, []]). - -ring_meta_get(Node, Key, Ring) -> - rpc:call(Node, riak_core_ring, get_meta, [Key, Ring]). - -register_extractor(Node, MimeType, Mod) -> - rpc:call(Node, yz_extractor, register, [MimeType, Mod]). - -get_map(Node) -> - rpc:call(Node, yz_extractor, get_map, []). - -verify_extractor(Node, PacketData, Mod) -> - rpc:call(Node, yz_extractor, run, [PacketData, Mod]). - -bucket_url({Host,Port}, {BType, BName}, Key) -> - ?FMT("http://~s:~B/types/~s/buckets/~s/keys/~s", - [Host, Port, BType, BName, Key]). - -test_extractor_works(Cluster, Packet) -> - [rt_intercept:add(ANode, {yz_noop_extractor, - [{{extract, 1}, extract_httpheader}]}) || - ANode <- Cluster], - [rt_intercept:wait_until_loaded(ANode) || ANode <- Cluster], - - ExpectedExtraction = [{method, 'GET'}, - {host, <<"www.google.com">>}, - {uri, <<"/">>}], - ?assertEqual(ExpectedExtraction, - verify_extractor(rt:select_random(Cluster), Packet, ?EXTRACTOR_MOD)). - -test_extractor_with_aae_expire(Cluster, Index, Bucket, Packet) -> - %% Now make sure we register extractor across all nodes - [register_extractor(ANode, ?EXTRACTOR_CT, ?EXTRACTOR_MOD) || - ANode <- Cluster], - - Key = <<"google">>, - - {Host, Port} = rt:select_random(yokozuna_rt:host_entries( - rt:connection_info( - Cluster))), - URL = bucket_url({Host, Port}, Bucket, - mochiweb_util:quote_plus(Key)), - - CT = ?EXTRACTOR_CT, - {ok, "204", _, _} = yokozuna_rt:http( - put, URL, [{"Content-Type", CT}], Packet), - - yokozuna_rt:commit(Cluster, Index), - - ANode = rt:select_random(Cluster), - yokozuna_rt:search_expect(ANode, Index, <<"host">>, - <<"www*">>, 1), - - rpc:multicall(Cluster, riak_kv_entropy_manager, enable, []), - - yokozuna_rt:expire_trees(Cluster), - yokozuna_rt:wait_for_full_exchange_round(Cluster, erlang:now()), - - yokozuna_rt:search_expect(ANode, Index, <<"host">>, - <<"www*">>, 1), - - APid = rt:pbc(rt:select_random(Cluster)), - yokozuna_rt:override_schema(APid, Cluster, Index, ?SCHEMANAME, - ?TEST_SCHEMA_UPGRADE), - - {ok, "200", RHeaders, _} = yokozuna_rt:http(get, URL, [{"Content-Type", CT}], - [], []), - VC = proplists:get_value("X-Riak-Vclock", RHeaders), - - {ok, "204", _, _} = yokozuna_rt:http( - put, URL, [{"Content-Type", CT}, {"X-Riak-Vclock", VC}], - Packet), - yokozuna_rt:commit(Cluster, Index), - - yokozuna_rt:search_expect(ANode, Index, <<"method">>, - <<"GET">>, 1), - - yokozuna_rt:expire_trees(Cluster), - yokozuna_rt:wait_for_full_exchange_round(Cluster, erlang:now()), - - yokozuna_rt:search_expect(ANode, Index, <<"method">>, - <<"GET">>, 1), - riakc_pb_socket:stop(APid). - -test_bad_extraction(Cluster) -> - %% Previous test enabled AAE, which makes the number of repairs here not consistent - %% Turn off AAE again just to make the test deterministic. - rpc:multicall(Cluster, riak_kv_entropy_manager, disable, []), - %% - %% register the no-op extractor on all the nodes with a content type - %% - [register_extractor(ANode, "application/bad-extractor", yz_noop_extractor) || - ANode <- Cluster], - %% - %% Set up the intercepts so that they extract non-unicode data - %% - [rt_intercept:add(ANode, {yz_noop_extractor, - [{{extract, 1}, extract_non_unicode_data}]}) || - ANode <- Cluster], - [rt_intercept:wait_until_loaded(ANode) || ANode <- Cluster], - %% - %% create and wire up the bucket to the Solr index/core - %% - yokozuna_rt:create_indexed_bucket_type(Cluster, ?TYPE3, ?INDEX3, ?SCHEMANAME), - %% - %% Grab the stats before - %% - {PreviousFailCount, PreviousErrorThresholdCount} = get_error_stats(Cluster), - %% - %% Put some data into Riak. This should cause the intercepted no-op - %% extractor to generate an object to be written into Solr that contains - %% non-unicode data. - {Host, Port} = rt:select_random( - yokozuna_rt:host_entries(rt:connection_info(Cluster))), - Key = <<"test_bad_extraction">>, - URL = bucket_url({Host, Port}, ?BUCKET3, Key), - Headers = [{"Content-Type", "application/bad-extractor"}], - Data = <<"blahblahblahblah">>, - {ok, "204", _, _} = yokozuna_rt:http(put, URL, Headers, Data), - %% - %% The put should pass, but because it's "bad data", there should - %% be no data in Riak. - %% - yokozuna_rt:verify_num_found_query(Cluster, ?INDEX3, 0), - %% - %% Verify the stats. There should be one more index failure, - %% but there should be more more "melts" (error threshold failures) - %% - yokozuna_rt:wait_until( - Cluster, - fun(_Node) -> - check_error_stats(Cluster, PreviousFailCount, PreviousErrorThresholdCount) - end - ), - ok. - -check_error_stats(Cluster, PreviousFailCount, PreviousErrorThresholdCount) -> - {FailCount, ErrorThresholdCount} = get_error_stats(Cluster), - lager:info( - "PreviousFailCount: ~p FailCount: ~p;" - " PreviousErrorThresholdCount: ~p; ErrorThresholdCount: ~p", - [PreviousFailCount, FailCount, - PreviousErrorThresholdCount, ErrorThresholdCount] - ), - PreviousFailCount + ?NVAL == FailCount - andalso PreviousErrorThresholdCount == ErrorThresholdCount. - - -get_error_stats(Cluster) -> - AllStats = [rpc:call(Node, yz_stat, get_stats, []) || Node <- Cluster], - { - lists:sum([get_count([index, bad_entry], count, Stats) || Stats <- AllStats]), - lists:sum([get_count([search_index_error_threshold_failure_count], value, Stats) || Stats <- AllStats]) - }. - -get_count(StatName, Type, Stats) -> - proplists:get_value( - Type, - proplists:get_value( - yz_stat:stat_name(StatName), - Stats - ) - ). diff --git a/tests/yz_handoff.erl b/tests/yz_handoff.erl deleted file mode 100644 index 7ae32ad24..000000000 --- a/tests/yz_handoff.erl +++ /dev/null @@ -1,206 +0,0 @@ -%% ------------------------------------------------------------------- -%% -%% Copyright (c) 2014 Basho Technologies, Inc. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%%------------------------------------------------------------------- --module(yz_handoff). --compile(export_all). --include_lib("eunit/include/eunit.hrl"). --include_lib("riakc/include/riakc.hrl"). - --define(GET(K,L), proplists:get_value(K, L)). --define(FMT(S, Args), lists:flatten(io_lib:format(S, Args))). --define(INDEX, <<"test_idx">>). --define(BUCKET, <<"test_bkt">>). --define(NUMRUNSTATES, 1). --define(SEQMAX, 1000). --define(TESTCYCLE, 20). --define(N, 3). --define(CFG, - [ - {riak_core, - [ - {ring_creation_size, 16}, - {n_val, ?N}, - {handoff_concurrency, 10}, - {vnode_management_timer, 1000} - ]}, - {riak_kv, - [ - %% allow AAE to build trees and exchange rapidly - {anti_entropy_build_limit, {100, 1000}}, - {anti_entropy_concurrency, 8}, - {handoff_rejected_max, infinity} - ]}, - {yokozuna, - [ - {anti_entropy_tick, 1000}, - {enabled, true} - ]} - ]). - --record(trial_state, { - solr_url_before, - solr_url_after, - leave_node, - join_node, - admin_node}). - -confirm() -> - %% Setup cluster initially - [Node1, Node2, _Node3, _Node4, _Node5] = Nodes = rt:build_cluster(5, ?CFG), - - rt:wait_for_cluster_service(Nodes, yokozuna), - - ConnInfo = ?GET(Node2, rt:connection_info([Node2])), - {Host, Port} = ?GET(http, ConnInfo), - Shards = [{N, node_solr_port(N)} || N <- Nodes], - - %% Generate keys, YZ only supports UTF-8 compatible keys - Keys = [<> || N <- lists:seq(1, ?SEQMAX), - not lists:any(fun(E) -> E > 127 end, - binary_to_list(<>))], - KeyCount = length(Keys), - - Pid = rt:pbc(Node2), - yokozuna_rt:write_data(Nodes, Pid, ?INDEX, ?BUCKET, Keys), - - %% Separate out shards for multiple runs - [Shard1|Shards2Rest] = Shards, - {_, SolrPort1} = Shard1, - [{_, SolrPort2}|_] = Shards2Rest, - SolrURL = internal_solr_url(Host, SolrPort1, ?INDEX, Shards), - BucketURL = bucket_keys_url(Host, Port, ?BUCKET), - SearchURL = search_url(Host, Port, ?INDEX), - - lager:info("Verify Replicas Count = (3 * docs/keys) count"), - verify_count(SolrURL, (KeyCount * ?N)), - - States = [#trial_state{solr_url_before = SolrURL, - solr_url_after = internal_solr_url(Host, SolrPort2, ?INDEX, Shards2Rest), - leave_node = Node1}, - #trial_state{solr_url_before = internal_solr_url(Host, SolrPort2, ?INDEX, Shards2Rest), - solr_url_after = SolrURL, - join_node = Node1, - admin_node = Node2}], - - %% Run set of leave/join trials and count/test #'s from the cluster - [[begin - check_data(Nodes, KeyCount, BucketURL, SearchURL, State), - check_counts(Pid, KeyCount, BucketURL) - end || State <- States] - || _ <- lists:seq(1,?NUMRUNSTATES)], - - pass. - -%%%=================================================================== -%%% Private -%%%=================================================================== - -node_solr_port(Node) -> - {ok, P} = riak_core_util:safe_rpc(Node, application, get_env, - [yokozuna, solr_port]), - P. - -internal_solr_url(Host, Port, Index) -> - ?FMT("http://~s:~B/internal_solr/~s", [Host, Port, Index]). -internal_solr_url(Host, Port, Index, Shards) -> - Ss = [internal_solr_url(Host, ShardPort, Index) - || {_, ShardPort} <- Shards], - ?FMT("http://~s:~B/internal_solr/~s/select?wt=json&q=*:*&shards=~s", - [Host, Port, Index, string:join(Ss, ",")]). - -%% @private -bucket_keys_url(Host, Port, BName) -> - ?FMT("http://~s:~B/buckets/~s/keys?keys=true", [Host, Port, BName]). - -%% @private -search_url(Host, Port, Index) -> - ?FMT("http://~s:~B/solr/~s/select?wt=json&q=*:*", [Host, Port, Index]). - -verify_count(Url, ExpectedCount) -> - AreUp = - fun() -> - {ok, "200", _, DBody} = yokozuna_rt:http(get, Url, [], [], "", 60000), - FoundCount = get_count(DBody), - lager:info("FoundCount: ~b, ExpectedCount: ~b", - [FoundCount, ExpectedCount]), - ExpectedCount =:= FoundCount - end, - ?assertEqual(ok, rt:wait_until(AreUp)), - ok. - -get_count(Resp) -> - Struct = mochijson2:decode(Resp), - kvc:path([<<"response">>, <<"numFound">>], Struct). - -get_keys_count(BucketURL) -> - {ok, "200", _, RBody} = yokozuna_rt:http(get, BucketURL, [], []), - Struct = mochijson2:decode(RBody), - length(kvc:path([<<"keys">>], Struct)). - -check_counts(Pid, InitKeyCount, BucketURL) -> - PBCounts = [begin {ok, Resp} = riakc_pb_socket:search( - Pid, ?INDEX, <<"*:*">>), - Resp#search_results.num_found - end || _ <- lists:seq(1,?TESTCYCLE)], - HTTPCounts = [begin {ok, "200", _, RBody} = yokozuna_rt:http( - get, BucketURL, [], []), - Struct = mochijson2:decode(RBody), - length(kvc:path([<<"keys">>], Struct)) - end || _ <- lists:seq(1,?TESTCYCLE)], - MinPBCount = lists:min(PBCounts), - MinHTTPCount = lists:min(HTTPCounts), - lager:info("Before-Node-Leave PB: ~b, After-Node-Leave PB: ~b", - [InitKeyCount, MinPBCount]), - ?assertEqual(InitKeyCount, MinPBCount), - lager:info("Before-Node-Leave PB: ~b, After-Node-Leave HTTP: ~b", - [InitKeyCount, MinHTTPCount]), - ?assertEqual(InitKeyCount, MinHTTPCount). - -check_data(Cluster, KeyCount, BucketURL, SearchURL, S) -> - CheckCount = KeyCount * ?N, - KeysBefore = get_keys_count(BucketURL), - - UpdatedCluster = leave_or_join(Cluster, S), - - yokozuna_rt:wait_for_aae(UpdatedCluster), - - KeysAfter = get_keys_count(BucketURL), - lager:info("KeysBefore: ~b, KeysAfter: ~b", [KeysBefore, KeysAfter]), - ?assertEqual(KeysBefore, KeysAfter), - - lager:info("Verify Search Docs Count =:= key count"), - lager:info("Run Search URL: ~s", [SearchURL]), - verify_count(SearchURL, KeysAfter), - lager:info("Verify Replicas Count = (3 * docs/keys) count"), - lager:info("Run Search URL: ~s", [S#trial_state.solr_url_after]), - verify_count(S#trial_state.solr_url_after, CheckCount). - -leave_or_join(Cluster, S=#trial_state{join_node=undefined}) -> - Node = S#trial_state.leave_node, - rt:leave(Node), - ?assertEqual(ok, rt:wait_until_unpingable(Node)), - Cluster -- [Node]; -leave_or_join(Cluster, S=#trial_state{leave_node=undefined}) -> - Node = S#trial_state.join_node, - NodeAdmin = S#trial_state.admin_node, - ok = rt:start_and_wait(Node), - ok = rt:join(Node, NodeAdmin), - ?assertEqual(ok, rt:wait_until_nodes_ready(Cluster)), - ?assertEqual(ok, rt:wait_until_no_pending_changes(Cluster)), - Cluster ++ [Node]. diff --git a/tests/yz_schema_change_reset.erl b/tests/yz_schema_change_reset.erl deleted file mode 100644 index 3efda592f..000000000 --- a/tests/yz_schema_change_reset.erl +++ /dev/null @@ -1,305 +0,0 @@ -%% ------------------------------------------------------------------- -%% -%% Copyright (c) 2015 Basho Technologies, Inc. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%%-------------------------------------------------------------------- --module(yz_schema_change_reset). --compile(export_all). --include_lib("eunit/include/eunit.hrl"). --include_lib("riakc/include/riakc.hrl"). - --define(GET(K,L), proplists:get_value(K, L)). --define(INDEX, <<"test_schema_change_reset">>). --define(TYPE, <<"test_schema_change">>). --define(BUCKET1, <<"test_schema_change_reset">>). --define(BUCKET2, {?TYPE, <<"test_schema_change_reset_2">>}). --define(SCHEMANAME, <<"test">>). - --define(TEST_SCHEMA, -<<" - - - - - - - - - - - - - - - - -_yz_id - - - - - - - - - - - - - - - - - - - - -">>). --define(TEST_SCHEMA_UPDATE, -<<" - - - - - - - - - - - - - - - - - - - -_yz_id - - - - - - - - - - - - - - - - - - - -">>). - --define(SEQMAX, 20). --define(CFG, - [{riak_core, - [ - {ring_creation_size, 16} - ]}, - {riak_kv, [ - {anti_entropy_concurrency, 8}, - {anti_entropy_build_limit, {100, 1000}} - ]}, - {yokozuna, - [ - {anti_entropy_tick, 1000}, - {enabled, true} - ]} - ]). - -confirm() -> - [Node1|_RestNodes] = Cluster = rt:build_cluster(4, ?CFG), - rt:wait_for_cluster_service(Cluster, yokozuna), - - GenKeys = yokozuna_rt:gen_keys(?SEQMAX), - KeyCount = length(GenKeys), - lager:info("KeyCount ~p", [KeyCount]), - - Pid = rt:pbc(rt:select_random(Cluster)), - - lager:info("Write initial data to index ~p with schema ~p", - [?INDEX, ?SCHEMANAME]), - - yokozuna_rt:write_data(Cluster, Pid, ?INDEX, - {?SCHEMANAME, ?TEST_SCHEMA}, - ?BUCKET1, GenKeys), - lager:info("Create and activate map-based bucket type ~s and tie it to search_index ~s", - [?TYPE, ?INDEX]), - rt:create_and_activate_bucket_type(Node1, ?TYPE, [{datatype, map}, - {search_index, ?INDEX}]), - - lager:info("Write and check age at integer per original schema"), - - NewObj1A = riakc_obj:new(?BUCKET1, <<"keyA">>, - <<"{\"age\":26}">>, - "application/json"), - - NewObj1B = riakc_obj:new(?BUCKET1, <<"keyB">>, - <<"{\"age\":99}">>, - "application/json"), - - {ok, _ObjA} = riakc_pb_socket:put(Pid, NewObj1A, [return_head]), - yokozuna_rt:commit(Cluster, ?INDEX), - {ok, _ObjB} = riakc_pb_socket:put(Pid, NewObj1B, [return_head]), - yokozuna_rt:commit(Cluster, ?INDEX), - - yokozuna_rt:verify_num_found_query(Cluster, ?INDEX, KeyCount + 2), - - yokozuna_rt:assert_search(Pid, Cluster, ?INDEX, - <<"age:26">>, {<<"age">>, <<"26">>}, []), - yokozuna_rt:assert_search(Pid, Cluster, ?INDEX, - <<"age:99">>, {<<"age">>, <<"99">>}, []), - - Map1 = riakc_map:update( - {<<"0_foo">>, register}, - fun(R) -> - riakc_register:set(<<"44ab">>, R) - end, riakc_map:new()), - ok = riakc_pb_socket:update_type( - Pid, - ?BUCKET2, - <<"keyMap1">>, - riakc_map:to_op(Map1)), - - {ok, Map2} = riakc_pb_socket:fetch_type(Pid, ?BUCKET2, <<"keyMap1">>), - Map3 = riakc_map:update( - {<<"1_baz">>, counter}, - fun(R) -> - riakc_counter:increment(10, R) - end, Map2), - ok = riakc_pb_socket:update_type( - Pid, - ?BUCKET2, - <<"keyMap1">>, - riakc_map:to_op(Map3)), - - yokozuna_rt:commit(Cluster, ?INDEX), - yokozuna_rt:assert_search(Pid, Cluster, ?INDEX, - <<"0_foo_register:44ab">>, - {<<"0_foo_register">>, <<"44ab">>}, - []), - - lager:info("Expire and re-check count before updating schema"), - - yokozuna_rt:expire_trees(Cluster), - yokozuna_rt:wait_for_aae(Cluster), - - yokozuna_rt:verify_num_found_query(Cluster, ?INDEX, KeyCount + 3), - - yokozuna_rt:override_schema(Pid, Cluster, ?INDEX, ?SCHEMANAME, ?TEST_SCHEMA_UPDATE), - - lager:info("Write and check hello_i at integer per schema update"), - - NewObj2 = riakc_obj:new(?BUCKET1, <<"key2">>, - <<"{\"hello_i\":36}">>, - "application/json"), - - {ok, _Obj2} = riakc_pb_socket:put(Pid, NewObj2, [return_head]), - yokozuna_rt:commit(Cluster, ?INDEX), - - yokozuna_rt:verify_num_found_query(Cluster, ?INDEX, KeyCount + 4), - yokozuna_rt:assert_search(Pid, Cluster, ?INDEX, - <<"hello_i:36">>, {<<"hello_i">>, <<"36">>}, []), - - lager:info("Write and check age at string per schema update"), - - NewObj3 = riakc_obj:new(?BUCKET1, <<"key3">>, - <<"{\"age\":\"3jlkjkl\"}">>, - "application/json"), - - {ok, _Obj3} = riakc_pb_socket:put(Pid, NewObj3, [return_head]), - yokozuna_rt:commit(Cluster, ?INDEX), - - yokozuna_rt:verify_num_found_query(Cluster, ?INDEX, KeyCount + 5), - yokozuna_rt:assert_search(Pid, Cluster, ?INDEX, - <<"age:3jlkjkl">>, {<<"age">>, <<"3jlkjkl">>}, - []), - - lager:info("Expire and re-check count to make sure we're correctly indexed - by the new schema"), - - yokozuna_rt:expire_trees(Cluster), - yokozuna_rt:wait_for_aae(Cluster), - - yokozuna_rt:verify_num_found_query(Cluster, ?INDEX, KeyCount + 5), - - ANode = rt:select_random(Cluster), - yokozuna_rt:search_expect(ANode, ?INDEX, <<"age">>, <<"*">>, 3), - - lager:info("Re-Put because AAE won't find a diff even though the types - have changed, as it only compares based on bkey currently. - Also, this re-put will work as we have a default bucket (type) - with allow_mult=false... no siblings"), - - {ok, _Obj4} = riakc_pb_socket:put(Pid, NewObj1A, [return_head]), - yokozuna_rt:commit(Cluster, ?INDEX), - - yokozuna_rt:assert_search(Pid, Cluster, ?INDEX, - <<"age:26">>, {<<"age">>, <<"26">>}, []), - - lager:info("Re-Put Map data by dec/inc counter to account for *change* and - allow previously unindexed counter to be searchable"), - - {ok, Map4} = riakc_pb_socket:fetch_type(Pid, ?BUCKET2, <<"keyMap1">>), - Map5 = riakc_map:update( - {<<"1_baz">>, counter}, - fun(R) -> - riakc_counter:decrement(0, R), - riakc_counter:increment(0, R) - end, Map4), - ok = riakc_pb_socket:update_type( - Pid, - ?BUCKET2, - <<"keyMap1">>, - riakc_map:to_op(Map5)), - - yokozuna_rt:commit(Cluster, ?INDEX), - yokozuna_rt:assert_search(Pid, Cluster, ?INDEX, - <<"0_foo_register:44ab">>, - {<<"0_foo_register">>, <<"44ab">>}, - []), - yokozuna_rt:assert_search(Pid, Cluster, ?INDEX, - <<"1_baz_counter:10">>, - {<<"1_baz_counter">>, <<"10">>}, - []), - - lager:info("Test nested json searches w/ unsearched fields ignored"), - - NewObj5 = riakc_obj:new(?BUCKET1, <<"key4">>, - <<"{\"quip\":\"blashj3\", - \"paths\":{\"quip\":\"88\"}}">>, - "application/json"), - {ok, _Obj5} = riakc_pb_socket:put(Pid, NewObj5, [return_head]), - - yokozuna_rt:commit(Cluster, ?INDEX), - yokozuna_rt:assert_search(Pid, Cluster, ?INDEX, - <<"paths.quip:88">>, - {<<"paths.quip">>, <<"88">>}, - []), - - riakc_pb_socket:stop(Pid), - - pass. - From bbb85a23aab1ee854e400aa4042a913ea77e22f9 Mon Sep 17 00:00:00 2001 From: Brett Hazen Date: Tue, 28 Feb 2017 14:09:17 -0700 Subject: [PATCH 19/35] Fix test failures caused by riak client not allowing list keys --- tests/bucket_types.erl | 4 ++ tests/http_security.erl | 3 ++ tests/overload.erl | 5 +- tests/pb_security.erl | 3 ++ tests/rolling_capabilities.erl | 5 +- tests/ts_cluster_capabilities_SUITE.erl | 53 +++++++-------------- tests/ts_cluster_comprehensive.erl | 5 +- tests/ts_cluster_list_irreg_keys_SUITE.erl | 5 +- tests/ts_cluster_stream_list_keys_SUITE.erl | 5 +- tests/ts_simple_pb_security_SUITE.erl | 5 +- tests/verify_api_timeouts.erl | 3 ++ tests/verify_capabilities.erl | 5 +- tests/verify_dt_data_upgrade.erl | 6 +-- tests/verify_job_enable_ac.erl | 5 +- tests/verify_job_enable_rc.erl | 5 +- tests/verify_listkeys.erl | 5 +- tests/verify_listkeys_eqcfsm.erl | 3 ++ tests/verify_mr_prereduce_node_down.erl | 5 +- 18 files changed, 80 insertions(+), 50 deletions(-) diff --git a/tests/bucket_types.erl b/tests/bucket_types.erl index 7e426be5a..3ea0e2052 100644 --- a/tests/bucket_types.erl +++ b/tests/bucket_types.erl @@ -8,6 +8,10 @@ confirm() -> application:start(inets), lager:info("Deploy some nodes"), + + %% Allow listing of buckets and keys for testing + application:set_env(riakc, allow_listing, true), + Nodes = rt:build_cluster(4, [], [ {riak_core, [{default_bucket_props, [ diff --git a/tests/http_security.erl b/tests/http_security.erl index fac70e329..413fb4c82 100644 --- a/tests/http_security.erl +++ b/tests/http_security.erl @@ -11,6 +11,9 @@ -define(assertDenied(Op), ?assertMatch({error, {forbidden, _}}, Op)). confirm() -> + %% Allow listing of buckets and keys for testing + application:set_env(riakc, allow_listing, true), + application:start(crypto), application:start(asn1), application:start(public_key), diff --git a/tests/overload.erl b/tests/overload.erl index 8f5aab142..9be681b3c 100644 --- a/tests/overload.erl +++ b/tests/overload.erl @@ -1,6 +1,6 @@ %% ------------------------------------------------------------------- %% -%% Copyright (c) 2013 Basho Technologies, Inc. +%% Copyright (c) 2013-2017 Basho Technologies, Inc. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -81,6 +81,9 @@ default_config(#config{ {riak_api, [{pb_backlog, 1024}]}]. confirm() -> + %% Allow listing of buckets and keys for testing + application:set_env(riakc, allow_listing, true), + [Node1 | _] = Nodes = setup(), ok = create_bucket_type(Nodes, ?NORMAL_TYPE, [{n_val, 3}]), diff --git a/tests/pb_security.erl b/tests/pb_security.erl index e43a5ac1d..531ffe239 100644 --- a/tests/pb_security.erl +++ b/tests/pb_security.erl @@ -11,6 +11,9 @@ -define(assertDenied(Op), ?assertMatch({error, <<"Permission",_/binary>>}, Op)). confirm() -> + %% Allow listing of buckets and keys for testing + application:set_env(riakc, allow_listing, true), + application:start(crypto), application:start(asn1), application:start(public_key), diff --git a/tests/rolling_capabilities.erl b/tests/rolling_capabilities.erl index 5b05788b4..716a4dddd 100644 --- a/tests/rolling_capabilities.erl +++ b/tests/rolling_capabilities.erl @@ -1,6 +1,6 @@ %% ------------------------------------------------------------------- %% -%% Copyright (c) 2012 Basho Technologies, Inc. +%% Copyright (c) 2012-2017 Basho Technologies, Inc. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -23,6 +23,9 @@ -include_lib("eunit/include/eunit.hrl"). confirm() -> + %% Allow listing of buckets and keys for testing + application:set_env(riakc, allow_listing, true), + TestMetaData = riak_test_runner:metadata(), Count = 4, OldVsn = proplists:get_value(upgrade_version, TestMetaData, previous), diff --git a/tests/ts_cluster_capabilities_SUITE.erl b/tests/ts_cluster_capabilities_SUITE.erl index b055eb042..30e84dfd9 100644 --- a/tests/ts_cluster_capabilities_SUITE.erl +++ b/tests/ts_cluster_capabilities_SUITE.erl @@ -1,6 +1,6 @@ %% ------------------------------------------------------------------- %% -%% Copyright (c) 2016 Basho Technologies, Inc. +%% Copyright (c) 2016-2017 Basho Technologies, Inc. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -30,19 +30,13 @@ %%-------------------------------------------------------------------- -define(TS_VERSION_CURRENT, "current"). - -%% git checkout riak_ts-1.3.1 && make locked-deps --define(TS_VERSION_1_3, riak_ts_1_3_1). - -define(SQL_SELECT_CAP, {riak_kv, sql_select_version}). suite() -> [{timetrap,{minutes,10}}]. init_per_suite(Config) -> - Versions = rt:find_version_by_name(["riak_ts-1.3.1", "riak_ts_ee-1.3.1"]), - Vsn131 = maybe_missing_1_3_1(Versions), - [{?TS_VERSION_1_3, Vsn131} | Config]. + Config. end_per_suite(_Config) -> ok. @@ -143,21 +137,19 @@ capability_not_specified_on_one_node_test(_Ctx) -> %% Riak TS Capability Tests %%-------------------------------------------------------------------- -sql_select_upgrade_a_node_from_1_3_test(Config) -> - Vsn131 = ?config(?TS_VERSION_1_3, Config), +sql_select_upgrade_a_node_from_legacy_test(_Ctx) -> [Node_A, Node_B, Node_C] = - rt:deploy_nodes([Vsn131, Vsn131, Vsn131]), + rt:deploy_nodes([legacy, legacy, legacy]), ok = rt:join_cluster([Node_A,Node_B,Node_C]), ok = rt:wait_until_ring_converged([Node_A,Node_B,Node_C]), ok = rt:upgrade(Node_A, ?TS_VERSION_CURRENT), ok = rt:wait_until_ring_converged([Node_A,Node_B,Node_C]), - ok = rt:wait_until_capability(Node_A, ?SQL_SELECT_CAP, v1), + ok = rt:wait_until_capability(Node_A, ?SQL_SELECT_CAP, v2), ok. -sql_select_join_with_all_nodes_upgraded_test(Config) -> - Vsn131 = ?config(?TS_VERSION_1_3, Config), +sql_select_join_with_all_nodes_upgraded_test(_Ctx) -> [Node_A, Node_B, Node_C] = - rt:deploy_nodes([Vsn131, Vsn131, Vsn131]), + rt:deploy_nodes([legacy, legacy, legacy]), ok = rt:join_cluster([Node_A,Node_B,Node_C]), rt:wait_until_ring_converged([Node_A,Node_B,Node_C]), rt:upgrade(Node_A, ?TS_VERSION_CURRENT), @@ -168,18 +160,14 @@ sql_select_join_with_all_nodes_upgraded_test(Config) -> rt:wait_until_capability(Node_C, ?SQL_SELECT_CAP, v3), ok. -%% This test passed for 1.4.0, expecting v1 where commented, but fails for 1.5.0 due to code change. We set it to expect -%% v3 until future changes are made. After 1.5.0 if this test will fail if new code is introduced to handle capability -%% communication. https://bashoeng.atlassian.net/browse/RTS-1415 -sql_select_downgrade_a_node_test(Config) -> - Vsn131 = ?config(?TS_VERSION_1_3, Config), +sql_select_downgrade_a_node_test(_Ctx) -> [Node_A, Node_B, Node_C] = - rt:deploy_nodes([Vsn131, ?TS_VERSION_CURRENT, ?TS_VERSION_CURRENT]), + rt:deploy_nodes([legacy, ?TS_VERSION_CURRENT, ?TS_VERSION_CURRENT]), ok = rt:join_cluster([Node_A,Node_B,Node_C]), rt:wait_until_ring_converged([Node_A,Node_B,Node_C]), % rt:wait_until_capability(Node_A, ?SQL_SELECT_CAP, v3), - rt:wait_until_capability(Node_B, ?SQL_SELECT_CAP, v1), - rt:wait_until_capability(Node_C, ?SQL_SELECT_CAP, v1), + rt:wait_until_capability(Node_B, ?SQL_SELECT_CAP, v2), + rt:wait_until_capability(Node_C, ?SQL_SELECT_CAP, v2), rt:upgrade(Node_A, ?TS_VERSION_CURRENT), rt:wait_until_ring_converged([Node_A,Node_B,Node_C]), rt:wait_until_capability(Node_A, ?SQL_SELECT_CAP, v3), @@ -189,8 +177,8 @@ sql_select_downgrade_a_node_test(Config) -> %% pending changes to how capabilities are communicated around the ring, this %% section will expect v2. Once capabilities are changed in a future version of riak %% (or if testing 1.4.0 as current) - %% the expected version should go back to v1. - rt:upgrade(Node_A, Vsn131), + %% the expected version should go back to v2. + rt:upgrade(Node_A, legacy), rt:wait_until_ring_converged([Node_A,Node_B,Node_C]), rt:wait_until_capability(Node_B, ?SQL_SELECT_CAP, v3), rt:wait_until_capability(Node_C, ?SQL_SELECT_CAP, v3), @@ -200,10 +188,9 @@ sql_select_downgrade_a_node_test(Config) -> %% Perform queries in mixed version cluster %%-------------------------------------------------------------------- -query_in_mixed_version_cluster_test(Config) -> - Vsn131 = ?config(?TS_VERSION_1_3, Config), +query_in_mixed_version_cluster_test(_Ctx) -> [Node_A, Node_B, Node_C] = - rt:deploy_nodes([?TS_VERSION_CURRENT, Vsn131, ?TS_VERSION_CURRENT]), + rt:deploy_nodes([?TS_VERSION_CURRENT, previous, ?TS_VERSION_CURRENT]), ok = rt:join_cluster([Node_A,Node_B,Node_C]), rt:wait_until_ring_converged([Node_A,Node_B,Node_C]), Table = "grouptab1", @@ -220,7 +207,7 @@ query_in_mixed_version_cluster_test(Config) -> [{1,1,B*C} || B <- lists:seq(1,10), C <- lists:seq(1000,5000,1000)]), ExpectedResultSet = [{N} || N <- lists:seq(1000,5000,1000)], %% - %% Test that the current version can query version 1.3 + %% Test that the current version can query older version %% Query = "SELECT c FROM grouptab1 " @@ -232,7 +219,7 @@ query_in_mixed_version_cluster_test(Config) -> {ok,{Cols, Rows}} ), %% - %% Test that the 1.3 can query the current version + %% Test that the previous can query the current version %% {ok, {Cols, Rows}} = run_query(rt:pbc(Node_B), Query), ts_data:assert_row_sets( @@ -240,9 +227,3 @@ query_in_mixed_version_cluster_test(Config) -> {ok,{Cols, Rows}} ). -%% If 1.3.1 is not found, use a meaningful atom to prompt the error message -%% rtdev:check_version/1 -maybe_missing_1_3_1([]) -> - missing_version_ts_1_3_1; -maybe_missing_1_3_1([Vsn]) -> - Vsn. diff --git a/tests/ts_cluster_comprehensive.erl b/tests/ts_cluster_comprehensive.erl index d48019ac0..5e06299a1 100644 --- a/tests/ts_cluster_comprehensive.erl +++ b/tests/ts_cluster_comprehensive.erl @@ -1,6 +1,6 @@ % ------------------------------------------------------------------- %% -%% Copyright (c) 2015 Basho Technologies, Inc. +%% Copyright (c) 2015-2017 Basho Technologies, Inc. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -39,6 +39,9 @@ confirm() -> run_tests(?PVAL_P1, ?PVAL_P2). run_tests(PvalP1, PvalP2) -> + %% Allow listing of buckets and keys for testing + application:set_env(riakc, allow_listing, true), + Data = make_data(PvalP1, PvalP2), io:format("Data to be written:\n~p\n...\n~p\n", [hd(Data), lists:last(Data)]), diff --git a/tests/ts_cluster_list_irreg_keys_SUITE.erl b/tests/ts_cluster_list_irreg_keys_SUITE.erl index b0ee90f8a..b885f69b6 100644 --- a/tests/ts_cluster_list_irreg_keys_SUITE.erl +++ b/tests/ts_cluster_list_irreg_keys_SUITE.erl @@ -1,6 +1,6 @@ %% ------------------------------------------------------------------- %% -%% Copyright (c) 2016 Basho Technologies, Inc. +%% Copyright (c) 2016-2017 Basho Technologies, Inc. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -39,6 +39,9 @@ %%-------------------------------------------------------------------- suite() -> + %% Allow listing of buckets and keys for testing + application:set_env(riakc, allow_listing, true), + [{timetrap,{minutes,10}}]. init_per_suite(Config) -> diff --git a/tests/ts_cluster_stream_list_keys_SUITE.erl b/tests/ts_cluster_stream_list_keys_SUITE.erl index 9553d66ef..8c48d9b9f 100644 --- a/tests/ts_cluster_stream_list_keys_SUITE.erl +++ b/tests/ts_cluster_stream_list_keys_SUITE.erl @@ -1,6 +1,6 @@ %% ------------------------------------------------------------------- %% -%% Copyright (c) 2016 Basho Technologies, Inc. +%% Copyright (c) 2016-2017 Basho Technologies, Inc. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -31,6 +31,9 @@ %%-------------------------------------------------------------------- suite() -> + %% Allow listing of buckets and keys for testing + application:set_env(riakc, allow_listing, true), + [{timetrap,{minutes,10}}]. init_per_suite(Config) -> diff --git a/tests/ts_simple_pb_security_SUITE.erl b/tests/ts_simple_pb_security_SUITE.erl index a4ca869ef..4d0f9f914 100644 --- a/tests/ts_simple_pb_security_SUITE.erl +++ b/tests/ts_simple_pb_security_SUITE.erl @@ -1,6 +1,6 @@ %% ------------------------------------------------------------------- %% -%% Copyright (c) 2016 Basho Technologies, Inc. +%% Copyright (c) 2016-2017 Basho Technologies, Inc. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -31,6 +31,9 @@ %%-------------------------------------------------------------------- suite() -> + %% Allow listing of buckets and keys for testing + application:set_env(riakc, allow_listing, true), + [{timetrap,{minutes,10}}]. init_per_suite(Config) -> diff --git a/tests/verify_api_timeouts.erl b/tests/verify_api_timeouts.erl index 0a6851cfd..1ddbe878f 100644 --- a/tests/verify_api_timeouts.erl +++ b/tests/verify_api_timeouts.erl @@ -8,6 +8,9 @@ -define(NUM_KEYS, 1000). confirm() -> + %% Allow listing of buckets and keys for testing + application:set_env(riakc, allow_listing, true), + %% test requires allow_mult=false b/c of rt:systest_read [Node] = rt:build_cluster(1), rt:wait_until_pingable(Node), diff --git a/tests/verify_capabilities.erl b/tests/verify_capabilities.erl index a1ff966df..22407f35b 100644 --- a/tests/verify_capabilities.erl +++ b/tests/verify_capabilities.erl @@ -1,6 +1,6 @@ %% ------------------------------------------------------------------- %% -%% Copyright (c) 2012-2013 Basho Technologies, Inc. +%% Copyright (c) 2012-2017 Basho Technologies, Inc. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -25,6 +25,9 @@ %% 1.4 {riak_kv, handoff_data_encoding} -> [encode_raw, encode_zlib] %% 1.3 {riak_kv, anti_entropy} -> [disabled, enabled_v1] confirm() -> + %% Allow listing of buckets and keys for testing + application:set_env(riakc, allow_listing, true), + lager:info("Deploying mixed set of nodes"), Legacy = case lists:member(legacy, rt:versions()) of true -> legacy; diff --git a/tests/verify_dt_data_upgrade.erl b/tests/verify_dt_data_upgrade.erl index ac9435676..907ce4b26 100644 --- a/tests/verify_dt_data_upgrade.erl +++ b/tests/verify_dt_data_upgrade.erl @@ -1,6 +1,6 @@ %% ------------------------------------------------------------------- %% -%% Copyright (c) 2016 Basho Technologies, Inc. +%% Copyright (c) 2016-2017 Basho Technologies, Inc. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -47,7 +47,7 @@ confirm() -> TestMetaData = riak_test_runner:metadata(), - OldVsn = proplists:get_value(upgrade_version, TestMetaData, lts), + OldVsn = proplists:get_value(upgrade_version, TestMetaData, legacy), NumNodes = 4, Vsns = [{OldVsn, ?CONFIG} || _ <- lists:seq(1, NumNodes)], @@ -116,7 +116,7 @@ confirm() -> ?assertEqual(FetchSet2, FetchSet3), %% Downgrade All Nodes and Compare - downgrade(Nodes, lts), + downgrade(Nodes, legacy), %% Create PB connection. Pid3 = rt:pbc(rt:select_random(Nodes)), diff --git a/tests/verify_job_enable_ac.erl b/tests/verify_job_enable_ac.erl index 0acf920cb..faa4091a3 100644 --- a/tests/verify_job_enable_ac.erl +++ b/tests/verify_job_enable_ac.erl @@ -1,6 +1,6 @@ %% ------------------------------------------------------------------- %% -%% Copyright (c) 2016 Basho Technologies, Inc. +%% Copyright (c) 2016-2017 Basho Technologies, Inc. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -36,6 +36,9 @@ [Class || {Class, Enabled} <- ?JOB_CLASS_DEFAULTS, Enabled]). confirm() -> + %% Allow listing of buckets and keys for testing + application:set_env(riakc, allow_listing, true), + lager:info("Deploying 1 node"), rt:set_backend(eleveldb), [Node] = rt:build_cluster(1, ?CFG), diff --git a/tests/verify_job_enable_rc.erl b/tests/verify_job_enable_rc.erl index 511690bd3..076302888 100644 --- a/tests/verify_job_enable_rc.erl +++ b/tests/verify_job_enable_rc.erl @@ -1,6 +1,6 @@ %% ------------------------------------------------------------------- %% -%% Copyright (c) 2016 Basho Technologies, Inc. +%% Copyright (c) 2016-2017 Basho Technologies, Inc. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -47,6 +47,9 @@ %% =================================================================== confirm() -> + %% Allow listing of buckets and keys for testing + application:set_env(riakc, allow_listing, true), + Configs = [ {current, {cuttlefish, ?COMMON_CONFIG ++ config(?TEST_OPS, Bool, [])}} diff --git a/tests/verify_listkeys.erl b/tests/verify_listkeys.erl index 33af44937..66a77f54a 100644 --- a/tests/verify_listkeys.erl +++ b/tests/verify_listkeys.erl @@ -1,6 +1,6 @@ %% ------------------------------------------------------------------- %% -%% Copyright (c) 2012 Basho Technologies, Inc. +%% Copyright (c) 2012-2017 Basho Technologies, Inc. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -29,6 +29,9 @@ -define(UNDEFINED_BUCKET_TYPE, <<"880bf69d-5dab-44ee-8762-d24c6f759ce1">>). confirm() -> + %% Allow listing of buckets and keys for testing + application:set_env(riakc, allow_listing, true), + [Node1, Node2, Node3, Node4] = Nodes = rt:deploy_nodes(4), ?assertEqual(ok, rt:wait_until_nodes_ready(Nodes)), diff --git a/tests/verify_listkeys_eqcfsm.erl b/tests/verify_listkeys_eqcfsm.erl index d8f5f01ee..66d209674 100644 --- a/tests/verify_listkeys_eqcfsm.erl +++ b/tests/verify_listkeys_eqcfsm.erl @@ -26,6 +26,9 @@ %% riak_test callback %% ==================================================================== confirm() -> + %% Allow listing of buckets and keys for testing + application:set_env(riakc, allow_listing, true), + ?assert(eqc:quickcheck(eqc:numtests(?NUM_TESTS, ?MODULE:prop_test()))), pass. %% ==================================================================== diff --git a/tests/verify_mr_prereduce_node_down.erl b/tests/verify_mr_prereduce_node_down.erl index 265dd7a29..796c7b2e5 100644 --- a/tests/verify_mr_prereduce_node_down.erl +++ b/tests/verify_mr_prereduce_node_down.erl @@ -1,6 +1,6 @@ %% ------------------------------------------------------------------- %% -%% Copyright (c) 2013 Basho Technologies, Inc. +%% Copyright (c) 2013-2017 Basho Technologies, Inc. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -42,6 +42,9 @@ %% @doc riak_test callback confirm() -> + %% Allow listing of buckets and keys for testing + application:set_env(riakc, allow_listing, true), + NodeCount = 4, lager:info("Build ~b-node cluster", [NodeCount]), [Primary,ToKill|_] = rt:build_cluster(NodeCount), From 0c1e5cf1f89afe4e38e17fff53264d667e70f439 Mon Sep 17 00:00:00 2001 From: Brett Hazen Date: Wed, 1 Mar 2017 15:26:53 -0700 Subject: [PATCH 20/35] Update previous = 2.2.0 and legacy = 2.0.8 --- basho_builds.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/basho_builds.yml b/basho_builds.yml index 1d2c4e505..368009993 100644 --- a/basho_builds.yml +++ b/basho_builds.yml @@ -1,13 +1,13 @@ docker_rt_riak_test_config: - version: current product_version: current - product: "riak_ts{% if ee == 'true' %}_ee{% endif %}" + product: "riak{% if ee == 'true' %}_ee{% endif %}" - version: previous - product_version: 1.5.2 - product: "riak_ts{% if ee == 'true' %}_ee{% endif %}" + product_version: 2.2.0 + product: "riak{% if ee == 'true' %}_ee{% endif %}" - version: legacy - product_version: 1.4.1 - product: "riak_ts{% if ee == 'true' %}_ee{% endif %}" + product_version: 2.0.8 + product: "riak{% if ee == 'true' %}_ee{% endif %}" - version: 2.0.2 product_version: 2.0.2 product: "riak{% if ee == 'true' %}_ee{% endif %}" From 6a23c8208b47edd2be84078140d6291f008e6e28 Mon Sep 17 00:00:00 2001 From: Andrei Zavada Date: Fri, 3 Mar 2017 02:56:57 +0200 Subject: [PATCH 21/35] updowngrade: better prepare riak.conf & caps to wait_until make code version-agnostic, only require newly introduced keys and caps to be registered when stepping to next release up/downgrade cycle. --- .../ts_cluster_updowngrade_group_by_SUITE.erl | 7 +- .../ts_cluster_updowngrade_order_by_SUITE.erl | 6 +- ...r_updowngrade_select_aggregation_SUITE.erl | 6 +- tests/ts_updown_util.erl | 93 +++++++++++++++---- 4 files changed, 85 insertions(+), 27 deletions(-) diff --git a/tests/ts_cluster_updowngrade_group_by_SUITE.erl b/tests/ts_cluster_updowngrade_group_by_SUITE.erl index 2bfdcfa1f..d20b914c5 100644 --- a/tests/ts_cluster_updowngrade_group_by_SUITE.erl +++ b/tests/ts_cluster_updowngrade_group_by_SUITE.erl @@ -1,6 +1,6 @@ %% ------------------------------------------------------------------- %% -%% Copyright (c) 2016 Basho Technologies, Inc. +%% Copyright (c) 2016, 2017 Basho Technologies, Inc. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -40,8 +40,8 @@ make_scenarios() -> need_query_node_transition = NeedQueryNodeTransition, need_pre_cluster_mixed = NeedPreClusterMixed, need_post_cluster_mixed = NeedPostClusterMixed, - ensure_full_caps = [{{riak_kv, sql_select_version}, v3}, {{riak_kv, riak_ql_ddl_rec_version}, v2}, {{riak_kv, decode_query_results_at_vnode}, true}], - ensure_degraded_caps = [{{riak_kv, sql_select_version}, v2}, {{riak_kv, riak_ql_ddl_rec_version}, v1}, {{riak_kv, decode_query_results_at_vnode}, false}], + ensure_full_caps = ts_updown_util:caps_to_ensure(full), + ensure_degraded_caps = ts_updown_util:caps_to_ensure(degraded), convert_config_to_previous = fun ts_updown_util:convert_riak_conf_to_previous/1} || TableNodeVsn <- [previous, current], QueryNodeVsn <- [previous, current], @@ -51,6 +51,7 @@ make_scenarios() -> NeedPostClusterMixed <- [true, false]], [add_tests(X) || X <- BaseScenarios]. + %% This test will not use config invariants %% see ts_cluster_updowngrade_select_aggregation_SUITE.erl for an example %% of how to use them diff --git a/tests/ts_cluster_updowngrade_order_by_SUITE.erl b/tests/ts_cluster_updowngrade_order_by_SUITE.erl index cbda22f41..d1a79397d 100644 --- a/tests/ts_cluster_updowngrade_order_by_SUITE.erl +++ b/tests/ts_cluster_updowngrade_order_by_SUITE.erl @@ -1,6 +1,6 @@ %% ------------------------------------------------------------------- %% -%% Copyright (c) 2016 Basho Technologies, Inc. +%% Copyright (c) 2016, 2017 Basho Technologies, Inc. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -60,8 +60,8 @@ make_scenarios() -> need_query_node_transition = NeedQueryNodeTransition, need_pre_cluster_mixed = NeedPreClusterMixed, need_post_cluster_mixed = NeedPostClusterMixed, - ensure_full_caps = [{{riak_kv, sql_select_version}, v3}, {{riak_kv, riak_ql_ddl_rec_version}, v2}, {{riak_kv, decode_query_results_at_vnode}, true}], - ensure_degraded_caps = [{{riak_kv, sql_select_version}, v2}, {{riak_kv, riak_ql_ddl_rec_version}, v1}, {{riak_kv, decode_query_results_at_vnode}, false}], + ensure_full_caps = ts_updown_util:caps_to_ensure(full), + ensure_degraded_caps = ts_updown_util:caps_to_ensure(degraded), convert_config_to_previous = fun ts_updown_util:convert_riak_conf_to_previous/1} || TableNodeVsn <- [previous, current], QueryNodeVsn <- [current], diff --git a/tests/ts_cluster_updowngrade_select_aggregation_SUITE.erl b/tests/ts_cluster_updowngrade_select_aggregation_SUITE.erl index fb5b48123..67b3ffb2c 100644 --- a/tests/ts_cluster_updowngrade_select_aggregation_SUITE.erl +++ b/tests/ts_cluster_updowngrade_select_aggregation_SUITE.erl @@ -1,6 +1,6 @@ %% ------------------------------------------------------------------- %% -%% Copyright (c) 2016 Basho Technologies, Inc. +%% Copyright (c) 2016, 2017 Basho Technologies, Inc. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -35,8 +35,8 @@ make_scenarios() -> need_query_node_transition = NeedQueryNodeTransition, need_pre_cluster_mixed = NeedPreClusterMixed, need_post_cluster_mixed = NeedPostClusterMixed, - ensure_full_caps = [{{riak_kv, sql_select_version}, v3}, {{riak_kv, riak_ql_ddl_rec_version}, v2}, {{riak_kv, decode_query_results_at_vnode}, true}], - ensure_degraded_caps = [{{riak_kv, sql_select_version}, v2}, {{riak_kv, riak_ql_ddl_rec_version}, v1}, {{riak_kv, decode_query_results_at_vnode}, false}], + ensure_full_caps = ts_updown_util:caps_to_ensure(full), + ensure_degraded_caps = ts_updown_util:caps_to_ensure(degraded), convert_config_to_previous = fun ts_updown_util:convert_riak_conf_to_previous/1} || TableNodeVsn <- [previous, current], QueryNodeVsn <- [previous, current], diff --git a/tests/ts_updown_util.erl b/tests/ts_updown_util.erl index 7f5b62706..8e783eca4 100644 --- a/tests/ts_updown_util.erl +++ b/tests/ts_updown_util.erl @@ -1,6 +1,6 @@ %% ------------------------------------------------------------------- %% -%% Copyright (c) 2016 Basho Technologies, Inc. +%% Copyright (c) 2016, 2017 Basho Technologies, Inc. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -50,6 +50,7 @@ -module(ts_updown_util). -export([ + caps_to_ensure/1, convert_riak_conf_to_previous/1, maybe_shutdown_client_node/1, setup/1, @@ -581,31 +582,87 @@ wait_until_active_table(TargetNode, PrevClientNode, Table, N) -> make_msg(Format, Payload) -> list_to_binary(fmt(Format, Payload)). + +get_riak_release_in_slot(VsnSlot) -> + case rtdev:get_version(VsnSlot) of + unknown -> + ct:fail("Failed to determine riak version in '~s' slot", [VsnSlot]); + Known -> + case re:run(Known, "riak_ts-(\\d+)\.(\\d+)\.(\\d+)", [{capture, all_but_first, list}]) of + {match, [V1, V2, V3]} -> + {list_to_integer(V1), + list_to_integer(V2), + list_to_integer(V3)}; + nomatch -> + ct:fail("Failed to parse riak version in '~s' slot", [VsnSlot]) + end + end. + +%% --------------------------------- + +%% Dealing with case-by-case upgrade particulars, such as: +%% +%% * newly introduced keys in riak.conf that need to be deleted on +%% downgrade; +%% +%% * capabilities that need to be ensured before running the tests +%% (arguments to `wait_until_capability`). + %% We need to comment out those settings which appear in version %% 1.x. For version 1.x-1 to work with riak.conf initially created in %% 1.x, the offending settings need to be deleted. We do it here, by %% commenting them out. -%% riak.conf created under 1.6 cannot be read by 1.5 because of new keys: -%% riak_kv.query.timeseries.qbuf_inmem_max, -%% -%% riak.conf created under 1.5 cannot be read by 1.4 because of new keys: -%% riak_kv.query.timeseries.max_returned_data_size, -%% riak_kv.query.timeseries.qbuf_soft_watermark, -%% riak_kv.query.timeseries.qbuf_hard_watermark, -%% riak_kv.query.timeseries.qbuf_expire_ms, -%% riak_kv.query.timeseries.qbuf_incomplete_release_ms -%% convert_riak_conf_to_previous(Config) -> DatafPath = ?CFG(new_data_dir, Config), RiakConfPath = filename:join(DatafPath, "../etc/riak.conf"), - {ok, Content0} = file:read_file(RiakConfPath), - Content9 = - re:replace( - Content0, - <<"^riak_kv.query.timeseries.qbuf_inmem_max">>, - <<"#\\1">>, [global, multiline, {return, binary}]), - ok = file:write_file(RiakConfPath, Content9). + {ok, Contents0} = file:read_file(RiakConfPath), + Contents9 = + lists:foldl( + fun(KeyToDelete, Contents) -> + re:replace(Contents, ["^", KeyToDelete], "#\\1", + [global, multiline, {return, list}]) + end, + Contents0, + get_riak_conf_new_keys()), + ok = file:write_file(RiakConfPath, Contents9). + +%% When a new release is cut, register newly introduced keys here: +get_riak_conf_new_keys() -> + %% the current version may have not been tagged yet, so look at + %% previous version + case get_riak_release_in_slot(previous) of + {1, 5, _} -> + ["riak_kv.query.timeseries.qbuf_inmem_max"]; + {1, 4, _} -> + ["riak_kv.query.timeseries.max_returned_data_size", + "riak_kv.query.timeseries.qbuf_soft_watermark", + "riak_kv.query.timeseries.qbuf_hard_watermark", + "riak_kv.query.timeseries.qbuf_expire_ms", + "riak_kv.query.timeseries.qbuf_incomplete_release_ms"] + end. + +%% Wait for these capabilities to settle at these versions at the end +%% of upgrade/downgrade: +caps_to_ensure(full) -> + case get_riak_release_in_slot(previous) of + {1, 5, _} -> + []; %% no new caps in 1.6 since 1.5 + {1, 4, _} -> + [{{riak_kv, sql_select_version}, v3}, + {{riak_kv, riak_ql_ddl_rec_version}, v2}, + {{riak_kv, decode_query_results_at_vnode}, true}] + end; +caps_to_ensure(degraded) -> + case get_riak_release_in_slot(previous) of + {1, 5, _} -> + []; %% no new caps in 1.6 since 1.5 + {1, 4, _} -> + [{{riak_kv, sql_select_version}, v2}, + {{riak_kv, riak_ql_ddl_rec_version}, v1}, + {{riak_kv, decode_query_results_at_vnode}, false}] + end. + %% Keep the old convert_riak_conf_to_previous functions around, for %% reference and occasional test rerun From d5c6bd6f59bbab23c63821c6d8219c88ae8d527d Mon Sep 17 00:00:00 2001 From: Brett Hazen Date: Thu, 2 Mar 2017 22:11:52 -0700 Subject: [PATCH 22/35] Lock deps for riak_ee-2.3.0noblock1 --- rebar.config.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rebar.config.lock b/rebar.config.lock index 8916829ad..206d0dd1a 100644 --- a/rebar.config.lock +++ b/rebar.config.lock @@ -31,7 +31,7 @@ "6c67dc58251e1a1c29b2e0c444b1e9c9aa2d4a24"}}, {riakc,".*", {git,"https://github.com/basho/riak-erlang-client", - "87958bdf1926d752fb1319b6788330176f6dd591"}}, + "fb96f889da1e74a69920f1f91be6fa3d8bd22705"}}, {ibrowse,".*", {git,"https://github.com/basho/ibrowse.git", "b28542d1e326ba44bcfaf7fd6d3c7f8761d20f08"}}, From 626c4d5ca31bd1695a92d299b01b1f8531ce5ee8 Mon Sep 17 00:00:00 2001 From: "John R. Daily" Date: Fri, 3 Mar 2017 09:18:55 -0500 Subject: [PATCH 23/35] Fix regular expression backslashes --- tests/ts_updown_util.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ts_updown_util.erl b/tests/ts_updown_util.erl index 8e783eca4..813eeda46 100644 --- a/tests/ts_updown_util.erl +++ b/tests/ts_updown_util.erl @@ -588,7 +588,7 @@ get_riak_release_in_slot(VsnSlot) -> unknown -> ct:fail("Failed to determine riak version in '~s' slot", [VsnSlot]); Known -> - case re:run(Known, "riak_ts-(\\d+)\.(\\d+)\.(\\d+)", [{capture, all_but_first, list}]) of + case re:run(Known, "riak_ts-(\\d+)\\.(\\d+)\\.(\\d+)", [{capture, all_but_first, list}]) of {match, [V1, V2, V3]} -> {list_to_integer(V1), list_to_integer(V2), From 0e8656f3208bb8c2e761408236b1203df9304709 Mon Sep 17 00:00:00 2001 From: Fred Dushin Date: Fri, 3 Mar 2017 20:50:10 -0500 Subject: [PATCH 24/35] Fix for missing gset stats --- tests/verify_riak_stats.erl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/verify_riak_stats.erl b/tests/verify_riak_stats.erl index 36831b473..169b53dff 100644 --- a/tests/verify_riak_stats.erl +++ b/tests/verify_riak_stats.erl @@ -415,6 +415,7 @@ common_stats() -> <<"asn1_version">>, <<"basho_stats_version">>, <<"bitcask_version">>, + <<"chronos_version">>, <<"clique_version">>, <<"cluster_info_version">>, <<"compiler_version">>, @@ -468,6 +469,7 @@ common_stats() -> <<"fuse_version">>, <<"goldrush_version">>, <<"gossip_received">>, + <<"gproc_version">>, <<"handoff_timeouts">>, <<"hll_bytes">>, <<"hll_bytes_mean">>, From 7f2a5511456e918ab4e0f78e2ab0d33f5894f203 Mon Sep 17 00:00:00 2001 From: Jenkins Date: Sat, 4 Mar 2017 00:25:20 -0800 Subject: [PATCH 25/35] Jenkins - update locked-deps for riak_ee-2.3.0nightly20170304 --- rebar.config.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rebar.config.lock b/rebar.config.lock index 206d0dd1a..19e6a9ad0 100644 --- a/rebar.config.lock +++ b/rebar.config.lock @@ -25,13 +25,13 @@ "51b79c5b05eb04640e13131f06ef96561b96ba8e"}}, {hamcrest,".*", {git,"https://github.com/basho/hamcrest-erlang.git", - "98bc7aa19ea081478c816824aa05fc5a48acae66"}}, + "ad3dbab419762fc2d5821abb88b989da006b85c6"}}, {riak_pb,".*", {git,"https://github.com/basho/riak_pb", - "6c67dc58251e1a1c29b2e0c444b1e9c9aa2d4a24"}}, + "08771aba2ce4935b715d32d1b92555efdc3da994"}}, {riakc,".*", {git,"https://github.com/basho/riak-erlang-client", - "fb96f889da1e74a69920f1f91be6fa3d8bd22705"}}, + "9167ad627348d4995ef60c93c63ce39a16eb9735"}}, {ibrowse,".*", {git,"https://github.com/basho/ibrowse.git", "b28542d1e326ba44bcfaf7fd6d3c7f8761d20f08"}}, From 4ba9ac79586fbe65b50518f727a2630a3748cc16 Mon Sep 17 00:00:00 2001 From: Fred Dushin Date: Sat, 4 Mar 2017 17:40:31 -0500 Subject: [PATCH 26/35] Added `lts` to basho_builds YAML --- basho_builds.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/basho_builds.yml b/basho_builds.yml index 368009993..247c240b2 100644 --- a/basho_builds.yml +++ b/basho_builds.yml @@ -8,6 +8,9 @@ docker_rt_riak_test_config: - version: legacy product_version: 2.0.8 product: "riak{% if ee == 'true' %}_ee{% endif %}" + - VERSION: lts + product_version: 2.0.8 + product: "riak{% if ee == 'true' %}_ee{% endif %}" - version: 2.0.2 product_version: 2.0.2 product: "riak{% if ee == 'true' %}_ee{% endif %}" From 3ae320c18eed3057cf8b3f7c0456d8b021036a30 Mon Sep 17 00:00:00 2001 From: Russell Brown Date: Tue, 7 Mar 2017 15:04:30 +0000 Subject: [PATCH 27/35] Re-add old kv679 test This test was part of the original kv679 suite, but at the time of 2.1 it was put in it's own branch. Riak Test is made up of passing tests, and this test (kv679_dataloss_fb.erl) still fails. Adding this back to a branch off develop in preparation for a fix of this issue. --- tests/kv679_dataloss_fb.erl | 212 ++++++++++++++++++++++++++++++++++++ tests/kv679_tombstone.erl | 7 +- 2 files changed, 217 insertions(+), 2 deletions(-) create mode 100644 tests/kv679_dataloss_fb.erl diff --git a/tests/kv679_dataloss_fb.erl b/tests/kv679_dataloss_fb.erl new file mode 100644 index 000000000..872f17ce6 --- /dev/null +++ b/tests/kv679_dataloss_fb.erl @@ -0,0 +1,212 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2017 Basho Technologies, Inc. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% ------------------------------------------------------------------- +%%% @copyright (C) 2017, Basho Technologies +%%% @doc +%%% riak_test for kv679 lost clock/fallback/handoff flavour. +%%% +%%% issue kv679 is a possible dataloss issue, it's basically caused by +%%% the fact that per key logical clocks can go backwards in time in +%%% certain situations. The situation under test here is as follows: +%%% +%% A coords a write to K [{a, 1}] and replicates to fallbacks D, E +%% A coords a write to K [{a, 2}] and replicates to primaries B, C +%% A coords a write K [{a, 3}] and replicates to primaries B, C +%% A loses it's clock for K (so far this is like the lost clock case above) +%% Read of A, D, E read repairs A with K=[{a, 1}] +%% A coords a write, issues [{a, 2}] again +%% Acked write is lost +%%% +%%% +%%% @end + +-module(kv679_dataloss_fb). +-behavior(riak_test). +-compile([export_all]). +-export([confirm/0]). + +-include_lib("eunit/include/eunit.hrl"). + +-define(BUCKET, <<"kv679">>). +-define(KEY, <<"test">>). + +confirm() -> + Conf = [ + {riak_kv, [{anti_entropy, {off, []}}]}, + {riak_core, [{default_bucket_props, [{allow_mult, true}, + {dvv_enabled, true}, + {ring_creation_size, 8}, + {vnode_management_timer, 1000}, + {handoff_concurrency, 100}, + {vnode_inactivity_timeout, 1000}]}]}, + {bitcask, [{sync_strategy, o_sync}, {io_mode, nif}]}], + + %% Such nodes 'cos I want a perfect preflist when 2 primaries are + %% down. i.e. I want to kill the fallbacks before they can + %% handoff without effecting the primaries + Nodes = rt:build_cluster(6, Conf), + + Clients = kv679_tombstone:create_pb_clients(Nodes), + + %% Get preflist for key + PL = kv679_tombstone:get_preflist(hd(Nodes)), + + ?assert(kv679_tombstone2:perfect_preflist(PL)), + + lager:info("Got preflist"), + + {CoordNode, _}=CoordClient = kv679_tombstone:coordinating_client(Clients, PL), + + OtherPrimaries = [Node || {{_Idx, Node}, Type} <- PL, + Type == primary, + Node /= CoordNode], + + [rt:stop_and_wait(N) || N <- OtherPrimaries], + + lager:info("Killed 2 primaries"), + + rt:wait_until(fun() -> + NewPL = kv679_tombstone:get_preflist(CoordNode), + primary_and_fallback_counts(NewPL) == {1, 2} + end), + + FBPL = kv679_tombstone:get_preflist(CoordNode), + + lager:info("Got a preflist with coord and 2 fbs ~p~n", [FBPL]), + + %% Write key twice at remaining, coordinating primary + kv679_tombstone:write_key(CoordClient, [<<"bob">>, <<"jim">>]), + + kv679_tombstone2:dump_clock(CoordClient), + + + lager:info("Clock at 2 fallbacks"), + + %% Kill the fallbacks before they can handoff + Fallbacks = [Node || {{_Idx, Node}, Type} <- FBPL, + Type == fallback], + + [rt:brutal_kill(FB) || FB <- Fallbacks], + + %% Bring back the primaries and do some more writes + [rt:start_and_wait(P) || P <- OtherPrimaries], + + lager:info("started primaries back up"), + + rt:wait_until(fun() -> + NewPL = kv679_tombstone:get_preflist(CoordNode), + NewPL == PL + end), + + kv679_tombstone:write_key(CoordClient, [<<"jon">>, <<"joe">>]), + + kv679_tombstone2:dump_clock(CoordClient), + + %% Kill those primaries with their frontier clocks + [rt:brutal_kill(P) || P <- OtherPrimaries], + + lager:info("killed primaries again"), + + %% delete the local data at the coordinator Key + kv679_dataloss:delete_datadir(hd(PL)), + + %% Start up those fallbacks + [rt:start_and_wait(F) || F <- Fallbacks], + + lager:info("restart fallbacks"), + + %% Wait for the fallback prefist + rt:wait_until(fun() -> + NewPL = kv679_tombstone:get_preflist(CoordNode), + NewPL == FBPL + end), + + %% Read the key, read repair will mean that the data deleted vnode + %% will have an old clock (gone back in time!) + + await_read_repair(CoordClient), + + kv679_tombstone2:dump_clock(CoordClient), + + %% write a new value, this _should_ be a sibling of what is on + %% crashed primaries + + lager:info("writing anne value", []), + + %% @TODO why is this write calling put new object?? + + kv679_tombstone:write_key(CoordClient, <<"anne">>), + + lager:info("dumping new vclock"), + + kv679_tombstone2:dump_clock(CoordClient), + + %% Time to start up those primaries, let handoff happen, and see + %% what happened to that last write + + [rt:start_and_wait(P) || P <- OtherPrimaries], + + lager:info("restart primaries _again_"), + + rt:wait_until(fun() -> + NewPL = kv679_tombstone:get_preflist(CoordNode), + NewPL == PL + end), + + lager:info("wait for handoffs"), + + [begin + rpc:call(FB, riak_core_vnode_manager, force_handoffs, []), + rt:wait_until_transfers_complete([FB]) + end || FB <- Fallbacks], + + lager:info("final get"), + + Res = kv679_tombstone:read_key(CoordClient), + + ?assertMatch({ok, _}, Res), + {ok, O} = Res, + + %% A nice riak would have somehow managed to make a sibling of the + %% last write + ?assertEqual([<<"anne">>, <<"joe">>], riakc_obj:get_values(O)), + + lager:info("Final Object ~p~n", [O]), + + pass. + +primary_and_fallback_counts(PL) -> + lists:foldl(fun({{_, _}, primary}, {P, F}) -> + {P+1, F}; + ({{_, _}, fallback}, {P, F}) -> + {P, F+1} + end, + {0, 0}, + PL). + +%% Wait until a read repair has occured, or at least, wait until there +%% is a value on disk at the coordinating/primary vnode (and assume it +%% must have got there via read repair) +await_read_repair(Client) -> + rt:wait_until(fun() -> + {ok, _O} = kv679_tombstone:read_key(Client), + {T, V} = kv679_tombstone:read_key(Client, [{pr,1},{r,1}, {sloppy_quorum, false}]), + lager:info("pr=1 fetch res ~p ~p", [T, V]), + T /= error + end). diff --git a/tests/kv679_tombstone.erl b/tests/kv679_tombstone.erl index 690e4570e..300a0eb4e 100644 --- a/tests/kv679_tombstone.erl +++ b/tests/kv679_tombstone.erl @@ -144,8 +144,11 @@ write_key({_, Client}, Val, Opts) when is_binary(Val) -> end, riakc_pb_socket:put(Client, Object, Opts). -read_key({_, Client}) -> - riakc_pb_socket:get(Client, ?BUCKET, ?KEY, []). +read_key({_, Client}, Opts) -> + riakc_pb_socket:get(Client, ?BUCKET, ?KEY, Opts). + +read_key(C) -> + read_key(C, []). delete_key({_, Client}) -> riakc_pb_socket:delete(Client, ?BUCKET, ?KEY), From e01d78c1165b5c5127f4a9f67d0d605231024756 Mon Sep 17 00:00:00 2001 From: Fred Dushin Date: Tue, 7 Mar 2017 19:57:29 -0500 Subject: [PATCH 28/35] Removed lts stanza I erroneously added before --- basho_builds.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/basho_builds.yml b/basho_builds.yml index 247c240b2..368009993 100644 --- a/basho_builds.yml +++ b/basho_builds.yml @@ -8,9 +8,6 @@ docker_rt_riak_test_config: - version: legacy product_version: 2.0.8 product: "riak{% if ee == 'true' %}_ee{% endif %}" - - VERSION: lts - product_version: 2.0.8 - product: "riak{% if ee == 'true' %}_ee{% endif %}" - version: 2.0.2 product_version: 2.0.2 product: "riak{% if ee == 'true' %}_ee{% endif %}" From 4fa651c808ed8d4947d603ca33635964289ea20b Mon Sep 17 00:00:00 2001 From: Doug Rohrer Date: Tue, 14 Mar 2017 16:54:08 -0400 Subject: [PATCH 29/35] Move to 2.0.9 for Legacy --- basho_builds.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/basho_builds.yml b/basho_builds.yml index 368009993..f6e528f6b 100644 --- a/basho_builds.yml +++ b/basho_builds.yml @@ -6,7 +6,7 @@ docker_rt_riak_test_config: product_version: 2.2.0 product: "riak{% if ee == 'true' %}_ee{% endif %}" - version: legacy - product_version: 2.0.8 + product_version: 2.0.9 product: "riak{% if ee == 'true' %}_ee{% endif %}" - version: 2.0.2 product_version: 2.0.2 From 3c7a2dff3800acc5e55b6569624942ac7ea6fcaf Mon Sep 17 00:00:00 2001 From: Brian Sparrow Date: Wed, 15 Mar 2017 10:12:39 -0400 Subject: [PATCH 30/35] Set force_hashtree_upgrade to true This removes the race condition where some nodes are upgrading and others arent in the middle of an active fullsync. We are not testing AAE upgrades with this test so we should disable them. --- tests/repl_aae_fullsync.erl | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/repl_aae_fullsync.erl b/tests/repl_aae_fullsync.erl index a14651f86..cdcb044f7 100644 --- a/tests/repl_aae_fullsync.erl +++ b/tests/repl_aae_fullsync.erl @@ -29,6 +29,7 @@ [ %% Specify fast building of AAE trees {sweep_tick, 10000}, + {force_hashtree_upgrade, true}, {anti_entropy, {on, [debug]}}, {anti_entropy_build_limit, {100, 1000}}, {anti_entropy_concurrency, 100} From d79be99277aaa9cd76fb2909f7846f4d982b1b96 Mon Sep 17 00:00:00 2001 From: Brian Sparrow Date: Wed, 15 Mar 2017 13:49:14 -0400 Subject: [PATCH 31/35] Remove unneeded code from riaknostic test --- tests/riaknostic_rt.erl | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/tests/riaknostic_rt.erl b/tests/riaknostic_rt.erl index 85ed822a5..4519e9ebc 100644 --- a/tests/riaknostic_rt.erl +++ b/tests/riaknostic_rt.erl @@ -22,19 +22,11 @@ -export([confirm/0]). -include_lib("eunit/include/eunit.hrl"). -%% Change when a new release comes out. --define(RIAKNOSTIC_URL, "https://github.com/basho/riaknostic/downloads/riaknostic-1.0.2.tar.gz"). - -%% REQUIRES (sh, curl, tar) - confirm() -> %% Build a small cluster [Node1, _Node2] = rt:build_cluster(2, []), ?assertEqual(ok, rt:wait_until_nodes_ready([Node1])), - %% Install riaknostic for Riak versions below 1.3.0 - riaknostic_bootstrap(Node1), - %% Run through all tests on Node1 check_riaknostic_execute(Node1), check_riaknostic_usage(Node1), @@ -45,26 +37,6 @@ confirm() -> lager:info("Test riaknostic: PASS"), pass. -riaknostic_bootstrap(Node) -> - lager:info("Check if riaknostic is installed"), - {ok, RiaknosticOut1} = rt:admin(Node, ["diag"]), - riaknostic_install((rt:str(RiaknosticOut1, "is not present!")), Node). - -%% riaknostic is already installed, move along -riaknostic_install(false, _Node) -> - ok; - -%% install riaknostic -riaknostic_install(true, Node) -> - %% Install - lager:info("Installing Riaknostic"), - {ok, LibDir} = rpc:call(Node, application, get_env, [riak_core, platform_lib_dir]), - Cmd = io_lib:format("sh -c \"cd ~s && curl -O -L ~s && tar xzf ~s\"", - [LibDir, ?RIAKNOSTIC_URL, filename:basename(?RIAKNOSTIC_URL)]), - lager:info("Running command: ~s", [Cmd]), - lager:debug("~p~n", [rpc:call(Node, os, cmd, [Cmd])]), - ok. - %% Check that riaknostic executes check_riaknostic_execute(Node) -> %% Execute From c252dd37b47a21d0c15c57748f5074744a4e93a6 Mon Sep 17 00:00:00 2001 From: Fred Dushin Date: Wed, 15 Mar 2017 14:09:42 -0400 Subject: [PATCH 32/35] Increased concurrency and lowered sweep interval, while also widening expiry interval, in an attempt to get AAE to converge more regularly. --- tests/verify_aae.erl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/verify_aae.erl b/tests/verify_aae.erl index 8a2d61cb9..06e7ec9d2 100644 --- a/tests/verify_aae.erl +++ b/tests/verify_aae.erl @@ -47,7 +47,8 @@ -define(CFG, [{riak_kv, [ - {sweep_tick, 3000}, + {sweep_tick, 1023}, + {sweep_concurrency, 8}, % Speedy AAE configuration {anti_entropy, {on, []}}, {anti_entropy_build_limit, {100, 1000}}, @@ -136,7 +137,7 @@ verify_aae(Nodes) -> start_tree_rebuilds(Nodes) -> rpc:multicall(Nodes, application, set_env, [riak_kv, anti_entropy_expire, - 15 * 1000]). + 30 * 1000]). acc_preflists(Pl, PlCounts) -> lists:foldl(fun(Idx, D) -> From 9c2dfa744c01b7da838dbd3e7ba908f6543fc8a1 Mon Sep 17 00:00:00 2001 From: Fred Dushin Date: Fri, 17 Mar 2017 16:04:07 -0400 Subject: [PATCH 33/35] Removed gproc version from stats, because we no longer rely on gproc --- tests/verify_riak_stats.erl | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/verify_riak_stats.erl b/tests/verify_riak_stats.erl index 169b53dff..7ef601ea1 100644 --- a/tests/verify_riak_stats.erl +++ b/tests/verify_riak_stats.erl @@ -469,7 +469,6 @@ common_stats() -> <<"fuse_version">>, <<"goldrush_version">>, <<"gossip_received">>, - <<"gproc_version">>, <<"handoff_timeouts">>, <<"hll_bytes">>, <<"hll_bytes_mean">>, From 01764947236e46363afb14bb461f0646b6275edf Mon Sep 17 00:00:00 2001 From: Russell Brown Date: Mon, 20 Mar 2017 18:03:01 +0000 Subject: [PATCH 34/35] Add test for further dataloss edge case This test shows a dataloss edge case in riak, even in 2.1 with per-key-actor-epochs enabled. The test is a litte convoluted, and is based on a quickcheck counter example, included in the riak_kv/test directory. In short, both this test, and the other kv679_dataloss_fb test, show that even with multiple replicas acking/storing, a single disk error on a single replica is enough to cause acked writes to be silently and permanently lost. For a replicated database, that is bad. --- .../riak_kv_bitcask_backend_intercepts.erl | 8 + src/rt.erl | 25 ++ tests/kv679_dataloss_fb.erl | 27 -- tests/kv679_dataloss_fb2.erl | 249 ++++++++++++++++++ tests/kv679_tombstone.erl | 8 +- tests/kv679_tombstone2.erl | 7 +- 6 files changed, 293 insertions(+), 31 deletions(-) create mode 100644 tests/kv679_dataloss_fb2.erl diff --git a/intercepts/riak_kv_bitcask_backend_intercepts.erl b/intercepts/riak_kv_bitcask_backend_intercepts.erl index dc42cbeb3..a5cc31b96 100644 --- a/intercepts/riak_kv_bitcask_backend_intercepts.erl +++ b/intercepts/riak_kv_bitcask_backend_intercepts.erl @@ -48,3 +48,11 @@ corrupting_get(Bucket, Key, ModState) -> corrupt_binary(O) -> crypto:rand_bytes(byte_size(O)). + +always_corrupt_get(Bucket, Key, ModState) -> + case ?M:get_orig(Bucket, Key, ModState) of + {ok, BinVal0, UpdModState} -> + BinVal = corrupt_binary(BinVal0), + {ok, BinVal, UpdModState}; + Else -> Else + end. diff --git a/src/rt.erl b/src/rt.erl index 507fa50f6..1cc104f24 100644 --- a/src/rt.erl +++ b/src/rt.erl @@ -561,6 +561,20 @@ heal({_NewCookie, OldCookie, P1, P2}) -> {_GN, []} = rpc:sbcast(Cluster, riak_core_node_watcher, broadcast), ok. +%% @doc heal the partition created by call to partition/2, but if some +%% node in P1 is down, just skip it, rather than failing. Returns {ok, +%% list(node())} where the list is those nodes down and therefore not +%% healed/reconnected. +heal_upnodes({_NewCookie, OldCookie, P1, P2}) -> + %% set OldCookie on UP P1 Nodes + Res = [{N, rpc:call(N, erlang, set_cookie, [N, OldCookie])} || N <- P1], + UpForReconnect = [N || {N, true} <- Res], + DownForReconnect = [N || {N, RPC} <- Res, RPC /= true], + Cluster = UpForReconnect ++ P2, + wait_until_connected(Cluster), + {_GN, []} = rpc:sbcast(Cluster, riak_core_node_watcher, broadcast), + {ok, DownForReconnect}. + %% @doc Spawn `Cmd' on the machine running the test harness spawn_cmd(Cmd) -> ?HARNESS:spawn_cmd(Cmd). @@ -778,6 +792,17 @@ wait_until_transfers_complete([Node0|_]) -> ?assertEqual(ok, wait_until(Node0, F)), ok. +%% @doc Waits until hinted handoffs from `Node0' are complete +wait_until_node_handoffs_complete(Node0) -> + lager:info("Wait until Node's transfers complete ~p", [Node0]), + F = fun(Node) -> + Handoffs = rpc:call(Node, riak_core_handoff_manager, status, [{direction, outbound}]), + lager:info("Handoffs: ~p", [Handoffs]), + Handoffs =:= [] + end, + ?assertEqual(ok, wait_until(Node0, F)), + ok. + wait_for_service(Node, Services) when is_list(Services) -> F = fun(N) -> case rpc:call(N, riak_core_node_watcher, services, [N]) of diff --git a/tests/kv679_dataloss_fb.erl b/tests/kv679_dataloss_fb.erl index 872f17ce6..c709e85f6 100644 --- a/tests/kv679_dataloss_fb.erl +++ b/tests/kv679_dataloss_fb.erl @@ -92,10 +92,7 @@ confirm() -> %% Write key twice at remaining, coordinating primary kv679_tombstone:write_key(CoordClient, [<<"bob">>, <<"jim">>]), - kv679_tombstone2:dump_clock(CoordClient), - - lager:info("Clock at 2 fallbacks"), %% Kill the fallbacks before they can handoff @@ -106,21 +103,16 @@ confirm() -> %% Bring back the primaries and do some more writes [rt:start_and_wait(P) || P <- OtherPrimaries], - lager:info("started primaries back up"), - rt:wait_until(fun() -> NewPL = kv679_tombstone:get_preflist(CoordNode), NewPL == PL end), - kv679_tombstone:write_key(CoordClient, [<<"jon">>, <<"joe">>]), - kv679_tombstone2:dump_clock(CoordClient), %% Kill those primaries with their frontier clocks [rt:brutal_kill(P) || P <- OtherPrimaries], - lager:info("killed primaries again"), %% delete the local data at the coordinator Key @@ -128,9 +120,7 @@ confirm() -> %% Start up those fallbacks [rt:start_and_wait(F) || F <- Fallbacks], - lager:info("restart fallbacks"), - %% Wait for the fallback prefist rt:wait_until(fun() -> NewPL = kv679_tombstone:get_preflist(CoordNode), @@ -139,38 +129,24 @@ confirm() -> %% Read the key, read repair will mean that the data deleted vnode %% will have an old clock (gone back in time!) - await_read_repair(CoordClient), - kv679_tombstone2:dump_clock(CoordClient), %% write a new value, this _should_ be a sibling of what is on %% crashed primaries - - lager:info("writing anne value", []), - - %% @TODO why is this write calling put new object?? - kv679_tombstone:write_key(CoordClient, <<"anne">>), - - lager:info("dumping new vclock"), - kv679_tombstone2:dump_clock(CoordClient), %% Time to start up those primaries, let handoff happen, and see %% what happened to that last write - [rt:start_and_wait(P) || P <- OtherPrimaries], - lager:info("restart primaries _again_"), - rt:wait_until(fun() -> NewPL = kv679_tombstone:get_preflist(CoordNode), NewPL == PL end), lager:info("wait for handoffs"), - [begin rpc:call(FB, riak_core_vnode_manager, force_handoffs, []), rt:wait_until_transfers_complete([FB]) @@ -179,16 +155,13 @@ confirm() -> lager:info("final get"), Res = kv679_tombstone:read_key(CoordClient), - ?assertMatch({ok, _}, Res), {ok, O} = Res, %% A nice riak would have somehow managed to make a sibling of the %% last write ?assertEqual([<<"anne">>, <<"joe">>], riakc_obj:get_values(O)), - lager:info("Final Object ~p~n", [O]), - pass. primary_and_fallback_counts(PL) -> diff --git a/tests/kv679_dataloss_fb2.erl b/tests/kv679_dataloss_fb2.erl new file mode 100644 index 000000000..245fdf7db --- /dev/null +++ b/tests/kv679_dataloss_fb2.erl @@ -0,0 +1,249 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2017 Basho Technologies, Inc. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% ------------------------------------------------------------------- +%%% @copyright (C) 2017, Basho Technologies +%%% @doc +%%% riak_test for kv679 lost clock/fallback/handoff flavour. +%%% +%%% issue kv679 is a possible dataloss issue, it's basically caused by +%%% the fact that per key logical clocks can go backwards in time in +%%% certain situations. The situation under test here is as follows: +%%% +%% write at P1 replicates to F1 clock is at [{p1, 1}] at p1, f1 +%% write twice more at P1, replciates to F2, clock is at [{p1, 3}] at p1, f2 +%% write at p2 replicates to f3, clock is at [{p2, 1}] at p2, f3 +%% handoff from f3->p1, during which the local read at p1 fails (disk error, whatever) +%% clock at p1,p2 is [{p2, 1}] f3 deletes after handoff (no more f3) +%% handoff from f1->p2 clock at p2 [{p1, 1}, {p2, 1}] +%% read repair from p2 -> p1, clock at p1 [{p1, 1}, {p2, 1}] +%% write on p1 replicate to p2, clock at [{p1, 2}, {p2, 1}] at p1, p2 +%% handoff f2->p2, merge causes last write to be lost since entry {p1, 3} > {p1, 2} +%% read repair p2->p1 and the acked write is silently lost forever. +%%% +%%% +%% NOTE this failure occurs even in riak2.1 with actor-epochs, since +%% it is caused by a local-not-found on a non-coordinating write. EQC +%% found this issue, the counter example is below. +%%% +%%% +%% [{set,{var,1},{call,kv679_eqc,put,[p1,p2,<<"A">>,959494,undefined,false]}}, +%% {set,{var,3},{call,kv679_eqc,put,[p3,p2,<<"A">>,960608,{var,2},false]}}, +%% {set,{var,5},{call,kv679_eqc,put,[p1,f3,<<"A">>,960760,{var,4},false]}}, +%% {set,{var,6},{call,kv679_eqc,put,[p1,f1,<<"A">>,960851,{var,4},false]}}, +%% {set,{var,7},{call,kv679_eqc,forget,[p1,<<"A">>]}}, +%% {set,{var,8},{call,kv679_eqc,replicate,[<<"A">>,p3,p1]}}, +%% {set,{var,9},{call,kv679_eqc,replicate,[<<"A">>,f3,p1]}}, +%% {set,{var,11},{call,kv679_eqc,put,[p1,p2,<<"A">>,962942,{var,10},false]}}] +%% +%%% +%%% +%%% @end + +-module(kv679_dataloss_fb2). +-behavior(riak_test). +-compile([export_all]). +-export([confirm/0]). +-import(kv679_dataloss_fb, [primary_and_fallback_counts/1]). + +-include_lib("eunit/include/eunit.hrl"). + +-define(BUCKET, <<"kv679">>). +-define(KEY, <<"test">>). +-define(NVAL, 2). + +confirm() -> + Conf = [ + {riak_kv, [{anti_entropy, {off, []}}]}, + {riak_core, [{default_bucket_props, [ + %% note reduced n_val + %% is to reduce + %% number of nodes + %% required for test + %% case and to + %% simplify case. + {n_val, ?NVAL}, + {allow_mult, true}, + {dvv_enabled, true}, + {ring_creation_size, 8}, + {vnode_management_timer, 1000}, + {handoff_concurrency, 100}, + {vnode_inactivity_timeout, 1000}]}]}, + {bitcask, [{sync_strategy, o_sync}, {io_mode, nif}]}], + + Nodes = rt:build_cluster(5, Conf), + Clients = kv679_tombstone:create_pb_clients(Nodes), + {_, TempPBC} = hd(Clients), + rt:pbc_set_bucket_prop(TempPBC, ?BUCKET, [{n_val, ?NVAL}]), + rt:wait_until_bucket_props(Nodes, ?BUCKET, [{n_val, ?NVAL}]), + + %% Get preflist for key + PL = kv679_tombstone:get_preflist(hd(Nodes), ?NVAL), + ?assert(kv679_tombstone2:perfect_preflist(PL, ?NVAL)), + + lager:info("Got preflist ~p", [PL]), + + {CoordNode, _}=CoordClient = kv679_tombstone:coordinating_client(Clients, PL), + OtherPrimaries = [Node || {{_Idx, Node}, Type} <- PL, + Type == primary, + Node /= CoordNode], + Fallbacks = [FB || FB <- Nodes, FB /= CoordNode andalso not lists:member(FB, OtherPrimaries)], + + ?assert(length(Fallbacks) == 3), + + lager:info("fallbacks ~p, primaries ~p~n", [Fallbacks, [CoordNode] ++ OtherPrimaries]), + + %% Partition the other primary and one non-preflist node + {_, _, _P1, P2} = PartInfo = rt:partition([CoordNode] ++ tl(Fallbacks), OtherPrimaries ++ [hd(Fallbacks)]), + lager:info("Partitioned ~p~n", [PartInfo]), + + rt:wait_until(fun() -> + NewPL = kv679_tombstone:get_preflist(CoordNode, ?NVAL), + lager:info("new PL ~p~n", [NewPL]), + primary_and_fallback_counts(NewPL) == {1, 1} + end), + + FBPL = kv679_tombstone:get_preflist(CoordNode, ?NVAL), + lager:info("Got a preflist with coord and 1 fb ~p~n", [FBPL]), + + %% Write key once at coordinating primary + kv679_tombstone:write_key(CoordClient, [<<"alice">>]), + kv679_tombstone2:dump_clock(CoordClient), + lager:info("Clock at fallback"), + + %% Kill the fallback before it can handoff + [FB1] = [Node || {{_Idx, Node}, Type} <- FBPL, + Type == fallback], + rt:brutal_kill(FB1), + + %% get a new preflist with a different fallback + rt:wait_until(fun() -> + NewPL = kv679_tombstone:get_preflist(CoordNode, ?NVAL), + primary_and_fallback_counts(NewPL) == {1, 1} andalso NewPL /= FBPL + end), + FBPL2 = kv679_tombstone:get_preflist(CoordNode, ?NVAL), + lager:info("Got a preflist with coord and 1 fb ~p~n", [FBPL2]), + ?assert(FBPL2 /= FBPL), + + %% do two more writes so that there exists out there a clock of + %% {Primary1, 1} on the first fallback node and {Primary1, 3} on + %% the second + kv679_tombstone:write_key(CoordClient, [<<"bob">>, <<"charlie">>]), + kv679_tombstone2:dump_clock(CoordClient), + lager:info("Clock at fallback2"), + + %% Kill the fallback before it can handoff + [FB2] = [Node || {{_Idx, Node}, Type} <- FBPL2, + Type == fallback], + rt:brutal_kill(FB2), + + %% meanwhile, in the other partition, let's write some data + P2PL = kv679_tombstone:get_preflist(hd(P2), ?NVAL), + lager:info("partition 2 PL ~p", [P2PL]), + [P2FB] = [Node || {{_Idx, Node}, Type} <- P2PL, + Type == fallback], + [P2P] = [Node || {{_Idx, Node}, Type} <- P2PL, + Type == primary], + P2Client = kv679_tombstone:coordinating_client(Clients, P2PL), + kv679_tombstone:write_key(P2Client, [<<"dave">>]), + kv679_tombstone2:dump_clock(P2Client), + lager:info("Clock in Partition 2"), + + %% set up a local read error on Primary 1 (this is any disk error, + %% delete the datadir, an intercept for a local crc failure or + %% deserialisation error) + rt_intercept:add(CoordNode, {riak_kv_bitcask_backend, [{{get, 3}, always_corrupt_get}]}), + + %% heal the partition, remember the Partition 1 fallbacks aren't + %% handing off yet (we killed them, but it could be any delay in + %% handoff, not a failure, killing them is just a way to control + %% timing (maybe should've used intercepts??)) + {ok, DownNodes} = rt:heal_upnodes(PartInfo), + lager:info("These nodes still need healing ~p", [DownNodes]), + + %% wait for the partition 2 fallback to hand off + rpc:call(P2FB, riak_core_vnode_manager, force_handoffs, []), + rt:wait_until_node_handoffs_complete(P2FB), + + %% what's the clock on primary 1 now? NOTE: this extra read was + %% the only way I could ensure that the handoff had occured before + %% cleaning out the intercept. A sleep also worked. + kv679_tombstone2:dump_clock(CoordClient), + rt_intercept:clean(CoordNode, riak_kv_bitcask_backend), + kv679_tombstone2:dump_clock(CoordClient), + + %% restart fallback one and wait for it to handoff + rt:start_and_wait(FB1), + lager:info("started fallback 1 back up"), + rpc:call(FB1, riak_core_vnode_manager, force_handoffs, []), + rt:wait_until_node_handoffs_complete(FB1), + + %% kill dev5 again (there is something very weird here, unless I + %% do this dev3 _never_ hands off (and since n=2 it is never used + %% in a preflist for read repair)) So the only way to get it's + %% data onto CoordNode is to ensure a preflist of FB1 and + %% CoordNode for a read! To be fair this rather ruins the test + %% case. @TODO investigate this handoff problem + rt:stop_and_wait(P2P), + rt:stop_and_wait(P2FB), + rt:wait_until(fun() -> + NewPL = kv679_tombstone:get_preflist(CoordNode, ?NVAL), + lager:info("new PL ~p~n", [NewPL]), + primary_and_fallback_counts(NewPL) == {1, 1} andalso + different_nodes(NewPL) + end), + + %% %% what's the clock on primary 1 now? + kv679_tombstone2:dump_clock(CoordClient), + + %% restart P2P and P2FB + rt:start_and_wait(P2P), + rt:start_and_wait(P2FB), + + %% get a primary PL + rt:wait_until(fun() -> + NewPL = kv679_tombstone:get_preflist(CoordNode, ?NVAL), + lager:info("new PL ~p~n", [NewPL]), + primary_and_fallback_counts(NewPL) == {2, 0} andalso + different_nodes(NewPL) + end), + + %% write a new value + kv679_tombstone:write_key(CoordClient, [<<"emma">>]), + kv679_tombstone2:dump_clock(CoordClient), + + %% finally start the last offline fallback, await all transfers, + %% and do a read, the last write, `emma' should be present + rt:start_and_wait(FB2), + rpc:call(FB2, riak_core_vnode_manager, force_handoffs, []), + rt:wait_until_transfers_complete(Nodes), + + lager:info("final get"), + Res = kv679_tombstone:read_key(CoordClient), + ?assertMatch({ok, _}, Res), + {ok, O} = Res, + + %% A nice riak would have somehow managed to make a sibling of the + %% last acked write, even with all the craziness + ?assertEqual([<<"charlie">>, <<"emma">>], lists:sort(riakc_obj:get_values(O))), + lager:info("Final Object ~p~n", [O]), + pass. + +different_nodes(PL) -> + Nodes = [Node || {{_, Node}, _} <- PL], + length(lists:usort(Nodes)) == length(PL). diff --git a/tests/kv679_tombstone.erl b/tests/kv679_tombstone.erl index 300a0eb4e..e96956b51 100644 --- a/tests/kv679_tombstone.erl +++ b/tests/kv679_tombstone.erl @@ -137,7 +137,8 @@ write_key({_, Client}, Val, Opts) when is_binary(Val) -> Object = case riakc_pb_socket:get(Client, ?BUCKET, ?KEY, []) of {ok, O1} -> lager:info("writing existing!"), - riakc_obj:update_value(O1, Val); + O2 = riakc_obj:update_metadata(O1, dict:new()), + riakc_obj:update_value(O2, Val); _ -> lager:info("writing new!"), riakc_obj:new(?BUCKET, ?KEY, Val) @@ -182,9 +183,12 @@ start_node(Node, Preflist) -> wait_for_new_pl(Preflist, Node). get_preflist(Node) -> + get_preflist(Node, 3). + +get_preflist(Node, NVal) -> Chash = rpc:call(Node, riak_core_util, chash_key, [{?BUCKET, ?KEY}]), UpNodes = rpc:call(Node, riak_core_node_watcher, nodes, [riak_kv]), - PL = rpc:call(Node, riak_core_apl, get_apl_ann, [Chash, 3, UpNodes]), + PL = rpc:call(Node, riak_core_apl, get_apl_ann, [Chash, NVal, UpNodes]), PL. kill_primary(Preflist) -> diff --git a/tests/kv679_tombstone2.erl b/tests/kv679_tombstone2.erl index 3135717e1..22d43205d 100644 --- a/tests/kv679_tombstone2.erl +++ b/tests/kv679_tombstone2.erl @@ -128,9 +128,12 @@ confirm() -> perfect_preflist(PL) -> - %% N=3 primaries, each on a unique node + perfect_preflist(PL, 3). + +perfect_preflist(PL, NVal) -> + %% N=NVal primaries, each on a unique node length(lists:usort([Node || {{_Idx, Node}, Type} <- PL, - Type == primary])) == 3. + Type == primary])) == NVal. get_coord_client_and_patsy(Clients, PL) -> {CoordNode, _}=CoordClient=kv679_tombstone:coordinating_client(Clients, PL), From 1e7e94aacb4e8c748963f4970b3b3cf82c8a54e3 Mon Sep 17 00:00:00 2001 From: Russell Brown Date: Mon, 3 Apr 2017 16:48:46 +0100 Subject: [PATCH 35/35] Add a more sophisticated equality check to verify The partition repair test deletes all the data at a partition, and then repairs it from neighbouring partitions. The subset of repaired data that was originally coordinated by the deleted partition's vnode showed up as `notfound` since the latest kv679 changes here https://github.com/basho/riak_kv/pull/1643/. The reason is that the fix in the KV repo adds a new actor to the repaired key's vclock. Prior to this change `verify` in partition_repair.erl did a simple equality check on the binary encoded riak_object values. This change takes into account that a new actor may be in the vclock at the repaired vnode, and uses a semantic equality check based on riak_object merge and riak object equal. --- tests/partition_repair.erl | 34 +++++++++++++++++++++++++++------- 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/tests/partition_repair.erl b/tests/partition_repair.erl index b5c54da61..4c9849438 100644 --- a/tests/partition_repair.erl +++ b/tests/partition_repair.erl @@ -169,7 +169,7 @@ kill_repair_verify({Partition, Node}, DataSuffix, Service) -> lager:info("Verify ~p on ~p is fully repaired", [Partition, Node]), Data2 = get_data(Service, {Partition, Node}), - {Verified, NotFound} = dict:fold(verify(Service, Data2), {0, []}, Stash), + {Verified, NotFound} = dict:fold(verify(Node, Service, Data2), {0, []}, Stash), case NotFound of [] -> ok; _ -> @@ -188,7 +188,7 @@ kill_repair_verify({Partition, Node}, DataSuffix, Service) -> {ok, [StashB]} = file:consult(StashPathB), ExpectToVerifyB = dict:size(StashB), BeforeData = get_data(Service, B), - {VerifiedB, NotFoundB} = dict:fold(verify(Service, BeforeData), {0, []}, StashB), + {VerifiedB, NotFoundB} = dict:fold(verify(Node, Service, BeforeData), {0, []}, StashB), case NotFoundB of [] -> ok; _ -> @@ -203,7 +203,7 @@ kill_repair_verify({Partition, Node}, DataSuffix, Service) -> {ok, [StashA]} = file:consult(StashPathA), ExpectToVerifyA = dict:size(StashA), AfterData = get_data(Service, A), - {VerifiedA, NotFoundA} = dict:fold(verify(Service, AfterData), {0, []}, StashA), + {VerifiedA, NotFoundA} = dict:fold(verify(Node, Service, AfterData), {0, []}, StashA), case NotFoundA of [] -> ok; _ -> @@ -214,19 +214,23 @@ kill_repair_verify({Partition, Node}, DataSuffix, Service) -> ?assertEqual(ExpectToVerifyA, VerifiedA). -verify(riak_kv, DataAfterRepair) -> +verify(Node, riak_kv, DataAfterRepair) -> fun(BKey, StashedValue, {Verified, NotFound}) -> StashedData={BKey, StashedValue}, case dict:find(BKey, DataAfterRepair) of error -> {Verified, [StashedData|NotFound]}; {ok, Value} -> - if Value == StashedValue -> {Verified+1, NotFound}; - true -> {Verified, [StashedData|NotFound]} + %% NOTE: since kv679 fixes, the binary values may + %% not be equal where a new epoch-actor-entry has + %% been added to the repaired value in the vnode + case gte(Node, Value, StashedValue, BKey) of + true -> {Verified+1, NotFound}; + false -> {Verified, [StashedData|NotFound]} end end end; -verify(riak_search, PostingsAfterRepair) -> +verify(_Node, riak_search, PostingsAfterRepair) -> fun(IFT, StashedPostings, {Verified, NotFound}) -> StashedPosting={IFT, StashedPostings}, case dict:find(IFT, PostingsAfterRepair) of @@ -241,6 +245,22 @@ verify(riak_search, PostingsAfterRepair) -> end end. +%% @private gte checks that `Value' is _at least_ `StashedValue'. With +%% the changes for kv679 when a vnode receives a write of a key that +%% contains the vnode's id as an entry in the version vector, it adds +%% a new actor-epoch-entry to the version vector to guard against data +%% loss from repeated events (remember a VV history is supposed to be +%% unique!) This function then must deserialise the stashed and vnode +%% data and check that they are equal, or if not, the only difference +%% is an extra epoch actor in the vnode value's vclock. +gte(Node, Value, StashedData, {B, K}) -> + VnodeObject = riak_object:from_binary(B, K, Value), + StashedObject = riak_object:from_binary(B, K, StashedData), + %% NOTE: we need a ring and all that jazz for bucket props, needed + %% by merge, so use an RPC to merge on a riak node. + Merged = rpc:call(Node, riak_object, syntactic_merge, [VnodeObject, StashedObject]), + riak_object:equal(VnodeObject, Merged). + is_true(X) -> X == true.