From c213253345d8722531378f4bfd99624cdc45f9d7 Mon Sep 17 00:00:00 2001 From: Zeyu Zhang Date: Sun, 9 Apr 2023 20:56:08 -0700 Subject: [PATCH 1/4] Allow dist input_handler to be added if the dist control is an internal port --- erts/doc/src/erlang.xml | 26 +++++++++------ erts/emulator/beam/dist.c | 4 +-- lib/ssl/src/inet_tls_dist.erl | 12 +++++-- lib/ssl/test/ssl_dist_SUITE.erl | 56 +++++++++++++++++++++++++++++++++ 4 files changed, 84 insertions(+), 14 deletions(-) diff --git a/erts/doc/src/erlang.xml b/erts/doc/src/erlang.xml index d7fa7b8149fb..96839c2f809b 100644 --- a/erts/doc/src/erlang.xml +++ b/erts/doc/src/erlang.xml @@ -1927,16 +1927,23 @@ end channel.

- Only the process registered as distribution - controller for the distribution channel identified by - DHandle is allowed to call this - function. + When the distribution controller for the distribution + channel identified by DHandle is a + process, it is the only process allowed to call this + function. + This function is also allowed to be called when the + distribution controller for the distribution channel + identified by DHandle is a port, + for the port owner to be able to call + erlang:dist_ctrl_put_data/2 + when the port is using + {deliver, term} + option.

This function is used when implementing an alternative - distribution carrier using processes as distribution - controllers. DHandle is retrieved - via the callback + distribution carrier. DHandle is + retrieved via the callback f_handshake_complete. More information can be found in the documentation of ERTS @@ -1967,9 +1974,8 @@ end

This function is used when implementing an alternative - distribution carrier using processes as distribution - controllers. DHandle is retrieved - via the callback + distribution carrier. DHandle is + retrieved via the callback f_handshake_complete. More information can be found in the documentation of ERTS diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c index ca6aa40185e9..9cee9f0678ae 100644 --- a/erts/emulator/beam/dist.c +++ b/erts/emulator/beam/dist.c @@ -4386,13 +4386,13 @@ dist_get_stat_1(BIF_ALIST_1) BIF_RETTYPE dist_ctrl_input_handler_2(BIF_ALIST_2) { - DistEntry *dep = ERTS_PROC_GET_DIST_ENTRY(BIF_P); Uint32 conn_id; + DistEntry *dep = erts_dhandle_to_dist_entry(BIF_ARG_1, &conn_id); if (!dep) BIF_ERROR(BIF_P, EXC_NOTSUP); - if (erts_dhandle_to_dist_entry(BIF_ARG_1, &conn_id) != dep) + if ((ERTS_PROC_GET_DIST_ENTRY(BIF_P) != dep) && !is_internal_port(dep->cid)) BIF_ERROR(BIF_P, BADARG); if (is_not_internal_pid(BIF_ARG_2)) diff --git a/lib/ssl/src/inet_tls_dist.erl b/lib/ssl/src/inet_tls_dist.erl index c93bb2759634..95fc3a5a6b8f 100644 --- a/lib/ssl/src/inet_tls_dist.erl +++ b/lib/ssl/src/inet_tls_dist.erl @@ -528,7 +528,7 @@ do_accept( timer = Timer, this_flags = 0, allowed = NewAllowed}, - dist_util:handshake_other_started(trace(HSData)); + dist_util:handshake_other_started(trace(hs_data_finalize(HSData))); {AcceptPid, exit} -> %% this can happen when connection was initiated, but dropped %% between TLS handshake completion and dist handshake start @@ -677,7 +677,7 @@ do_setup( this_flags = 0, other_version = Version, request_type = Type}, - dist_util:handshake_we_started(trace(HSData)) + dist_util:handshake_we_started(trace(hs_data_finalize(HSData))) else Other -> %% Other Node may have closed since @@ -1088,6 +1088,14 @@ ktls_opt_cipher( {error, {ktls_notsup, {cipher, TLS_version, CipherSpec, _CipherState}}}. +hs_data_finalize(HSData) -> + case application:get_env(kernel, dist_hs_data_finalize_fun) of + {ok, Fun} when is_function(Fun) -> + Fun(HSData); + _ -> + HSData + end. + %% ------------------------------------------------------------------------- diff --git a/lib/ssl/test/ssl_dist_SUITE.erl b/lib/ssl/test/ssl_dist_SUITE.erl index 0194ff4dce00..22e7a179fa71 100644 --- a/lib/ssl/test/ssl_dist_SUITE.erl +++ b/lib/ssl/test/ssl_dist_SUITE.erl @@ -23,6 +23,7 @@ -behaviour(ct_suite). +-include_lib("kernel/include/dist_util.hrl"). -include_lib("kernel/include/net_address.hrl"). -include_lib("common_test/include/ct.hrl"). -include_lib("public_key/include/public_key.hrl"). @@ -359,6 +360,7 @@ ktls_verify(Config) -> basic_test(NH1, NH2, KTLSConfig), 0 = ktls_count_tls_dist(NH1), 0 = ktls_count_tls_dist(NH2), + input_handler_test(NH1, NH2, KTLSConfig), ok end, KTLSConfig). @@ -381,6 +383,60 @@ ktls_count_tls_dist(Node) -> 0 end. +%% Verify that dist input handler can be set if dist_ctrl is port; +%% and can send pessage properly +input_handler_test(NH1, NH2, KTLSConfig) -> + #node_handle{nodename = Node2} = NH2, + apply_on_ssl_node(NH1, application, set_env, [ + kernel, dist_hs_data_finalize_fun, fun ktls_hs_data_finalize/1 + ]), + true = apply_on_ssl_node(NH1, erlang, disconnect_node, [Node2]), + true = apply_on_ssl_node(NH1, net_kernel, connect_node, [Node2]), + basic_test(NH1, NH2, KTLSConfig), + true = apply_on_ssl_node(NH1, persistent_term, get, [ktls_input_handler_received]). + +ktls_hs_data_finalize(HSData = #hs_data{socket = Socket}) -> + Receiver = spawn_link(fun ktls_input_handler/0), + ok = gen_tcp:controlling_process(Socket, Receiver), + HSData#hs_data{ + f_setopts_post_nodeup = fun (_) -> ok end, + f_handshake_complete = fun (S, _Node, DHandle) -> + Receiver ! {handshake_complete, S, DHandle, self()} + end + }. + + +ktls_input_handler() -> + receive + {handshake_complete, Socket, DHandle, DistUtil} -> + ok = erlang:dist_ctrl_input_handler(DHandle, self()), + inet:setopts( + Socket, + [ + {active, true}, + {deliver, term}, + {packet, 4}, + binary, + {nodelay, true} + ] + ), + ktls_input_handler_loop(Socket, DHandle, DistUtil) + end. + +ktls_input_handler_loop(Socket, DHandle, DistUtil) -> + receive + {tcp, Socket, Data} -> + persistent_term:put(ktls_input_handler_received, true), + erlang:dist_ctrl_put_data(DHandle, Data), + ktls_input_handler_loop(Socket, DHandle, DistUtil); + {tcp_error, Socket, _Error} -> + DistUtil ! {tcp_closed, Socket}, + inet:tcp_close(Socket); + {tcp_closed, Socket} -> + DistUtil ! {tcp_closed, Socket}, + inet:tcp_close(Socket) + end. + %%-------------------------------------------------------------------- %% Test net_kernel:monitor_nodes with nodedown_reason (OTP-17838) monitor_nodes(Config) when is_list(Config) -> From a00e9702af5e562d9a04a5e6bb83c20c67c0756f Mon Sep 17 00:00:00 2001 From: zzydxm Date: Tue, 25 Apr 2023 09:28:09 -0700 Subject: [PATCH 2/4] Update erts/doc/src/erlang.xml Co-authored-by: Rickard Green --- erts/doc/src/erlang.xml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/erts/doc/src/erlang.xml b/erts/doc/src/erlang.xml index 96839c2f809b..3af27e7bea85 100644 --- a/erts/doc/src/erlang.xml +++ b/erts/doc/src/erlang.xml @@ -1933,12 +1933,11 @@ end function. This function is also allowed to be called when the distribution controller for the distribution channel - identified by DHandle is a port, - for the port owner to be able to call - erlang:dist_ctrl_put_data/2 - when the port is using - {deliver, term} - option. + identified by DHandle is a port. + The data received by the port should in this case be delivered to + the process identified by InputHandler + which in turn should call + erlang:dist_ctrl_put_data/2.

This function is used when implementing an alternative From decad7ecc0b3b5fae88baf321df8aa65e8f1c4e5 Mon Sep 17 00:00:00 2001 From: Raimo Niskanen Date: Wed, 3 May 2023 11:20:07 +0200 Subject: [PATCH 3/4] Test input handler from internal inet_epmd cryptcookie framework --- .../test/inet_epmd_cryptcookie_inet_ktls.erl | 63 ++++++++++++++++++- lib/ssl/test/ssl_dist_bench_SUITE.erl | 21 ++++++- 2 files changed, 80 insertions(+), 4 deletions(-) diff --git a/lib/ssl/test/inet_epmd_cryptcookie_inet_ktls.erl b/lib/ssl/test/inet_epmd_cryptcookie_inet_ktls.erl index d350cc81391a..b2c082969c9f 100644 --- a/lib/ssl/test/inet_epmd_cryptcookie_inet_ktls.erl +++ b/lib/ssl/test/inet_epmd_cryptcookie_inet_ktls.erl @@ -103,7 +103,8 @@ accept_controller(_NetAddress, Controller, Socket) -> %% ------------------------------------------------------------ accepted(NetAddress, _Timer, Socket) -> - inet_epmd_dist:hs_data(NetAddress, Socket). + input_handler_setup( + inet_epmd_dist:hs_data(NetAddress, Socket)). %% ------------------------------------------------------------ connect(NetAddress, _Timer, Options) -> @@ -123,7 +124,8 @@ connect(NetAddress, _Timer, Options) -> inet_ktls_info(Socket, cryptcookie:ktls_info(CipherState)), ok ?= inet_tls_dist:set_ktls(KtlsInfo), ok ?= inet:setopts(Socket, [{packet, 2}, {mode, list}]), - inet_epmd_dist:hs_data(NetAddress, Socket) + input_handler_setup( + inet_epmd_dist:hs_data(NetAddress, Socket)) else {error, _} = Error -> Error @@ -204,6 +206,63 @@ stream_controlling_process(Stream = {_, [_ | Socket], _}, Pid) -> erlang:error({?MODULE, ?FUNCTION_NAME, Reason}) end. +%% ------------------------------------------------------------ + +input_handler_setup(#hs_data{} = HsData) -> + case init:get_argument(inet_ktls) of + {ok, [["port"]]} -> % No input_handler process + %% Just use the distribution port + HsData; + {ok, [["input_handler"]]} -> % Set up an input_handler process + %% Add an f_handshake_complete fun that spawns the input handler + %% and calls the f_setopts_post_nodeup fun + #hs_data{ + socket = Socket, + f_setopts_post_nodeup = FSetoptsPostNodeup} = HsData, + HsData#hs_data{ + f_setopts_post_nodeup = + fun (S) when S =:= Socket -> + ok + end, + f_handshake_complete = + fun (S, _Node, DHandle) when S =:= Socket -> + handshake_complete(S, FSetoptsPostNodeup, DHandle) + end} + end; +input_handler_setup({error, _} = Error) -> + Error. + +handshake_complete(Socket, FSetoptsPostNodeup, DHandle) -> + InputHandler = + spawn_link( + fun () -> + input_handler(Socket, DHandle) + end), + ok = ?DRIVER:controlling_process(Socket, InputHandler), + ok = erlang:dist_ctrl_input_handler(DHandle, InputHandler), + ok = FSetoptsPostNodeup(Socket). + +input_handler(Socket, DHandle) -> + receive + {tcp, Socket, Data} -> + erlang:dist_ctrl_put_data(DHandle, Data); + {tcp_error, Socket, _Error} = Reason -> + ?DRIVER:close(Socket), + exit(Reason); + {tcp_closed, Socket} = Reason -> + ?DRIVER:close(Socket), + exit(Reason); + Other -> + Reason = {unexpected_message, Other}, + error_report([?FUNCTION_NAME, {reason, Reason}]), + input_handler(Socket, DHandle) + end. + +%% ------------------------------------------------------------ + +error_report(Report) -> + error_logger:error_report(Report). + %% ------------------------------------------------------------ supported() -> maybe diff --git a/lib/ssl/test/ssl_dist_bench_SUITE.erl b/lib/ssl/test/ssl_dist_bench_SUITE.erl index 989007bec068..8e32e872ad17 100644 --- a/lib/ssl/test/ssl_dist_bench_SUITE.erl +++ b/lib/ssl/test/ssl_dist_bench_SUITE.erl @@ -77,6 +77,7 @@ groups() -> {cryptcookie_socket_ktls, categories()}, {dist_cryptcookie_inet, categories()}, {cryptcookie_inet_ktls, categories()}, + {cryptcookie_inet_ktls_ih, categories()}, %% %% categories() {setup, [{repeat, 1}], @@ -110,7 +111,8 @@ cryptcookie_backends() -> [{group, dist_cryptcookie_socket}, {group, cryptcookie_socket_ktls}, {group, dist_cryptcookie_inet}, - {group, cryptcookie_inet_ktls}]. + {group, cryptcookie_inet_ktls}, + {group, cryptcookie_inet_ktls_ih}]. categories() -> [{group, setup}, @@ -290,7 +292,22 @@ init_per_group(cryptcookie_inet_ktls, Config) -> ok -> [{ssl_dist, false}, {ssl_dist_prefix, "Crypto-Inet-kTLS"}, {ssl_dist_args, - "-proto_dist inet_epmd -inet_epmd cryptcookie_inet_ktls"} + "-proto_dist inet_epmd -inet_epmd cryptcookie_inet_ktls " + "-inet_ktls port"} + | Config]; + Problem -> + {skip, Problem} + catch + Class : Reason : Stacktrace -> + {fail, {Class, Reason, Stacktrace}} + end; +init_per_group(cryptcookie_inet_ktls_ih, Config) -> + try inet_epmd_cryptcookie_inet_ktls:supported() of + ok -> + [{ssl_dist, false}, {ssl_dist_prefix, "Crypto-Inet-kTLS-IH"}, + {ssl_dist_args, + "-proto_dist inet_epmd -inet_epmd cryptcookie_inet_ktls " + "-inet_ktls input_handler"} | Config]; Problem -> {skip, Problem} From cef93e8bffba8cc15f2b35fc97bb86d1f7c2fb9c Mon Sep 17 00:00:00 2001 From: Raimo Niskanen Date: Wed, 3 May 2023 17:18:56 +0200 Subject: [PATCH 4/4] Revert test hook in inet_tls_dist --- lib/ssl/src/inet_tls_dist.erl | 12 ++----- lib/ssl/test/ssl_dist_SUITE.erl | 56 --------------------------------- 2 files changed, 2 insertions(+), 66 deletions(-) diff --git a/lib/ssl/src/inet_tls_dist.erl b/lib/ssl/src/inet_tls_dist.erl index 95fc3a5a6b8f..c93bb2759634 100644 --- a/lib/ssl/src/inet_tls_dist.erl +++ b/lib/ssl/src/inet_tls_dist.erl @@ -528,7 +528,7 @@ do_accept( timer = Timer, this_flags = 0, allowed = NewAllowed}, - dist_util:handshake_other_started(trace(hs_data_finalize(HSData))); + dist_util:handshake_other_started(trace(HSData)); {AcceptPid, exit} -> %% this can happen when connection was initiated, but dropped %% between TLS handshake completion and dist handshake start @@ -677,7 +677,7 @@ do_setup( this_flags = 0, other_version = Version, request_type = Type}, - dist_util:handshake_we_started(trace(hs_data_finalize(HSData))) + dist_util:handshake_we_started(trace(HSData)) else Other -> %% Other Node may have closed since @@ -1088,14 +1088,6 @@ ktls_opt_cipher( {error, {ktls_notsup, {cipher, TLS_version, CipherSpec, _CipherState}}}. -hs_data_finalize(HSData) -> - case application:get_env(kernel, dist_hs_data_finalize_fun) of - {ok, Fun} when is_function(Fun) -> - Fun(HSData); - _ -> - HSData - end. - %% ------------------------------------------------------------------------- diff --git a/lib/ssl/test/ssl_dist_SUITE.erl b/lib/ssl/test/ssl_dist_SUITE.erl index 22e7a179fa71..0194ff4dce00 100644 --- a/lib/ssl/test/ssl_dist_SUITE.erl +++ b/lib/ssl/test/ssl_dist_SUITE.erl @@ -23,7 +23,6 @@ -behaviour(ct_suite). --include_lib("kernel/include/dist_util.hrl"). -include_lib("kernel/include/net_address.hrl"). -include_lib("common_test/include/ct.hrl"). -include_lib("public_key/include/public_key.hrl"). @@ -360,7 +359,6 @@ ktls_verify(Config) -> basic_test(NH1, NH2, KTLSConfig), 0 = ktls_count_tls_dist(NH1), 0 = ktls_count_tls_dist(NH2), - input_handler_test(NH1, NH2, KTLSConfig), ok end, KTLSConfig). @@ -383,60 +381,6 @@ ktls_count_tls_dist(Node) -> 0 end. -%% Verify that dist input handler can be set if dist_ctrl is port; -%% and can send pessage properly -input_handler_test(NH1, NH2, KTLSConfig) -> - #node_handle{nodename = Node2} = NH2, - apply_on_ssl_node(NH1, application, set_env, [ - kernel, dist_hs_data_finalize_fun, fun ktls_hs_data_finalize/1 - ]), - true = apply_on_ssl_node(NH1, erlang, disconnect_node, [Node2]), - true = apply_on_ssl_node(NH1, net_kernel, connect_node, [Node2]), - basic_test(NH1, NH2, KTLSConfig), - true = apply_on_ssl_node(NH1, persistent_term, get, [ktls_input_handler_received]). - -ktls_hs_data_finalize(HSData = #hs_data{socket = Socket}) -> - Receiver = spawn_link(fun ktls_input_handler/0), - ok = gen_tcp:controlling_process(Socket, Receiver), - HSData#hs_data{ - f_setopts_post_nodeup = fun (_) -> ok end, - f_handshake_complete = fun (S, _Node, DHandle) -> - Receiver ! {handshake_complete, S, DHandle, self()} - end - }. - - -ktls_input_handler() -> - receive - {handshake_complete, Socket, DHandle, DistUtil} -> - ok = erlang:dist_ctrl_input_handler(DHandle, self()), - inet:setopts( - Socket, - [ - {active, true}, - {deliver, term}, - {packet, 4}, - binary, - {nodelay, true} - ] - ), - ktls_input_handler_loop(Socket, DHandle, DistUtil) - end. - -ktls_input_handler_loop(Socket, DHandle, DistUtil) -> - receive - {tcp, Socket, Data} -> - persistent_term:put(ktls_input_handler_received, true), - erlang:dist_ctrl_put_data(DHandle, Data), - ktls_input_handler_loop(Socket, DHandle, DistUtil); - {tcp_error, Socket, _Error} -> - DistUtil ! {tcp_closed, Socket}, - inet:tcp_close(Socket); - {tcp_closed, Socket} -> - DistUtil ! {tcp_closed, Socket}, - inet:tcp_close(Socket) - end. - %%-------------------------------------------------------------------- %% Test net_kernel:monitor_nodes with nodedown_reason (OTP-17838) monitor_nodes(Config) when is_list(Config) ->