From 888233131ffa9ee5dda282f920f7a0c1723b9a79 Mon Sep 17 00:00:00 2001 From: Bucky Kittinger Date: Thu, 2 Jan 2020 14:51:57 -0500 Subject: [PATCH 01/15] bump version --- CMakeLists.txt | 2 +- README.md | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 859d47b69f1..dd9eb965061 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -28,7 +28,7 @@ set( CXX_STANDARD_REQUIRED ON) set(VERSION_MAJOR 1) set(VERSION_MINOR 8) -set(VERSION_PATCH 7) +set(VERSION_PATCH 8) #set(VERSION_SUFFIX develop) if(VERSION_SUFFIX) diff --git a/README.md b/README.md index 6cb4bee7081..9d2c44817ec 100644 --- a/README.md +++ b/README.md @@ -44,13 +44,13 @@ $ brew remove eosio #### Ubuntu 18.04 Package Install ```sh -$ wget https://github.com/eosio/eos/releases/download/v1.8.7/eosio_1.8.7-1-ubuntu-18.04_amd64.deb -$ sudo apt install ./eosio_1.8.7-1-ubuntu-18.04_amd64.deb +$ wget https://github.com/eosio/eos/releases/download/v1.8.8/eosio_1.8.8-1-ubuntu-18.04_amd64.deb +$ sudo apt install ./eosio_1.8.8-1-ubuntu-18.04_amd64.deb ``` #### Ubuntu 16.04 Package Install ```sh -$ wget https://github.com/eosio/eos/releases/download/v1.8.7/eosio_1.8.7-1-ubuntu-16.04_amd64.deb -$ sudo apt install ./eosio_1.8.7-1-ubuntu-16.04_amd64.deb +$ wget https://github.com/eosio/eos/releases/download/v1.8.8/eosio_1.8.8-1-ubuntu-16.04_amd64.deb +$ sudo apt install ./eosio_1.8.8-1-ubuntu-16.04_amd64.deb ``` #### Ubuntu Package Uninstall ```sh @@ -58,8 +58,8 @@ $ sudo apt remove eosio ``` #### Centos RPM Package Install ```sh -$ wget https://github.com/eosio/eos/releases/download/v1.8.7/eosio-1.8.7-1.el7.x86_64.rpm -$ sudo yum install ./eosio-1.8.7-1.el7.x86_64.rpm +$ wget https://github.com/eosio/eos/releases/download/v1.8.8/eosio-1.8.8-1.el7.x86_64.rpm +$ sudo yum install ./eosio-1.8.8-1.el7.x86_64.rpm ``` #### Centos RPM Package Uninstall ```sh From 84e6bc0215474b191e8644553724f387a5b3f197 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Fri, 3 Jan 2020 09:38:38 -0500 Subject: [PATCH 02/15] Disable eosio-sync-from-genesis test on eosio-lrt pipeline --- .cicd/generate-pipeline.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index 96aeb22643e..c423719d3c8 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -374,7 +374,7 @@ EOF fi fi # trigger eosio-sync-from-genesis for every build -if [[ -z $BUILDKITE_TRIGGERED_FROM_BUILD_ID && -z "${SKIP_INSTALL}${SKIP_LINUX}${SKIP_DOCKER}" ]]; then +if [[ "$BUILDKITE_PIPELINE_SLUG" == 'eosio' && -z "${SKIP_INSTALL}${SKIP_LINUX}${SKIP_DOCKER}" ]]; then cat < Date: Fri, 3 Jan 2020 09:40:55 -0500 Subject: [PATCH 03/15] Include the actual source pipeline slug in the build message --- .cicd/generate-pipeline.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index c423719d3c8..dc8a3287459 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -339,7 +339,7 @@ if [[ -z $BUILDKITE_TRIGGERED_FROM_BUILD_ID && $TRIGGER_JOB == "true" ]]; then trigger: "eosio-lrt" async: true build: - message: "Triggered by EOSIO build ${BUILDKITE_BUILD_NUMBER}" + message: "Triggered by $BUILDKITE_PIPELINE_SLUG build $BUILDKITE_BUILD_NUMBER" commit: "${BUILDKITE_COMMIT}" branch: "${BUILDKITE_BRANCH}" env: @@ -361,7 +361,7 @@ if [[ -z $BUILDKITE_TRIGGERED_FROM_BUILD_ID && $TRIGGER_JOB = "true" ]]; then trigger: "eos-multiversion-tests" async: true build: - message: "Triggered by EOSIO build ${BUILDKITE_BUILD_NUMBER}" + message: "Triggered by $BUILDKITE_PIPELINE_SLUG build $BUILDKITE_BUILD_NUMBER" commit: "${BUILDKITE_COMMIT}" branch: "${BUILDKITE_BRANCH}" env: @@ -380,7 +380,7 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == 'eosio' && -z "${SKIP_INSTALL}${SKIP_LINUX}$ trigger: "eosio-sync-from-genesis" async: false build: - message: "Triggered by EOSIO build ${BUILDKITE_BUILD_NUMBER}" + message: "Triggered by $BUILDKITE_PIPELINE_SLUG build $BUILDKITE_BUILD_NUMBER" commit: "${BUILDKITE_COMMIT}" branch: "${BUILDKITE_BRANCH}" env: From 1fa1290d3fc04f3ea0d65e77a8e407c061a87a25 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Fri, 3 Jan 2020 09:42:34 -0500 Subject: [PATCH 04/15] Consistent variable names across Buildkite --- .cicd/generate-pipeline.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index dc8a3287459..18bdd24b82c 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -346,7 +346,7 @@ if [[ -z $BUILDKITE_TRIGGERED_FROM_BUILD_ID && $TRIGGER_JOB == "true" ]]; then BUILDKITE_PULL_REQUEST: "${BUILDKITE_PULL_REQUEST}" BUILDKITE_PULL_REQUEST_BASE_BRANCH: "${BUILDKITE_PULL_REQUEST_BASE_BRANCH}" BUILDKITE_PULL_REQUEST_REPO: "${BUILDKITE_PULL_REQUEST_REPO}" - BUILDKITE_TRIGGERED_FROM_BUILD_LINK: "${BUILDKITE_BUILD_URL}" + BUILDKITE_TRIGGERED_FROM_BUILD_URL: "${BUILDKITE_BUILD_URL}" SKIP_BUILD: "true" PINNED: "${PINNED}" @@ -368,7 +368,7 @@ if [[ -z $BUILDKITE_TRIGGERED_FROM_BUILD_ID && $TRIGGER_JOB = "true" ]]; then BUILDKITE_PULL_REQUEST: "${BUILDKITE_PULL_REQUEST}" BUILDKITE_PULL_REQUEST_BASE_BRANCH: "${BUILDKITE_PULL_REQUEST_BASE_BRANCH}" BUILDKITE_PULL_REQUEST_REPO: "${BUILDKITE_PULL_REQUEST_REPO}" - BUILDKITE_TRIGGERED_FROM_BUILD_LINK: "${BUILDKITE_BUILD_URL}" + BUILDKITE_TRIGGERED_FROM_BUILD_URL: "${BUILDKITE_BUILD_URL}" EOF fi @@ -384,7 +384,7 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == 'eosio' && -z "${SKIP_INSTALL}${SKIP_LINUX}$ commit: "${BUILDKITE_COMMIT}" branch: "${BUILDKITE_BRANCH}" env: - BUILDKITE_TRIGGERED_FROM_BUILD_LINK: "${BUILDKITE_BUILD_URL}" + BUILDKITE_TRIGGERED_FROM_BUILD_URL: "${BUILDKITE_BUILD_URL}" SKIP_JUNGLE: "${SKIP_JUNGLE}" SKIP_KYLIN: "${SKIP_KYLIN}" SKIP_MAIN: "${SKIP_MAIN}" From e02a7a5d4001d7d30f678b4e016a1b54c3feddc0 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Fri, 3 Jan 2020 09:49:45 -0500 Subject: [PATCH 05/15] Remove dashes from test name --- .cicd/generate-pipeline.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index 18bdd24b82c..7e93f219bd3 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -376,7 +376,7 @@ fi # trigger eosio-sync-from-genesis for every build if [[ "$BUILDKITE_PIPELINE_SLUG" == 'eosio' && -z "${SKIP_INSTALL}${SKIP_LINUX}${SKIP_DOCKER}" ]]; then cat < Date: Fri, 3 Jan 2020 09:53:29 -0500 Subject: [PATCH 06/15] Add SKIP_SYNC_TESTS variable --- .cicd/generate-pipeline.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index 7e93f219bd3..e2dab663e3f 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -374,7 +374,7 @@ EOF fi fi # trigger eosio-sync-from-genesis for every build -if [[ "$BUILDKITE_PIPELINE_SLUG" == 'eosio' && -z "${SKIP_INSTALL}${SKIP_LINUX}${SKIP_DOCKER}" ]]; then +if [[ "$BUILDKITE_PIPELINE_SLUG" == 'eosio' && -z "${SKIP_INSTALL}${SKIP_LINUX}${SKIP_DOCKER}${SKIP_SYNC_TESTS}" ]]; then cat < Date: Fri, 3 Jan 2020 09:58:18 -0500 Subject: [PATCH 07/15] Fix duplicate deletion of the same docker container on some tagged builds --- .cicd/docker-tag.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cicd/docker-tag.sh b/.cicd/docker-tag.sh index d9f03c06a52..18ef347a0a3 100755 --- a/.cicd/docker-tag.sh +++ b/.cicd/docker-tag.sh @@ -18,5 +18,5 @@ docker push "$REPO:$PREFIX-$SANITIZED_BRANCH" [[ -z "$BUILDKITE_TAG" ]] || docker push "$REPO:$PREFIX-$SANITIZED_TAG" echo '+++ :put_litter_in_its_place: Cleaning Up' docker rmi "$REPO:$PREFIX-$SANITIZED_BRANCH" -[[ -z "$BUILDKITE_TAG" ]] || docker rmi "$REPO:$PREFIX-$SANITIZED_TAG" +[[ -z "$BUILDKITE_TAG" || "$SANITIZED_BRANCH" == "$SANITIZED_TAG" ]] || docker rmi "$REPO:$PREFIX-$SANITIZED_TAG" docker rmi "$IMAGE" \ No newline at end of file From dd5c9a76954b11fdd4b918bdce0fd98a7b37299d Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Thu, 2 Jan 2020 17:21:12 -0500 Subject: [PATCH 08/15] Add eosio-resume-from-state configuration to pipeline.jsonc --- pipeline.jsonc | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pipeline.jsonc b/pipeline.jsonc index b9430524cf4..c526684f7ab 100644 --- a/pipeline.jsonc +++ b/pipeline.jsonc @@ -32,5 +32,17 @@ "SKIP_PRE_V180": "true", "SKIP_V180": "false" } + }, + "eosio-resume-from-state": + { + "test": + [ + { + "tag": "v1.8.6" + }, + { + "tag": "v1.8.7" + } + ] } } \ No newline at end of file From 23bdb0f1272027e4e7759a8591556dd18b53c389 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Fri, 3 Jan 2020 09:58:34 -0500 Subject: [PATCH 09/15] Add eosio-resume-from-state test to eosio pipeline --- .cicd/generate-pipeline.sh | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index e2dab663e3f..2ab222638c3 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -390,6 +390,25 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == 'eosio' && -z "${SKIP_INSTALL}${SKIP_LINUX}$ SKIP_MAIN: "${SKIP_MAIN}" TIMEOUT: "${TIMEOUT}" +EOF +fi +# trigger eosio-resume-from-state for every build +if [[ "$BUILDKITE_PIPELINE_SLUG" == 'eosio' && -z "${SKIP_INSTALL}${SKIP_LINUX}${SKIP_DOCKER}${SKIP_SYNC_TESTS}" ]]; then + cat < Date: Thu, 19 Dec 2019 17:06:22 -0600 Subject: [PATCH 10/15] Make multiversion protocol test conditional. The multiversion protocol test does not work outside of the CI/CD environment. Add "ENABLE_MULTIVERSION_PROTOCOL_TEST" -D flag to CMakeLists, defaulting to false, and enable it in CI/CD build script. --- .cicd/build.sh | 2 +- CMakeLists.txt | 1 + tests/CMakeLists.txt | 10 ++++++---- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.cicd/build.sh b/.cicd/build.sh index 644b869fbfc..e22658d57b6 100755 --- a/.cicd/build.sh +++ b/.cicd/build.sh @@ -2,7 +2,7 @@ set -eo pipefail . ./.cicd/helpers/general.sh mkdir -p $BUILD_DIR -CMAKE_EXTRAS="-DCMAKE_BUILD_TYPE='Release'" +CMAKE_EXTRAS="-DCMAKE_BUILD_TYPE='Release' -DENABLE_MULTIVERSION_PROTOCOL_TEST=true" if [[ "$(uname)" == 'Darwin' ]]; then # You can't use chained commands in execute if [[ "$TRAVIS" == 'true' ]]; then diff --git a/CMakeLists.txt b/CMakeLists.txt index dd9eb965061..4cfe302130c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -64,6 +64,7 @@ endif() set(CMAKE_EXPORT_COMPILE_COMMANDS "ON") set(BUILD_DOXYGEN FALSE CACHE BOOL "Build doxygen documentation on every make") set(BUILD_MONGO_DB_PLUGIN FALSE CACHE BOOL "Build mongo database plugin") +set(ENABLE_MULTIVERSION_PROTOCOL_TEST FALSE CACHE BOOL "Enable nodeos multiversion protocol test") # add defaults for openssl if ("${OPENSSL_ROOT_DIR}" STREQUAL "") diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 519852d370c..3a6b1ea7bf8 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -121,11 +121,13 @@ set_property(TEST nodeos_startup_catchup_lr_test PROPERTY LABELS long_running_te add_test(NAME nodeos_short_fork_take_over_lr_test COMMAND tests/nodeos_short_fork_take_over_test.py -v --wallet-port 9905 --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) set_property(TEST nodeos_short_fork_take_over_lr_test PROPERTY LABELS long_running_tests) -set(ALTERNATE_VERSION_LABELS_FILE "${CMAKE_BINARY_DIR}/tests/multiversion_paths.conf") +if(ENABLE_MULTIVERSION_PROTOCOL_TEST) + set(ALTERNATE_VERSION_LABELS_FILE "${CMAKE_BINARY_DIR}/tests/multiversion_paths.conf") -add_test(NAME nodeos_multiple_version_protocol_feature_mv_test COMMAND tests/nodeos_multiple_version_protocol_feature_test.py - -v --clean-run --dump-error-detail --alternate-version-labels-file ${ALTERNATE_VERSION_LABELS_FILE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) -set_property(TEST nodeos_multiple_version_protocol_feature_mv_test PROPERTY LABELS mixed_version_tests) + add_test(NAME nodeos_multiple_version_protocol_feature_mv_test COMMAND tests/nodeos_multiple_version_protocol_feature_test.py + -v --clean-run --dump-error-detail --alternate-version-labels-file ${ALTERNATE_VERSION_LABELS_FILE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) + set_property(TEST nodeos_multiple_version_protocol_feature_mv_test PROPERTY LABELS mixed_version_tests) +endif() add_test(NAME nodeos_producer_watermark_lr_test COMMAND tests/nodeos_producer_watermark_test.py -v --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) set_property(TEST nodeos_producer_watermark_lr_test PROPERTY LABELS long_running_tests) From 766ae8c953d730c54c7dfd348702a31cbc60390e Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Mon, 6 Jan 2020 16:46:13 -0600 Subject: [PATCH 11/15] Update sync_next_expected_num via checking sync_last_requested_num instead of checking sync_state as sync_state might be temp in diff state. Also add blk_applied to sync_update_expected. --- plugins/net_plugin/net_plugin.cpp | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index 203077c21e0..ce096d6d431 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -684,7 +684,8 @@ namespace eosio { void reassign_fetch(const connection_ptr& c, go_away_reason reason); bool verify_catchup(const connection_ptr& c, uint32_t num, const block_id_type& id); void rejected_block(const connection_ptr& c, uint32_t blk_num); - void recv_block(const connection_ptr& c, const block_id_type& blk_id, uint32_t blk_num); + void recv_block(const connection_ptr& c, const block_id_type& blk_id, uint32_t blk_num, bool blk_applied); + void sync_update_expected(const connection_ptr& c, const block_id_type& blk_id, uint32_t blk_num, bool blk_applied); void recv_handshake(const connection_ptr& c, const handshake_message& msg); void recv_notice(const connection_ptr& c, const notice_message& msg); }; @@ -1598,16 +1599,23 @@ namespace eosio { c->send_handshake(); } } - void sync_manager::recv_block(const connection_ptr& c, const block_id_type& blk_id, uint32_t blk_num) { - fc_dlog(logger, "got block ${bn} from ${p}",("bn",blk_num)("p",c->peer_name())); - if (state == lib_catchup) { - if (blk_num != sync_next_expected_num) { - fc_wlog( logger, "expected block ${ne} but got ${bn}, from connection: ${p}", - ("ne",sync_next_expected_num)("bn",blk_num)("p",c->peer_name()) ); + + void sync_manager::sync_update_expected(const connection_ptr& c, const block_id_type& blk_id, uint32_t blk_num, bool blk_applied) { + if( blk_num <= sync_last_requested_num ) { + fc_dlog( logger, "sync_last_requested_num: ${r}, sync_next_expected_num: ${e}, sync_known_lib_num: ${k}, sync_req_span: ${s}", + ("r", sync_last_requested_num)("e", sync_next_expected_num)("k", sync_known_lib_num)("s", sync_req_span) ); + if (blk_num != sync_next_expected_num && !blk_applied) { + fc_dlog( logger, "expected block ${ne} but got ${bn}, from connection: ${p}", + ("ne", sync_next_expected_num)( "bn", blk_num )( "p", c->peer_name() ) ); return; } sync_next_expected_num = blk_num + 1; } + } + + void sync_manager::recv_block(const connection_ptr& c, const block_id_type& blk_id, uint32_t blk_num, bool blk_applied) { + fc_dlog(logger, "got block ${bn} from ${p}",("bn",blk_num)("p",c->peer_name())); + sync_update_expected(c, blk_id, blk_num, blk_applied); if (state == head_catchup) { fc_dlog(logger, "sync_manager in head_catchup state"); set_state(in_sync); @@ -2192,8 +2200,7 @@ namespace eosio { } } if( cc.fetch_block_by_id( blk_id ) ) { - if( sync_master->syncing_with_peer() ) - sync_master->recv_block( conn, blk_id, blk_num ); + sync_master->recv_block( conn, blk_id, blk_num, false ); conn->cancel_wait(); conn->pending_message_buffer.advance_read_ptr( message_length ); return true; @@ -2595,8 +2602,7 @@ namespace eosio { try { if( cc.fetch_block_by_id(blk_id)) { - if( sync_master->syncing_with_peer() ) - sync_master->recv_block( c, blk_id, blk_num ); + sync_master->recv_block( c, blk_id, blk_num, false ); c->cancel_wait(); return; } @@ -2645,7 +2651,7 @@ namespace eosio { c->trx_state.modify( ctx, ubn ); } } - sync_master->recv_block(c, blk_id, blk_num); + sync_master->recv_block(c, blk_id, blk_num, true); } else { sync_master->rejected_block(c, blk_num); From 7675dada49adba4fbc8ddb1f73c645d2b2250626 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Thu, 9 Jan 2020 08:32:53 -0600 Subject: [PATCH 12/15] Add better logging of exceptions in emit --- libraries/chain/controller.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index 21481c66e3d..4123e0ac77f 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -351,16 +351,18 @@ struct controller_impl { try { s( std::forward( a )); } catch (std::bad_alloc& e) { - wlog( "std::bad_alloc" ); + wlog( "std::bad_alloc: ${w}", ("w", e.what()) ); throw e; } catch (boost::interprocess::bad_alloc& e) { - wlog( "bad alloc" ); + wlog( "boost::interprocess::bad alloc: ${w}", ("w", e.what()) ); throw e; } catch ( controller_emit_signal_exception& e ) { - wlog( "${details}", ("details", e.to_detail_string()) ); + wlog( "controller_emit_signal_exception: ${details}", ("details", e.to_detail_string()) ); throw e; } catch ( fc::exception& e ) { - wlog( "${details}", ("details", e.to_detail_string()) ); + wlog( "fc::exception: ${details}", ("details", e.to_detail_string()) ); + } catch ( std::exception& e ) { + wlog( "std::exception: ${details}", ("details", e.what()) ); } catch ( ... ) { wlog( "signal handler threw exception" ); } From 5b910a1a74d04fe528f8f7b98d3752339519704a Mon Sep 17 00:00:00 2001 From: Matt Witherspoon <32485495+spoonincode@users.noreply.github.com> Date: Wed, 20 Nov 2019 13:48:46 -0500 Subject: [PATCH 13/15] return more from producer_plugin's get_runtime_options() --- libraries/chain/controller.cpp | 4 ++++ libraries/chain/include/eosio/chain/controller.hpp | 1 + plugins/producer_plugin/producer_plugin.cpp | 7 ++++++- 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index 21481c66e3d..c69e24b0349 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -3001,6 +3001,10 @@ void controller::set_subjective_cpu_leeway(fc::microseconds leeway) { my->subjective_cpu_leeway = leeway; } +fc::optional controller::get_subjective_cpu_leeway() const { + return my->subjective_cpu_leeway; +} + void controller::set_greylist_limit( uint32_t limit ) { EOS_ASSERT( 0 < limit && limit <= chain::config::maximum_elastic_resource_multiplier, misc_exception, diff --git a/libraries/chain/include/eosio/chain/controller.hpp b/libraries/chain/include/eosio/chain/controller.hpp index d20e7b5b5fc..1f9ff9bf871 100644 --- a/libraries/chain/include/eosio/chain/controller.hpp +++ b/libraries/chain/include/eosio/chain/controller.hpp @@ -272,6 +272,7 @@ namespace eosio { namespace chain { validation_mode get_validation_mode()const; void set_subjective_cpu_leeway(fc::microseconds leeway); + fc::optional get_subjective_cpu_leeway() const; void set_greylist_limit( uint32_t limit ); uint32_t get_greylist_limit()const; diff --git a/plugins/producer_plugin/producer_plugin.cpp b/plugins/producer_plugin/producer_plugin.cpp index f207602aee6..4b36400534a 100644 --- a/plugins/producer_plugin/producer_plugin.cpp +++ b/plugins/producer_plugin/producer_plugin.cpp @@ -956,7 +956,12 @@ producer_plugin::runtime_options producer_plugin::get_runtime_options() const { my->_max_irreversible_block_age_us.count() < 0 ? -1 : my->_max_irreversible_block_age_us.count() / 1'000'000, my->_produce_time_offset_us, my->_last_block_time_offset_us, - my->_max_scheduled_transaction_time_per_block_ms + my->_max_scheduled_transaction_time_per_block_ms, + my->chain_plug->chain().get_subjective_cpu_leeway() ? + my->chain_plug->chain().get_subjective_cpu_leeway()->count() : + fc::optional(), + my->_incoming_defer_ratio, + my->chain_plug->chain().get_greylist_limit() }; } From 749385cb472e5a4f16a607845e14d1f8a2615d0f Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Thu, 9 Jan 2020 10:44:44 -0600 Subject: [PATCH 14/15] Consolidated Security Fixes for 1.8.9 - Limit size of response to API requests Co-Authored-By: heifnerk@objectcomputing.com --- libraries/chain/abi_serializer.cpp | 123 +++++++++--------- .../include/eosio/chain/abi_serializer.hpp | 74 ++++++----- libraries/fc | 2 +- plugins/http_plugin/http_plugin.cpp | 100 +++++++++----- plugins/mongo_db_plugin/mongo_db_plugin.cpp | 16 +-- programs/cleos/help_text.cpp.in | 2 +- programs/cleos/httpc.cpp | 2 +- programs/cleos/main.cpp | 4 +- programs/eosio-blocklog/main.cpp | 2 +- programs/eosio-launcher/main.cpp | 4 +- tests/Cluster.py | 2 +- tests/Node.py | 2 +- unittests/abi_tests.cpp | 6 +- unittests/wasm_tests.cpp | 2 +- 14 files changed, 198 insertions(+), 143 deletions(-) diff --git a/libraries/chain/abi_serializer.cpp b/libraries/chain/abi_serializer.cpp index 2f453e0cf93..91950de1f12 100644 --- a/libraries/chain/abi_serializer.cpp +++ b/libraries/chain/abi_serializer.cpp @@ -3,8 +3,6 @@ * @copyright defined in eos/LICENSE */ #include -#include -#include #include #include #include @@ -21,6 +19,7 @@ namespace eosio { namespace chain { using boost::algorithm::ends_with; using std::string; + using std::string_view; template inline fc::variant variant_from_stream(fc::datastream& stream) { @@ -120,7 +119,8 @@ namespace eosio { namespace chain { for( const auto& td : abi.types ) { EOS_ASSERT(_is_type(td.type, ctx), invalid_type_inside_abi, "invalid type ${type}", ("type",td.type)); - EOS_ASSERT(!_is_type(td.new_type_name, ctx), duplicate_abi_type_def_exception, "type already exists", ("new_type_name",td.new_type_name)); + EOS_ASSERT(!_is_type(td.new_type_name, ctx), duplicate_abi_type_def_exception, + "type already exists", ("new_type_name",impl::limit_size(td.new_type_name))); typedefs[td.new_type_name] = td.type; } @@ -150,60 +150,58 @@ namespace eosio { namespace chain { validate(ctx); } - bool abi_serializer::is_builtin_type(const type_name& type)const { + bool abi_serializer::is_builtin_type(const std::string_view& type)const { return built_in_types.find(type) != built_in_types.end(); } - bool abi_serializer::is_integer(const type_name& type) const { - string stype = type; - return boost::starts_with(stype, "uint") || boost::starts_with(stype, "int"); + bool abi_serializer::is_integer(const std::string_view& type) const { + return boost::starts_with(type, "uint") || boost::starts_with(type, "int"); } - int abi_serializer::get_integer_size(const type_name& type) const { - string stype = type; - EOS_ASSERT( is_integer(type), invalid_type_inside_abi, "${stype} is not an integer type", ("stype",stype)); - if( boost::starts_with(stype, "uint") ) { - return boost::lexical_cast(stype.substr(4)); + int abi_serializer::get_integer_size(const std::string_view& type) const { + EOS_ASSERT( is_integer(type), invalid_type_inside_abi, "${type} is not an integer type", ("type",impl::limit_size(type))); + if( boost::starts_with(type, "uint") ) { + return boost::lexical_cast(type.substr(4)); } else { - return boost::lexical_cast(stype.substr(3)); + return boost::lexical_cast(type.substr(3)); } } - bool abi_serializer::is_struct(const type_name& type)const { + bool abi_serializer::is_struct(const std::string_view& type)const { return structs.find(resolve_type(type)) != structs.end(); } - bool abi_serializer::is_array(const type_name& type)const { - return ends_with(string(type), "[]"); + bool abi_serializer::is_array(const string_view& type)const { + return ends_with(type, "[]"); } - bool abi_serializer::is_optional(const type_name& type)const { - return ends_with(string(type), "?"); + bool abi_serializer::is_optional(const string_view& type)const { + return ends_with(type, "?"); } - bool abi_serializer::is_type(const type_name& type, const fc::microseconds& max_serialization_time)const { + bool abi_serializer::is_type(const std::string_view& type, const fc::microseconds& max_serialization_time)const { impl::abi_traverse_context ctx(max_serialization_time); return _is_type(type, ctx); } - type_name abi_serializer::fundamental_type(const type_name& type)const { + std::string_view abi_serializer::fundamental_type(const std::string_view& type)const { if( is_array(type) ) { - return type_name(string(type).substr(0, type.size()-2)); + return type.substr(0, type.size()-2); } else if ( is_optional(type) ) { - return type_name(string(type).substr(0, type.size()-1)); + return type.substr(0, type.size()-1); } else { return type; } } - type_name abi_serializer::_remove_bin_extension(const type_name& type) { + std::string_view abi_serializer::_remove_bin_extension(const std::string_view& type) { if( ends_with(type, "$") ) return type.substr(0, type.size()-1); else return type; } - bool abi_serializer::_is_type(const type_name& rtype, impl::abi_traverse_context& ctx )const { + bool abi_serializer::_is_type(const std::string_view& rtype, impl::abi_traverse_context& ctx )const { auto h = ctx.enter_scope(); auto type = fundamental_type(rtype); if( built_in_types.find(type) != built_in_types.end() ) return true; @@ -213,65 +211,68 @@ namespace eosio { namespace chain { return false; } - const struct_def& abi_serializer::get_struct(const type_name& type)const { + const struct_def& abi_serializer::get_struct(const std::string_view& type)const { auto itr = structs.find(resolve_type(type) ); - EOS_ASSERT( itr != structs.end(), invalid_type_inside_abi, "Unknown struct ${type}", ("type",type) ); + EOS_ASSERT( itr != structs.end(), invalid_type_inside_abi, "Unknown struct ${type}", ("type",impl::limit_size(type)) ); return itr->second; } void abi_serializer::validate( impl::abi_traverse_context& ctx )const { for( const auto& t : typedefs ) { try { - vector types_seen{t.first, t.second}; + vector types_seen{t.first, t.second}; auto itr = typedefs.find(t.second); while( itr != typedefs.end() ) { ctx.check_deadline(); - EOS_ASSERT( find(types_seen.begin(), types_seen.end(), itr->second) == types_seen.end(), abi_circular_def_exception, "Circular reference in type ${type}", ("type",t.first) ); + EOS_ASSERT( find(types_seen.begin(), types_seen.end(), itr->second) == types_seen.end(), abi_circular_def_exception, + "Circular reference in type ${type}", ("type", impl::limit_size(t.first)) ); types_seen.emplace_back(itr->second); itr = typedefs.find(itr->second); } } FC_CAPTURE_AND_RETHROW( (t) ) } for( const auto& t : typedefs ) { try { - EOS_ASSERT(_is_type(t.second, ctx), invalid_type_inside_abi, "${type}", ("type",t.second) ); + EOS_ASSERT(_is_type(t.second, ctx), invalid_type_inside_abi, "${type}", ("type",impl::limit_size(t.second)) ); } FC_CAPTURE_AND_RETHROW( (t) ) } for( const auto& s : structs ) { try { if( s.second.base != type_name() ) { - struct_def current = s.second; - vector types_seen{current.name}; - while( current.base != type_name() ) { + const struct_def* current = &s.second; + vector types_seen{current->name}; + while( current->base != type_name() ) { ctx.check_deadline(); - const auto& base = get_struct(current.base); //<-- force struct to inherit from another struct - EOS_ASSERT( find(types_seen.begin(), types_seen.end(), base.name) == types_seen.end(), abi_circular_def_exception, "Circular reference in struct ${type}", ("type",s.second.name) ); + const struct_def& base = get_struct(current->base); //<-- force struct to inherit from another struct + EOS_ASSERT( find(types_seen.begin(), types_seen.end(), base.name) == types_seen.end(), abi_circular_def_exception, + "Circular reference in struct ${type}", ("type",impl::limit_size(s.second.name)) ); types_seen.emplace_back(base.name); - current = base; + current = &base; } } for( const auto& field : s.second.fields ) { try { ctx.check_deadline(); - EOS_ASSERT(_is_type(_remove_bin_extension(field.type), ctx), invalid_type_inside_abi, "${type}", ("type",field.type) ); + EOS_ASSERT(_is_type(_remove_bin_extension(field.type), ctx), invalid_type_inside_abi, + "${type}", ("type",impl::limit_size(field.type)) ); } FC_CAPTURE_AND_RETHROW( (field) ) } } FC_CAPTURE_AND_RETHROW( (s) ) } for( const auto& s : variants ) { try { for( const auto& type : s.second.types ) { try { ctx.check_deadline(); - EOS_ASSERT(_is_type(type, ctx), invalid_type_inside_abi, "${type}", ("type",type) ); + EOS_ASSERT(_is_type(type, ctx), invalid_type_inside_abi, "${type}", ("type",impl::limit_size(type)) ); } FC_CAPTURE_AND_RETHROW( (type) ) } } FC_CAPTURE_AND_RETHROW( (s) ) } for( const auto& a : actions ) { try { ctx.check_deadline(); - EOS_ASSERT(_is_type(a.second, ctx), invalid_type_inside_abi, "${type}", ("type",a.second) ); + EOS_ASSERT(_is_type(a.second, ctx), invalid_type_inside_abi, "${type}", ("type",impl::limit_size(a.second)) ); } FC_CAPTURE_AND_RETHROW( (a) ) } for( const auto& t : tables ) { try { ctx.check_deadline(); - EOS_ASSERT(_is_type(t.second, ctx), invalid_type_inside_abi, "${type}", ("type",t.second) ); + EOS_ASSERT(_is_type(t.second, ctx), invalid_type_inside_abi, "${type}", ("type",impl::limit_size(t.second)) ); } FC_CAPTURE_AND_RETHROW( (t) ) } } - type_name abi_serializer::resolve_type(const type_name& type)const { + std::string_view abi_serializer::resolve_type(const std::string_view& type)const { auto itr = typedefs.find(type); if( itr != typedefs.end() ) { for( auto i = typedefs.size(); i > 0; --i ) { // avoid infinite recursion - const type_name& t = itr->second; + const std::string_view& t = itr->second; itr = typedefs.find( t ); if( itr == typedefs.end() ) return t; } @@ -279,7 +280,7 @@ namespace eosio { namespace chain { return type; } - void abi_serializer::_binary_to_variant( const type_name& type, fc::datastream& stream, + void abi_serializer::_binary_to_variant( const std::string_view& type, fc::datastream& stream, fc::mutable_variant_object& obj, impl::binary_to_variant_context& ctx )const { auto h = ctx.enter_scope(); @@ -312,11 +313,11 @@ namespace eosio { namespace chain { } } - fc::variant abi_serializer::_binary_to_variant( const type_name& type, fc::datastream& stream, + fc::variant abi_serializer::_binary_to_variant( const std::string_view& type, fc::datastream& stream, impl::binary_to_variant_context& ctx )const { auto h = ctx.enter_scope(); - type_name rtype = resolve_type(type); + auto rtype = resolve_type(type); auto ftype = fundamental_type(rtype); auto btype = built_in_types.find(ftype ); if( btype != built_in_types.end() ) { @@ -324,7 +325,7 @@ namespace eosio { namespace chain { return btype->second.first(stream, is_array(rtype), is_optional(rtype)); } EOS_RETHROW_EXCEPTIONS( unpack_exception, "Unable to unpack ${class} type '${type}' while processing '${p}'", ("class", is_array(rtype) ? "array of built-in" : is_optional(rtype) ? "optional of built-in" : "built-in") - ("type", ftype)("p", ctx.get_path_string()) ) + ("type", impl::limit_size(ftype))("p", ctx.get_path_string()) ) } if ( is_array(rtype) ) { ctx.hint_array_type_if_in_array(); @@ -377,26 +378,26 @@ namespace eosio { namespace chain { return fc::variant( std::move(mvo) ); } - fc::variant abi_serializer::_binary_to_variant( const type_name& type, const bytes& binary, impl::binary_to_variant_context& ctx )const + fc::variant abi_serializer::_binary_to_variant( const std::string_view& type, const bytes& binary, impl::binary_to_variant_context& ctx )const { auto h = ctx.enter_scope(); fc::datastream ds( binary.data(), binary.size() ); return _binary_to_variant(type, ds, ctx); } - fc::variant abi_serializer::binary_to_variant( const type_name& type, const bytes& binary, const fc::microseconds& max_serialization_time, bool short_path )const { + fc::variant abi_serializer::binary_to_variant( const std::string_view& type, const bytes& binary, const fc::microseconds& max_serialization_time, bool short_path )const { impl::binary_to_variant_context ctx(*this, max_serialization_time, type); ctx.short_path = short_path; return _binary_to_variant(type, binary, ctx); } - fc::variant abi_serializer::binary_to_variant( const type_name& type, fc::datastream& binary, const fc::microseconds& max_serialization_time, bool short_path )const { + fc::variant abi_serializer::binary_to_variant( const std::string_view& type, fc::datastream& binary, const fc::microseconds& max_serialization_time, bool short_path )const { impl::binary_to_variant_context ctx(*this, max_serialization_time, type); ctx.short_path = short_path; return _binary_to_variant(type, binary, ctx); } - void abi_serializer::_variant_to_binary( const type_name& type, const fc::variant& var, fc::datastream& ds, impl::variant_to_binary_context& ctx )const + void abi_serializer::_variant_to_binary( const std::string_view& type, const fc::variant& var, fc::datastream& ds, impl::variant_to_binary_context& ctx )const { try { auto h = ctx.enter_scope(); auto rtype = resolve_type(type); @@ -493,9 +494,9 @@ namespace eosio { namespace chain { } else { EOS_THROW( invalid_type_inside_abi, "Unknown type ${type}", ("type",ctx.maybe_shorten(type)) ); } - } FC_CAPTURE_AND_RETHROW( (type)(var) ) } + } FC_CAPTURE_AND_RETHROW() } - bytes abi_serializer::_variant_to_binary( const type_name& type, const fc::variant& var, impl::variant_to_binary_context& ctx )const + bytes abi_serializer::_variant_to_binary( const std::string_view& type, const fc::variant& var, impl::variant_to_binary_context& ctx )const { try { auto h = ctx.enter_scope(); if( !_is_type(type, ctx) ) { @@ -507,15 +508,15 @@ namespace eosio { namespace chain { _variant_to_binary(type, var, ds, ctx); temp.resize(ds.tellp()); return temp; - } FC_CAPTURE_AND_RETHROW( (type)(var) ) } + } FC_CAPTURE_AND_RETHROW() } - bytes abi_serializer::variant_to_binary( const type_name& type, const fc::variant& var, const fc::microseconds& max_serialization_time, bool short_path )const { + bytes abi_serializer::variant_to_binary( const std::string_view& type, const fc::variant& var, const fc::microseconds& max_serialization_time, bool short_path )const { impl::variant_to_binary_context ctx(*this, max_serialization_time, type); ctx.short_path = short_path; return _variant_to_binary(type, var, ctx); } - void abi_serializer::variant_to_binary( const type_name& type, const fc::variant& var, fc::datastream& ds, const fc::microseconds& max_serialization_time, bool short_path )const { + void abi_serializer::variant_to_binary( const std::string_view& type, const fc::variant& var, fc::datastream& ds, const fc::microseconds& max_serialization_time, bool short_path )const { impl::variant_to_binary_context ctx(*this, max_serialization_time, type); ctx.short_path = short_path; _variant_to_binary(type, var, ds, ctx); @@ -561,7 +562,7 @@ namespace eosio { namespace chain { return {std::move(callback)}; } - void abi_traverse_context_with_path::set_path_root( const type_name& type ) { + void abi_traverse_context_with_path::set_path_root( const std::string_view& type ) { auto rtype = abis.resolve_type(type); if( abis.is_array(rtype) ) { @@ -627,7 +628,7 @@ namespace eosio { namespace chain { return (*str == 0) ? 0 : const_strlen(str + 1) + 1; } - void output_name( std::ostream& s, const string& str, bool shorten, size_t max_length = 64 ) { + void output_name( std::ostream& s, const string_view& str, bool shorten, size_t max_length = 64 ) { constexpr size_t min_num_characters_at_ends = 4; constexpr size_t preferred_num_tail_end_characters = 6; constexpr const char* fill_in = "..."; @@ -790,15 +791,21 @@ namespace eosio { namespace chain { return visitor.s.str(); } - string abi_traverse_context_with_path::maybe_shorten( const string& str ) { + string abi_traverse_context_with_path::maybe_shorten( const std::string_view& str ) { if( !short_path ) - return str; + return std::string(str); std::stringstream s; output_name( s, str, true ); return s.str(); } + string limit_size( const std::string_view& str ) { + std::stringstream s; + output_name( s, str, false ); + return s.str(); + } + fc::scoped_exit> variant_to_binary_context::disallow_extensions_unless( bool condition ) { std::function callback = [old_allow_extensions=allow_extensions, this](){ allow_extensions = old_allow_extensions; diff --git a/libraries/chain/include/eosio/chain/abi_serializer.hpp b/libraries/chain/include/eosio/chain/abi_serializer.hpp index 79c6d472942..9c3ba8368de 100644 --- a/libraries/chain/include/eosio/chain/abi_serializer.hpp +++ b/libraries/chain/include/eosio/chain/abi_serializer.hpp @@ -36,28 +36,31 @@ struct abi_serializer { abi_serializer( const abi_def& abi, const fc::microseconds& max_serialization_time ); void set_abi(const abi_def& abi, const fc::microseconds& max_serialization_time); - type_name resolve_type(const type_name& t)const; - bool is_array(const type_name& type)const; - bool is_optional(const type_name& type)const; - bool is_type(const type_name& type, const fc::microseconds& max_serialization_time)const; - bool is_builtin_type(const type_name& type)const; - bool is_integer(const type_name& type) const; - int get_integer_size(const type_name& type) const; - bool is_struct(const type_name& type)const; - type_name fundamental_type(const type_name& type)const; - - const struct_def& get_struct(const type_name& type)const; + /// @return string_view of `t` or internal string type + std::string_view resolve_type(const std::string_view& t)const; + bool is_array(const std::string_view& type)const; + bool is_optional(const std::string_view& type)const; + bool is_type(const std::string_view& type, const fc::microseconds& max_serialization_time)const; + bool is_builtin_type(const std::string_view& type)const; + bool is_integer(const std::string_view& type) const; + int get_integer_size(const std::string_view& type) const; + bool is_struct(const std::string_view& type)const; + + /// @return string_view of `type` + std::string_view fundamental_type(const std::string_view& type)const; + + const struct_def& get_struct(const std::string_view& type)const; type_name get_action_type(name action)const; type_name get_table_type(name action)const; optional get_error_message( uint64_t error_code )const; - fc::variant binary_to_variant( const type_name& type, const bytes& binary, const fc::microseconds& max_serialization_time, bool short_path = false )const; - fc::variant binary_to_variant( const type_name& type, fc::datastream& binary, const fc::microseconds& max_serialization_time, bool short_path = false )const; + fc::variant binary_to_variant( const std::string_view& type, const bytes& binary, const fc::microseconds& max_serialization_time, bool short_path = false )const; + fc::variant binary_to_variant( const std::string_view& type, fc::datastream& binary, const fc::microseconds& max_serialization_time, bool short_path = false )const; - bytes variant_to_binary( const type_name& type, const fc::variant& var, const fc::microseconds& max_serialization_time, bool short_path = false )const; - void variant_to_binary( const type_name& type, const fc::variant& var, fc::datastream& ds, const fc::microseconds& max_serialization_time, bool short_path = false )const; + bytes variant_to_binary( const std::string_view& type, const fc::variant& var, const fc::microseconds& max_serialization_time, bool short_path = false )const; + void variant_to_binary( const std::string_view& type, const fc::variant& var, fc::datastream& ds, const fc::microseconds& max_serialization_time, bool short_path = false )const; template static void to_variant( const T& o, fc::variant& vo, Resolver resolver, const fc::microseconds& max_serialization_time ); @@ -91,27 +94,27 @@ struct abi_serializer { private: - map typedefs; - map structs; - map actions; - map tables; - map error_messages; - map variants; + map> typedefs; + map> structs; + map actions; + map tables; + map error_messages; + map> variants; - map> built_in_types; + map, std::less<>> built_in_types; void configure_built_in_types(); - fc::variant _binary_to_variant( const type_name& type, const bytes& binary, impl::binary_to_variant_context& ctx )const; - fc::variant _binary_to_variant( const type_name& type, fc::datastream& binary, impl::binary_to_variant_context& ctx )const; - void _binary_to_variant( const type_name& type, fc::datastream& stream, + fc::variant _binary_to_variant( const std::string_view& type, const bytes& binary, impl::binary_to_variant_context& ctx )const; + fc::variant _binary_to_variant( const std::string_view& type, fc::datastream& binary, impl::binary_to_variant_context& ctx )const; + void _binary_to_variant( const std::string_view& type, fc::datastream& stream, fc::mutable_variant_object& obj, impl::binary_to_variant_context& ctx )const; - bytes _variant_to_binary( const type_name& type, const fc::variant& var, impl::variant_to_binary_context& ctx )const; - void _variant_to_binary( const type_name& type, const fc::variant& var, + bytes _variant_to_binary( const std::string_view& type, const fc::variant& var, impl::variant_to_binary_context& ctx )const; + void _variant_to_binary( const std::string_view& type, const fc::variant& var, fc::datastream& ds, impl::variant_to_binary_context& ctx )const; - static type_name _remove_bin_extension(const type_name& type); - bool _is_type( const type_name& type, impl::abi_traverse_context& ctx )const; + static std::string_view _remove_bin_extension(const std::string_view& type); + bool _is_type( const std::string_view& type, impl::abi_traverse_context& ctx )const; void validate( impl::abi_traverse_context& ctx )const; @@ -184,25 +187,25 @@ namespace impl { using path_item = static_variant; struct abi_traverse_context_with_path : public abi_traverse_context { - abi_traverse_context_with_path( const abi_serializer& abis, fc::microseconds max_serialization_time, const type_name& type ) + abi_traverse_context_with_path( const abi_serializer& abis, fc::microseconds max_serialization_time, const std::string_view& type ) : abi_traverse_context( max_serialization_time ), abis(abis) { set_path_root(type); } - abi_traverse_context_with_path( const abi_serializer& abis, fc::microseconds max_serialization_time, fc::time_point deadline, const type_name& type ) + abi_traverse_context_with_path( const abi_serializer& abis, fc::microseconds max_serialization_time, fc::time_point deadline, const std::string_view& type ) : abi_traverse_context( max_serialization_time, deadline ), abis(abis) { set_path_root(type); } - abi_traverse_context_with_path( const abi_serializer& abis, const abi_traverse_context& ctx, const type_name& type ) + abi_traverse_context_with_path( const abi_serializer& abis, const abi_traverse_context& ctx, const std::string_view& type ) : abi_traverse_context(ctx), abis(abis) { set_path_root(type); } - void set_path_root( const type_name& type ); + void set_path_root( const std::string_view& type ); fc::scoped_exit> push_to_path( const path_item& item ); @@ -213,7 +216,7 @@ namespace impl { string get_path_string()const; - string maybe_shorten( const string& str ); + string maybe_shorten( const std::string_view& str ); protected: const abi_serializer& abis; @@ -238,6 +241,9 @@ namespace impl { bool allow_extensions = true; }; + /// limits the string size to default max_length of output_name + string limit_size( const std::string_view& str ); + /** * Determine if a type contains ABI related info, perhaps deeply nested * @tparam T - the type to check diff --git a/libraries/fc b/libraries/fc index 0784e1b5737..25a24852325 160000 --- a/libraries/fc +++ b/libraries/fc @@ -1 +1 @@ -Subproject commit 0784e1b573739e7769c07d7509184c5005d7a2d4 +Subproject commit 25a24852325f9f4c8510e671d351af007fae3713 diff --git a/plugins/http_plugin/http_plugin.cpp b/plugins/http_plugin/http_plugin.cpp index cb3b663f226..ff7036f7d86 100644 --- a/plugins/http_plugin/http_plugin.cpp +++ b/plugins/http_plugin/http_plugin.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -140,7 +141,7 @@ namespace eosio { string access_control_allow_headers; string access_control_max_age; bool access_control_allow_credentials = false; - size_t max_body_size; + size_t max_body_size{1024*1024}; websocket_server_type server; @@ -148,6 +149,7 @@ namespace eosio { optional thread_pool; std::atomic bytes_in_flight{0}; size_t max_bytes_in_flight = 0; + fc::microseconds max_response_time{30*1000}; optional https_listen_endpoint; string https_cert_chain; @@ -219,6 +221,7 @@ namespace eosio { template static void handle_exception(typename websocketpp::server::connection_ptr con) { string err = "Internal Service error, http: "; + const auto deadline = fc::time_point::now() + fc::exception::format_time_limit; try { con->set_status( websocketpp::http::status_code::internal_server_error ); try { @@ -227,21 +230,28 @@ namespace eosio { err += e.to_detail_string(); elog( "${e}", ("e", err)); error_results results{websocketpp::http::status_code::internal_server_error, - "Internal Service Error", error_results::error_info(e, verbose_http_errors )}; - con->set_body( fc::json::to_string( results )); + "Internal Service Error", error_results::error_info( e, verbose_http_errors )}; + con->set_body( fc::json::to_string( results, deadline )); } catch (const std::exception& e) { err += e.what(); elog( "${e}", ("e", err)); error_results results{websocketpp::http::status_code::internal_server_error, - "Internal Service Error", error_results::error_info(fc::exception( FC_LOG_MESSAGE( error, e.what())), verbose_http_errors )}; - con->set_body( fc::json::to_string( results )); + "Internal Service Error", + error_results::error_info( fc::exception( FC_LOG_MESSAGE( error, e.what())), + verbose_http_errors )}; + con->set_body( fc::json::to_string( results, deadline )); } catch (...) { err += "Unknown Exception"; error_results results{websocketpp::http::status_code::internal_server_error, "Internal Service Error", - error_results::error_info(fc::exception( FC_LOG_MESSAGE( error, "Unknown Exception" )), verbose_http_errors )}; - con->set_body( fc::json::to_string( results )); + error_results::error_info( + fc::exception( FC_LOG_MESSAGE( error, "Unknown Exception" )), + verbose_http_errors )}; + con->set_body( fc::json::to_string( results, deadline )); } + } catch (fc::timeout_exception& e) { + con->set_body( R"xxx({"message": "Internal Server Error"})xxx" ); + elog( "Timeout exception ${te} attempting to handle exception: ${e}", ("te", e.to_detail_string())("e", err) ); } catch (...) { con->set_body( R"xxx({"message": "Internal Server Error"})xxx" ); std::cerr << "Exception attempting to handle exception: " << err << std::endl; @@ -262,6 +272,18 @@ namespace eosio { return true; } + template + bool verify_max_bytes_in_flight( const T& con ) { + if( bytes_in_flight > max_bytes_in_flight ) { + dlog( "503 - too many bytes in flight: ${bytes}", ("bytes", bytes_in_flight.load()) ); + error_results results{websocketpp::http::status_code::too_many_requests, "Busy", error_results::error_info()}; + con->set_body( fc::json::to_string( results, fc::time_point::maximum() )); + con->set_status( websocketpp::http::status_code::too_many_requests ); + return false; + } + return true; + } + template void handle_http_request(typename websocketpp::server::connection_ptr con) { try { @@ -290,13 +312,7 @@ namespace eosio { con->append_header( "Content-type", "application/json" ); - if( bytes_in_flight > max_bytes_in_flight ) { - dlog( "503 - too many bytes in flight: ${bytes}", ("bytes", bytes_in_flight.load()) ); - error_results results{websocketpp::http::status_code::too_many_requests, "Busy", error_results::error_info()}; - con->set_body( fc::json::to_string( results )); - con->set_status( websocketpp::http::status_code::too_many_requests ); - return; - } + if( !verify_max_bytes_in_flight( con ) ) return; std::string body = con->get_request_body(); std::string resource = con->get_uri()->get_resource(); @@ -305,34 +321,57 @@ namespace eosio { con->defer_http_response(); bytes_in_flight += body.size(); app().post( appbase::priority::low, - [&ioc = thread_pool->get_executor(), &bytes_in_flight = this->bytes_in_flight, handler_itr, - resource{std::move( resource )}, body{std::move( body )}, con]() { + [&ioc = thread_pool->get_executor(), &bytes_in_flight = this->bytes_in_flight, + handler_itr, this, resource{std::move( resource )}, body{std::move( body )}, con]() mutable { + const size_t body_size = body.size(); + if( !verify_max_bytes_in_flight( con ) ) { + con->send_http_response(); + bytes_in_flight -= body_size; + return; + } try { - handler_itr->second( resource, body, - [&ioc, &bytes_in_flight, con]( int code, fc::variant response_body ) { - boost::asio::post( ioc, [response_body{std::move( response_body )}, &bytes_in_flight, con, code]() mutable { - std::string json = fc::json::to_string( response_body ); - response_body.clear(); - const size_t json_size = json.size(); - bytes_in_flight += json_size; - con->set_body( std::move( json ) ); - con->set_status( websocketpp::http::status_code::value( code ) ); + handler_itr->second( std::move( resource ), std::move( body ), + [&ioc, &bytes_in_flight, con, this]( int code, fc::variant response_body ) { + size_t response_size = 0; + try { + response_size = fc::raw::pack_size( response_body ); + } catch(...) {} + bytes_in_flight += response_size; + if( !verify_max_bytes_in_flight( con ) ) { con->send_http_response(); - bytes_in_flight -= json_size; - } ); + bytes_in_flight -= response_size; + } else { + boost::asio::post( ioc, + [response_body{std::move( response_body )}, response_size, &bytes_in_flight, + con, code, max_response_time=max_response_time]() mutable { + std::string json; + try { + json = fc::json::to_string( response_body, fc::time_point::now() + max_response_time ); + con->set_body( std::move( json ) ); + con->set_status( websocketpp::http::status_code::value( code ) ); + } catch( ... ) { + handle_exception( con ); + } + response_body.clear(); + const size_t json_size = json.size(); + bytes_in_flight += json_size; + con->send_http_response(); + bytes_in_flight -= (json_size + response_size); + } ); + } }); - bytes_in_flight -= body.size(); } catch( ... ) { handle_exception( con ); con->send_http_response(); } + bytes_in_flight -= body_size; } ); } else { dlog( "404 - not found: ${ep}", ("ep", resource)); error_results results{websocketpp::http::status_code::not_found, "Not Found", error_results::error_info(fc::exception( FC_LOG_MESSAGE( error, "Unknown Endpoint" )), verbose_http_errors )}; - con->set_body( fc::json::to_string( results )); + con->set_body( fc::json::to_string( results, fc::time_point::now() + max_response_time )); con->set_status( websocketpp::http::status_code::not_found ); } } catch( ... ) { @@ -435,6 +474,8 @@ namespace eosio { "The maximum body size in bytes allowed for incoming RPC requests") ("http-max-bytes-in-flight-mb", bpo::value()->default_value(500), "Maximum size in megabytes http_plugin should use for processing http requests. 503 error response when exceeded." ) + ("http-max-response-time-ms", bpo::value()->default_value(30), + "Maximum time for processing a request.") ("verbose-http-errors", bpo::bool_switch()->default_value(false), "Append the error log to HTTP responses") ("http-validate-host", boost::program_options::value()->default_value(true), @@ -522,6 +563,7 @@ namespace eosio { "http-threads ${num} must be greater than 0", ("num", my->thread_pool_size)); my->max_bytes_in_flight = options.at( "http-max-bytes-in-flight-mb" ).as() * 1024 * 1024; + my->max_response_time = fc::microseconds( options.at("http-max-response-time-ms").as() * 1000 ); //watch out for the returns above when adding new code here } FC_LOG_AND_RETHROW() diff --git a/plugins/mongo_db_plugin/mongo_db_plugin.cpp b/plugins/mongo_db_plugin/mongo_db_plugin.cpp index e05f1a01815..ffb07b81f4f 100644 --- a/plugins/mongo_db_plugin/mongo_db_plugin.cpp +++ b/plugins/mongo_db_plugin/mongo_db_plugin.cpp @@ -763,7 +763,7 @@ void mongo_db_plugin_impl::_process_accepted_transaction( const chain::transacti trans_doc.append( kvp( "trx_id", trx_id_str ) ); auto v = to_variant_with_abi( trx ); - string trx_json = fc::json::to_string( v ); + string trx_json = fc::json::to_string( v, fc::time_point::maximum() ); try { const auto& trx_value = bsoncxx::from_json( trx_json ); @@ -782,12 +782,12 @@ void mongo_db_plugin_impl::_process_accepted_transaction( const chain::transacti string signing_keys_json; if( t->signing_keys_future.valid() ) { - signing_keys_json = fc::json::to_string( std::get<2>( t->signing_keys_future.get() ) ); + signing_keys_json = fc::json::to_string( std::get<2>( t->signing_keys_future.get() ), fc::time_point::maximum() ); } else { flat_set keys; trx.get_signature_keys( *chain_id, fc::time_point::maximum(), keys, false ); if( !keys.empty() ) { - signing_keys_json = fc::json::to_string( keys ); + signing_keys_json = fc::json::to_string( keys, fc::time_point::maximum() ); } } @@ -844,7 +844,7 @@ mongo_db_plugin_impl::add_action_trace( mongocxx::bulk_write& bulk_action_traces action_traces_doc.append( kvp( "_id", make_custom_oid() ) ); auto v = to_variant_with_abi( atrace ); - string json = fc::json::to_string( v ); + string json = fc::json::to_string( v, fc::time_point::maximum() ); try { const auto& value = bsoncxx::from_json( json ); action_traces_doc.append( bsoncxx::builder::concatenate_doc{value.view()} ); @@ -904,7 +904,7 @@ void mongo_db_plugin_impl::_process_applied_transaction( const chain::transactio if( store_transaction_traces && write_ttrace ) { try { auto v = to_variant_with_abi( *t ); - string json = fc::json::to_string( v ); + string json = fc::json::to_string( v, fc::time_point::maximum() ); try { const auto& value = bsoncxx::from_json( json ); trans_traces_doc.append( bsoncxx::builder::concatenate_doc{value.view()} ); @@ -973,7 +973,7 @@ void mongo_db_plugin_impl::_process_accepted_block( const chain::block_state_ptr const chain::block_header_state& bhs = *bs; - auto json = fc::json::to_string( bhs ); + auto json = fc::json::to_string( bhs, fc::time_point::maximum() ); try { const auto& value = bsoncxx::from_json( json ); block_state_doc.append( kvp( "block_header_state", value ) ); @@ -1013,7 +1013,7 @@ void mongo_db_plugin_impl::_process_accepted_block( const chain::block_state_ptr kvp( "block_id", block_id_str ) ); auto v = to_variant_with_abi( *bs->block ); - auto json = fc::json::to_string( v ); + auto json = fc::json::to_string( v, fc::time_point::maximum() ); try { const auto& value = bsoncxx::from_json( json ); block_doc.append( kvp( "block", value ) ); @@ -1321,7 +1321,7 @@ void mongo_db_plugin_impl::update_account(const chain::action& act) } if( account ) { abi_def abi_def = fc::raw::unpack( setabi.abi ); - const string json_str = fc::json::to_string( abi_def ); + const string json_str = fc::json::to_string( abi_def, fc::time_point::maximum() ); try{ auto update_from = make_document( diff --git a/programs/cleos/help_text.cpp.in b/programs/cleos/help_text.cpp.in index a133e17cade..b97784cef29 100644 --- a/programs/cleos/help_text.cpp.in +++ b/programs/cleos/help_text.cpp.in @@ -290,7 +290,7 @@ bool print_recognized_errors(const fc::exception& e, const bool verbose_errors) explanation += "\n" + localized_with_variant(log.get_format().data(), log.get_data()); } else if (log.get_data().size() > 0 && verbose_errors) { // Show data-only log only if verbose_errors option is enabled - explanation += "\n" + fc::json::to_string(log.get_data()); + explanation += "\n" + fc::json::to_string(log.get_data(), fc::time_point::maximum()); } // Check if there's stack trace to be added if (!log.get_context().get_method().empty() && verbose_errors) { diff --git a/programs/cleos/httpc.cpp b/programs/cleos/httpc.cpp index 7d9326b9ed7..96cc13975c3 100644 --- a/programs/cleos/httpc.cpp +++ b/programs/cleos/httpc.cpp @@ -189,7 +189,7 @@ namespace eosio { namespace client { namespace http { bool print_response ) { std::string postjson; if( !postdata.is_null() ) { - postjson = print_request ? fc::json::to_pretty_string( postdata ) : fc::json::to_string( postdata ); + postjson = print_request ? fc::json::to_pretty_string( postdata ) : fc::json::to_string( postdata, fc::time_point::maximum() ); } const auto& url = cp.url; diff --git a/programs/cleos/main.cpp b/programs/cleos/main.cpp index 104fdec9989..80f226c782e 100644 --- a/programs/cleos/main.cpp +++ b/programs/cleos/main.cpp @@ -360,7 +360,7 @@ void print_action( const fc::variant& at ) { const auto& act = at["act"].get_object(); auto code = act["account"].as_string(); auto func = act["name"].as_string(); - auto args = fc::json::to_string( act["data"] ); + auto args = fc::json::to_string( act["data"], fc::time_point::maximum() ); auto console = at["console"].as_string(); /* @@ -2796,7 +2796,7 @@ int main( int argc, char** argv ) { args = fc::json::to_pretty_string( act["data"] ); } else { - args = fc::json::to_string( act["data"] ); + args = fc::json::to_string( act["data"], fc::time_point::maximum() ); if( !fullact ) { args = args.substr(0,60) + "..."; } diff --git a/programs/eosio-blocklog/main.cpp b/programs/eosio-blocklog/main.cpp index 3a674bf31ee..8e4d78723d6 100644 --- a/programs/eosio-blocklog/main.cpp +++ b/programs/eosio-blocklog/main.cpp @@ -103,7 +103,7 @@ void blocklog::read_log() { (pretty_output.get_object()); fc::variant v(std::move(enhanced_object)); if (no_pretty_print) - fc::json::to_stream(*out, v, fc::json::stringify_large_ints_and_doubles); + fc::json::to_stream(*out, v, fc::time_point::maximum(), fc::json::stringify_large_ints_and_doubles); else *out << fc::json::to_pretty_string(v) << "\n"; }; diff --git a/programs/eosio-launcher/main.cpp b/programs/eosio-launcher/main.cpp index b21f45df74d..93cb9b1af64 100644 --- a/programs/eosio-launcher/main.cpp +++ b/programs/eosio-launcher/main.cpp @@ -1199,7 +1199,7 @@ launcher_def::write_logging_config_file(tn_node_def &node) { if( gelf_enabled ) pp.appenders.push_back( "net" ); log_config.loggers.emplace_back( pp ); - auto str = fc::json::to_pretty_string( log_config, fc::json::stringify_large_ints_and_doubles ); + auto str = fc::json::to_pretty_string( log_config, fc::time_point::maximum(), fc::json::stringify_large_ints_and_doubles ); cfg.write( str.c_str(), str.size() ); cfg.close(); } @@ -1249,7 +1249,7 @@ launcher_def::write_setprods_file() { if (p.producer_name != "eosio") no_bios.schedule.push_back(p); } - auto str = fc::json::to_pretty_string( no_bios, fc::json::stringify_large_ints_and_doubles ); + auto str = fc::json::to_pretty_string( no_bios, fc::time_point::maximum(), fc::json::stringify_large_ints_and_doubles ); psfile.write( str.c_str(), str.size() ); psfile.close(); } diff --git a/tests/Cluster.py b/tests/Cluster.py index ae5a2f19ae4..699858e355f 100644 --- a/tests/Cluster.py +++ b/tests/Cluster.py @@ -219,7 +219,7 @@ def launch(self, pnodes=1, unstartedNodes=0, totalNodes=1, prodCount=1, topo="me if self.staging: cmdArr.append("--nogen") - nodeosArgs="--max-transaction-time -1 --abi-serializer-max-time-ms 990000 --filter-on \"*\" --p2p-max-nodes-per-host %d" % (totalNodes) + nodeosArgs="--max-transaction-time -1 --http-max-response-time-ms 9999 --abi-serializer-max-time-ms 990000 --filter-on \"*\" --p2p-max-nodes-per-host %d" % (totalNodes) if not self.walletd: nodeosArgs += " --plugin eosio::wallet_api_plugin" if self.enableMongo: diff --git a/tests/Node.py b/tests/Node.py index 3ceec3a207e..f1a39dffc64 100644 --- a/tests/Node.py +++ b/tests/Node.py @@ -925,7 +925,7 @@ def publishContract(self, account, contractDir, wasmFile, abiFile, waitForTransB if not shouldFail: end=time.perf_counter() msg=ex.output.decode("utf-8") - Utils.Print("ERROR: Exception during code hash retrieval. cmd Duration: %.3f sec. %s" % (end-start, msg)) + Utils.Print("ERROR: Exception during set contract. cmd Duration: %.3f sec. %s" % (end-start, msg)) return None else: retMap={} diff --git a/unittests/abi_tests.cpp b/unittests/abi_tests.cpp index 7e6c84fb6f2..dab7532b28c 100644 --- a/unittests/abi_tests.cpp +++ b/unittests/abi_tests.cpp @@ -39,7 +39,7 @@ fc::variant verify_byte_round_trip_conversion( const abi_serializer& abis, const auto var2 = abis.binary_to_variant(type, bytes, max_serialization_time); - std::string r = fc::json::to_string(var2); + std::string r = fc::json::to_string(var2, fc::time_point::now() + max_serialization_time); auto bytes2 = abis.variant_to_binary(type, var2, max_serialization_time); @@ -54,7 +54,7 @@ void verify_round_trip_conversion( const abi_serializer& abis, const type_name& auto bytes = abis.variant_to_binary(type, var, max_serialization_time); BOOST_REQUIRE_EQUAL(fc::to_hex(bytes), hex); auto var2 = abis.binary_to_variant(type, bytes, max_serialization_time); - BOOST_REQUIRE_EQUAL(fc::json::to_string(var2), expected_json); + BOOST_REQUIRE_EQUAL(fc::json::to_string(var2, fc::time_point::now() + max_serialization_time), expected_json); auto bytes2 = abis.variant_to_binary(type, var2, max_serialization_time); BOOST_REQUIRE_EQUAL(fc::to_hex(bytes2), hex); } @@ -84,7 +84,7 @@ fc::variant verify_type_round_trip_conversion( const abi_serializer& abis, const fc::variant var2; abi_serializer::to_variant(obj, var2, get_resolver(), max_serialization_time); - std::string r = fc::json::to_string(var2); + std::string r = fc::json::to_string(var2, fc::time_point::now() + max_serialization_time); auto bytes2 = abis.variant_to_binary(type, var2, max_serialization_time); diff --git a/unittests/wasm_tests.cpp b/unittests/wasm_tests.cpp index c29ec8da8c4..a0744439ccc 100644 --- a/unittests/wasm_tests.cpp +++ b/unittests/wasm_tests.cpp @@ -996,7 +996,7 @@ BOOST_FIXTURE_TEST_CASE(eosio_abi, TESTER) try { // see abi_serializer::to_abi() abi_serializer::to_variant(*result, pretty_output, get_resolver(), abi_serializer_max_time); - BOOST_TEST(fc::json::to_string(pretty_output).find("newaccount") != std::string::npos); + BOOST_TEST(fc::json::to_string(pretty_output, fc::time_point::now() + abi_serializer_max_time).find("newaccount") != std::string::npos); produce_block(); } FC_LOG_AND_RETHROW() From c52114393e5431b3e26ca752fe0d669fa6b4779a Mon Sep 17 00:00:00 2001 From: arhag Date: Fri, 10 Jan 2020 13:15:52 -0500 Subject: [PATCH 15/15] bump version to 1.8.9 --- CMakeLists.txt | 2 +- README.md | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4cfe302130c..8b923b39783 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -28,7 +28,7 @@ set( CXX_STANDARD_REQUIRED ON) set(VERSION_MAJOR 1) set(VERSION_MINOR 8) -set(VERSION_PATCH 8) +set(VERSION_PATCH 9) #set(VERSION_SUFFIX develop) if(VERSION_SUFFIX) diff --git a/README.md b/README.md index 9d2c44817ec..1723638aa15 100644 --- a/README.md +++ b/README.md @@ -44,13 +44,13 @@ $ brew remove eosio #### Ubuntu 18.04 Package Install ```sh -$ wget https://github.com/eosio/eos/releases/download/v1.8.8/eosio_1.8.8-1-ubuntu-18.04_amd64.deb -$ sudo apt install ./eosio_1.8.8-1-ubuntu-18.04_amd64.deb +$ wget https://github.com/eosio/eos/releases/download/v1.8.9/eosio_1.8.9-1-ubuntu-18.04_amd64.deb +$ sudo apt install ./eosio_1.8.9-1-ubuntu-18.04_amd64.deb ``` #### Ubuntu 16.04 Package Install ```sh -$ wget https://github.com/eosio/eos/releases/download/v1.8.8/eosio_1.8.8-1-ubuntu-16.04_amd64.deb -$ sudo apt install ./eosio_1.8.8-1-ubuntu-16.04_amd64.deb +$ wget https://github.com/eosio/eos/releases/download/v1.8.9/eosio_1.8.9-1-ubuntu-16.04_amd64.deb +$ sudo apt install ./eosio_1.8.9-1-ubuntu-16.04_amd64.deb ``` #### Ubuntu Package Uninstall ```sh @@ -58,8 +58,8 @@ $ sudo apt remove eosio ``` #### Centos RPM Package Install ```sh -$ wget https://github.com/eosio/eos/releases/download/v1.8.8/eosio-1.8.8-1.el7.x86_64.rpm -$ sudo yum install ./eosio-1.8.8-1.el7.x86_64.rpm +$ wget https://github.com/eosio/eos/releases/download/v1.8.9/eosio-1.8.9-1.el7.x86_64.rpm +$ sudo yum install ./eosio-1.8.9-1.el7.x86_64.rpm ``` #### Centos RPM Package Uninstall ```sh