l0rinc · l0rinc · Feb 27, 2025 · Mar 25, 2025 · Mar 25, 2025 · Feb 27, 2025
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -358,7 +358,7 @@ jobs:
           ./src/univalue/unitester.exe
 
       - name: Run benchmarks
-        run: ./bin/bench_bitcoin.exe -sanity-check -priority-level=high
+        run: ./bin/bench_bitcoin.exe -sanity-check
 
       - name: Adjust paths in test/config.ini
         shell: pwsh

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -154,7 +154,7 @@ endif()
 cmake_dependent_option(WITH_DBUS "Enable DBus support." ON "CMAKE_SYSTEM_NAME STREQUAL \"Linux\" AND BUILD_GUI" OFF)
 
 option(ENABLE_IPC "Build multiprocess bitcoin-node and bitcoin-gui executables in addition to monolithic bitcoind and bitcoin-qt executables. Requires libmultiprocess library. Experimental." OFF)
-cmake_dependent_option(WITH_EXTERNAL_LIBMULTIPROCESS "Build with external libmultiprocess library instead of with local git subtree when ENABLE_IPC is enabled. This is not normally recommended, but can be useful when cross-compiling or making changes to the upstream project." OFF "ENABLE_IPC" OFF)
+cmake_dependent_option(WITH_EXTERNAL_LIBMULTIPROCESS "Build with external libmultiprocess library instead of with local git subtree when ENABLE_IPC is enabled. This is not normally recommended, but can be useful for developing libmultiprocess itself." OFF "ENABLE_IPC" OFF)
 if(ENABLE_IPC AND WITH_EXTERNAL_LIBMULTIPROCESS)
   find_package(Libmultiprocess REQUIRED COMPONENTS Lib)
   find_package(LibmultiprocessNative REQUIRED COMPONENTS Bin

diff --git a/ci/lint/04_install.sh b/ci/lint/04_install.sh
@@ -12,10 +12,11 @@ pushd "/"
 
 ${CI_RETRY_EXE} apt-get update
 # Lint dependencies:
+# - cargo (used to run the lint tests)
 # - curl/xz-utils (to install shellcheck)
 # - git (used in many lint scripts)
 # - gpg (used by verify-commits)
-${CI_RETRY_EXE} apt-get install -y curl xz-utils git gpg
+${CI_RETRY_EXE} apt-get install -y cargo curl xz-utils git gpg
 
 PYTHON_PATH="/python_build"
 if [ ! -d "${PYTHON_PATH}/bin" ]; then
@@ -35,17 +36,6 @@ export PATH="${PYTHON_PATH}/bin:${PATH}"
 command -v python3
 python3 --version
 
-export LINT_RUNNER_PATH="/lint_test_runner"
-if [ ! -d "${LINT_RUNNER_PATH}" ]; then
-  ${CI_RETRY_EXE} apt-get install -y cargo
-  (
-    cd "/test/lint/test_runner" || exit 1
-    cargo build
-    mkdir -p "${LINT_RUNNER_PATH}"
-    mv target/debug/test_runner "${LINT_RUNNER_PATH}"
-  )
-fi
-
 ${CI_RETRY_EXE} pip3 install \
   codespell==2.2.6 \
   lief==0.13.2 \

diff --git a/ci/lint/06_script.sh b/ci/lint/06_script.sh
@@ -16,7 +16,7 @@ if [ -n "$CIRRUS_PR" ]; then
   fi
 fi
 
-RUST_BACKTRACE=1 "${LINT_RUNNER_PATH}/test_runner"
+RUST_BACKTRACE=1 cargo run --manifest-path "./test/lint/test_runner/Cargo.toml"
 
 if [ "$CIRRUS_REPO_FULL_NAME" = "bitcoin/bitcoin" ] && [ "$CIRRUS_PR" = "" ] ; then
     # Sanity check only the last few commits to get notified of missing sigs,

diff --git a/ci/lint/container-entrypoint.sh b/ci/lint/container-entrypoint.sh
@@ -11,7 +11,6 @@ export LC_ALL=C
 git config --global --add safe.directory /bitcoin
 
 export PATH="/python_build/bin:${PATH}"
-export LINT_RUNNER_PATH="/lint_test_runner"
 
 if [ -z "$1" ]; then
   bash -ic "./ci/lint/06_script.sh"

diff --git a/ci/lint_imagefile b/ci/lint_imagefile
@@ -4,7 +4,7 @@
 
 # See test/lint/README.md for usage.
 
-FROM mirror.gcr.io/debian:bookworm
+FROM mirror.gcr.io/ubuntu:24.04
 
 ENV DEBIAN_FRONTEND=noninteractive
 ENV LC_ALL=C.UTF-8
@@ -13,7 +13,6 @@ COPY ./ci/retry/retry /ci_retry
 COPY ./.python-version /.python-version
 COPY ./ci/lint/container-entrypoint.sh /entrypoint.sh
 COPY ./ci/lint/04_install.sh /install.sh
-COPY ./test/lint/test_runner /test/lint/test_runner
 
 RUN /install.sh && \
   echo 'alias lint="./ci/lint/06_script.sh"' >> ~/.bashrc && \

diff --git a/ci/lint_run.sh b/ci/lint_run.sh
@@ -9,5 +9,4 @@ set -o errexit -o pipefail -o xtrace
 
 # Only used in .cirrus.yml. Refer to test/lint/README.md on how to run locally.
 export PATH="/python_build/bin:${PATH}"
-export LINT_RUNNER_PATH="/lint_test_runner"
 ./ci/lint/06_script.sh
diff --git a/ci/test/00_setup_env_arm.sh b/ci/test/00_setup_env_arm.sh
@@ -10,7 +10,7 @@ export HOST=arm-linux-gnueabihf
 export DPKG_ADD_ARCH="armhf"
 export PACKAGES="python3-zmq g++-arm-linux-gnueabihf busybox libc6:armhf libstdc++6:armhf libfontconfig1:armhf libxcb1:armhf"
 export CONTAINER_NAME=ci_arm_linux
-export CI_IMAGE_NAME_TAG="mirror.gcr.io/ubuntu:noble"  # Check that https://packages.ubuntu.com/noble/g++-arm-linux-gnueabihf (version 13.3, similar to guix) can cross-compile
+export CI_IMAGE_NAME_TAG="mirror.gcr.io/ubuntu:24.04"  # Check that https://packages.ubuntu.com/noble/g++-arm-linux-gnueabihf (version 13.x, similar to guix) can cross-compile
 export CI_IMAGE_PLATFORM="linux/arm64"
 export USE_BUSY_BOX=true
 export RUN_UNIT_TESTS=true

diff --git a/ci/test/00_setup_env_win64.sh b/ci/test/00_setup_env_win64.sh
@@ -7,7 +7,7 @@
 export LC_ALL=C.UTF-8
 
 export CONTAINER_NAME=ci_win64
-export CI_IMAGE_NAME_TAG="mirror.gcr.io/ubuntu:noble"  # Check that g++-mingw-w64-x86-64-posix (version 13.2, similar to guix) can cross-compile
+export CI_IMAGE_NAME_TAG="mirror.gcr.io/ubuntu:24.04"  # Check that https://packages.ubuntu.com/noble/g++-mingw-w64-x86-64-posix (version 13.x, similar to guix) can cross-compile
 export CI_IMAGE_PLATFORM="linux/amd64"
 export HOST=x86_64-w64-mingw32
 export PACKAGES="g++-mingw-w64-x86-64-posix nsis"

diff --git a/doc/multiprocess.md b/doc/multiprocess.md
@@ -35,6 +35,10 @@ The `cmake` build will pick up settings and library locations from the depends d
 
 When cross-compiling and not using depends, native code generation tools from [libmultiprocess](https://github.com/bitcoin-core/libmultiprocess) and [Cap'n Proto](https://capnproto.org/) are required. They can be passed to the cmake build by specifying `-DMPGEN_EXECUTABLE=/path/to/mpgen -DCAPNP_EXECUTABLE=/path/to/capnp -DCAPNPC_CXX_EXECUTABLE=/path/to/capnpc-c++` options.
 
+### External libmultiprocess installation
+
+By default when `-DENABLE_IPC=ON` is enabled, the libmultiprocess sources at [../src/ipc/libmultiprocess/](../src/ipc/libmultiprocess/) are built as part of the bitcoin cmake build, but alternately an external [libmultiprocess](https://github.com/bitcoin-core/libmultiprocess/) cmake package can be used instead by following its [installation instructions](https://github.com/bitcoin-core/libmultiprocess/blob/master/doc/install.md) and specifying `-DWITH_EXTERNAL_LIBMULTIPROCESS=ON` to the bitcoin build, so it will use the external package instead of the sources. This can be useful when making changes to the upstream project. If libmultiprocess is not installed in a default system location it is possible to specify the [`CMAKE_PREFIX_PATH`](https://cmake.org/cmake/help/latest/envvar/CMAKE_PREFIX_PATH.html) environment variable to point to the installation prefix where libmultiprocess is installed.
+
 ## Usage
 
 `bitcoin-node` is a drop-in replacement for `bitcoind`, and `bitcoin-gui` is a drop-in replacement for `bitcoin-qt`, and there are no differences in use or external behavior between the new and old executables. But internally after [#10102](https://github.com/bitcoin/bitcoin/pull/10102), `bitcoin-gui` will spawn a `bitcoin-node` process to run P2P and RPC code, communicating with it across a socket pair, and `bitcoin-node` will spawn `bitcoin-wallet` to run wallet code, also communicating over a socket pair. This will let node, wallet, and GUI code run in separate address spaces for better isolation, and allow future improvements like being able to start and stop components independently on different machines and environments.

diff --git a/doc/zmq.md b/doc/zmq.md
@@ -87,40 +87,69 @@ For instance:
                -zmqpubrawtx=ipc:///tmp/bitcoind.tx.raw \
                -zmqpubhashtxhwm=10000
 
-Each PUB notification has a topic and body, where the header
-corresponds to the notification type. For instance, for the
-notification `-zmqpubhashtx` the topic is `hashtx` (no null
-terminator). These options can also be provided in bitcoin.conf.
+Notification types correspond to message topics (details in next section). For instance,
+for the notification `-zmqpubhashtx` the topic is `hashtx`. These options can also be
+provided in bitcoin.conf.
 
-The topics are:
+### Message format
 
-`sequence`: the body is structured as the following based on the type of message:
+All ZMQ messages share the same structure with three parts: _topic_ string,
+message _body_, and _message sequence number_:
 
-    <32-byte hash>C :                 Blockhash connected
-    <32-byte hash>D :                 Blockhash disconnected
-    <32-byte hash>R<8-byte LE uint> : Transactionhash removed from mempool for non-block inclusion reason
-    <32-byte hash>A<8-byte LE uint> : Transactionhash added mempool
+    | topic     | body                                                 | message sequence number  |
+    |-----------+------------------------------------------------------+--------------------------|
+    | rawtx     | <serialized transaction>                             | <4-byte LE uint>         |
+    | hashtx    | <reversed 32-byte transaction hash>                  | <4-byte LE uint>         |
+    | rawblock  | <serialized block>                                   | <4-byte LE uint>         |
+    | hashblock | <reversed 32-byte block hash>                        | <4-byte LE uint>         |
+    | sequence  | <reversed 32-byte block hash>C                       | <4-byte LE uint>         |
+    | sequence  | <reversed 32-byte block hash>D                       | <4-byte LE uint>         |
+    | sequence  | <reversed 32-byte transaction hash>R<8-byte LE uint> | <4-byte LE uint>         |
+    | sequence  | <reversed 32-byte transaction hash>A<8-byte LE uint> | <4-byte LE uint>         |
 
-Where the 8-byte uints correspond to the mempool sequence number.
+where:
 
-`rawtx`: Notifies about all transactions, both when they are added to mempool or when a new block arrives. This means a transaction could be published multiple times. First, when it enters the mempool and then again in each block that includes it. The messages are ZMQ multipart messages with three parts. The first part is the topic (`rawtx`), the second part is the serialized transaction, and the last part is a sequence number (representing the message count to detect lost messages).
+ - message sequence number represents message count to detect lost messages, distinct for each topic
+ - all transaction and block hashes are in _reversed byte order_ (i. e. with bytes
+   produced by hashing function reversed), the same format as the RPC interface and block
+   explorers use to display transaction and block hashes
 
-    | rawtx | <serialized transaction> | <uint32 sequence number in Little Endian>
+#### rawtx
 
-`hashtx`: Notifies about all transactions, both when they are added to mempool or when a new block arrives. This means a transaction could be published multiple times. First, when it enters the mempool and then again in each block that includes it. The messages are ZMQ multipart messages with three parts. The first part is the topic (`hashtx`), the second part is the 32-byte transaction hash, and the last part is a sequence number (representing the message count to detect lost messages).
+Notifies about all transactions, both when they are added to mempool or when a new block
+arrives. This means a transaction could be published multiple times: first when it enters
+mempool and then again in each block that includes it. The body part of the message is the
+serialized transaction.
 
-    | hashtx | <32-byte transaction hash in Little Endian> | <uint32 sequence number in Little Endian>
+#### hashtx
 
+Notifies about all transactions, both when they are added to mempool or when a new block
+arrives. This means a transaction could be published multiple times: first when it enters
+mempool and then again in each block that includes it. The body part of the mesage is the
+32-byte transaction hash in reversed byte order.
 
-`rawblock`: Notifies when the chain tip is updated. When assumeutxo is in use, this notification will not be issued for historical blocks connected to the background validation chainstate. Messages are ZMQ multipart messages with three parts. The first part is the topic (`rawblock`), the second part is the serialized block, and the last part is a sequence number (representing the message count to detect lost messages).
+#### rawblock
 
-    | rawblock | <serialized block> | <uint32 sequence number in Little Endian>
+Notifies when the chain tip is updated. When assumeutxo is in use, this notification will
+not be issued for historical blocks connected to the background validation chainstate. The
+body part of the message is the serialized block.
 
-`hashblock`: Notifies when the chain tip is updated. When assumeutxo is in use, this notification will not be issued for historical blocks connected to the background validation chainstate. Messages are ZMQ multipart messages with three parts. The first part is the topic (`hashblock`), the second part is the 32-byte block hash, and the last part is a sequence number (representing the message count to detect lost messages).
+#### hashblock
 
-    | hashblock | <32-byte block hash in Little Endian> | <uint32 sequence number in Little Endian>
+Notifies when the chain tip is updated. When assumeutxo is in use, this notification will
+not be issued for historical blocks connected to the background validation chainstate. The
+body part of the message is the 32-byte block hash in reversed byte order.
 
-**_NOTE:_**  Note that the 32-byte hashes are in Little Endian and not in the Big Endian format that the RPC interface and block explorers use to display transaction and block hashes.
+#### sequence
+
+The 8-byte LE uints correspond to _mempool sequence number_ and the types of bodies are:
+
+   - `C` : block with this hash connected
+   - `D` : block with this hash disconnected
+   - `R` : transaction with this hash removed from mempool for non-block inclusion reason
+   - `A` : transaction with this hash added to mempool
+
+### Implementing ZMQ client
 
 ZeroMQ endpoint specifiers for TCP (and others) are documented in the
 [ZeroMQ API](http://api.zeromq.org/4-0:_start).
@@ -138,7 +167,7 @@ operating system configuration and must be configured prior to connection establ
 For example, when running on GNU/Linux, one might use the following
 to lower the keepalive setting to 10 minutes:
 
-sudo sysctl -w net.ipv4.tcp_keepalive_time=600
+    sudo sysctl -w net.ipv4.tcp_keepalive_time=600
 
 Setting the keepalive values appropriately for your operating environment may
 improve connectivity in situations where long-lived connections are silently

diff --git a/src/bench/readwriteblock.cpp b/src/bench/readwriteblock.cpp
@@ -27,7 +27,7 @@ static CBlock CreateTestBlock()
     return block;
 }
 
-static void SaveBlockBench(benchmark::Bench& bench)
+static void WriteBlockBench(benchmark::Bench& bench)
 {
     const auto testing_setup{MakeNoLogFileContext<const TestingSetup>(ChainType::MAIN)};
     auto& blockman{testing_setup->m_node.chainman->m_blockman};
@@ -63,6 +63,6 @@ static void ReadRawBlockBench(benchmark::Bench& bench)
     });
 }
 
-BENCHMARK(SaveBlockBench, benchmark::PriorityLevel::HIGH);
+BENCHMARK(WriteBlockBench, benchmark::PriorityLevel::HIGH);
 BENCHMARK(ReadBlockBench, benchmark::PriorityLevel::HIGH);
 BENCHMARK(ReadRawBlockBench, benchmark::PriorityLevel::HIGH);
diff --git a/src/bench/wallet_migration.cpp b/src/bench/wallet_migration.cpp
@@ -32,6 +32,7 @@ static void WalletMigration(benchmark::Bench& bench)
     std::unique_ptr<CWallet> wallet = std::make_unique<CWallet>(test_setup->m_node.chain.get(), "", CreateMockableWalletDatabase());
     wallet->chainStateFlushed(ChainstateRole::NORMAL, CBlockLocator{});
     LegacyDataSPKM* legacy_spkm = wallet->GetOrCreateLegacyDataSPKM();
+    WalletBatch batch{wallet->GetDatabase()};
 
     // Add watch-only addresses
     std::vector<CScript> scripts_watch_only;
@@ -42,6 +43,7 @@ static void WalletMigration(benchmark::Bench& bench)
         const CScript& script = scripts_watch_only.emplace_back(GetScriptForDestination(dest));
         assert(legacy_spkm->LoadWatchOnly(script));
         assert(wallet->SetAddressBook(dest, strprintf("watch_%d", w), /*purpose=*/std::nullopt));
+        batch.WriteWatchOnly(script, CKeyMetadata());
     }
 
     // Generate transactions and local addresses
@@ -58,6 +60,7 @@ static void WalletMigration(benchmark::Bench& bench)
         mtx.vout.emplace_back(COIN, scripts_watch_only.at(j % NUM_WATCH_ONLY_ADDR));
         mtx.vin.resize(2);
         wallet->AddToWallet(MakeTransactionRef(mtx), TxStateInactive{}, /*update_wtx=*/nullptr, /*fFlushOnClose=*/false, /*rescanning_old_block=*/true);
+        batch.WriteKey(pubkey, key.GetPrivKey(), CKeyMetadata());
     }
 
     bench.epochs(/*numEpochs=*/1).run([&context, &wallet] {