Move build branch to commit 479530dbc PLUS CHERRY PICKS

CHANGELOG.md

@@ -1,6 +1,30 @@
 # Changelog
 
-## 23.4.2
+## 23.4.5
+
+### Breaking Changes
+
+### Additions and Improvements
+
+### Bug Fixes
+ - Remove miner-related option warnings if the change isn't using Ethash consensus algorithm [#5669](https://github.com/hyperledger/besu/pull/5669)
+
+### Download Links
+
+## 23.4.4
+
+### Breaking Changes
+
+### Additions and Improvements
+
+### Bug Fixes
+- Use the node's configuration to determine if DNS enode URLs are allowed in calls to `admin_addPeer` and `admin_removePeer` [#5584](https://github.com/hyperledger/besu/pull/5584)
+
+### Download Links
+
+---
+
+## 23.4.3
 
 ### Breaking Changes
 - Move blockchain related variables in a dedicated storage, to pave the way to future optimizations [#5471](https://github.com/hyperledger/besu/pull/5471). The migration is performed automatically at startup,
@@ -12,15 +36,22 @@ and in case a rollback is needed, before installing a previous version, the migr
 - Add support for Shanghai changes to the GraphQL service [#5496](https://github.com/hyperledger/besu/pull/5496)
 - Unite the tx-pool CLI options under the same Tx Pool Options group in UX. [#5466](https://github.com/hyperledger/besu/issues/5466)
 - Tidy DEBUG logs by moving engine API full logging to TRACE [#5529](https://github.com/hyperledger/besu/pull/5529)
-- remove PoW validation if merge is enabled as it is not needed anymore [#5538](https://github.com/hyperledger/besu/pull/5538)
+- Remove PoW validation if merge is enabled as it is not needed anymore [#5538](https://github.com/hyperledger/besu/pull/5538)
 - Use BlobDB for blockchain storage to reduce initial sync time and write amplification [#5475](https://github.com/hyperledger/besu/pull/5475)
 - Update to Tuweni 2.4.1. [#5513](https://github.com/hyperledger/besu/pull/5513)
+- Add healing flat db mechanism with experimental CLI options `--Xsnapsync-synchronizer-flat-db-healing-enabled=true` [#5319](https://github.com/hyperledger/besu/pull/5319)
 
 ### Bug Fixes
-- check to ensure storage and transactions are not closed prior to reading/writing [#5527](https://github.com/hyperledger/besu/pull/5527) 
+- Fix backwards sync bug where chain is rolled back too far, especially when restarting Nimbus [#5497](https://github.com/hyperledger/besu/pull/5497)
+- Check to ensure storage and transactions are not closed prior to reading/writing [#5527](https://github.com/hyperledger/besu/pull/5527) 
+- Fix the unavailability of account code and storage on GraphQl/Bonsai [#5548](https://github.com/hyperledger/besu/pull/5548)
 
 ### Download Links
 
+## 23.4.2
+
+- Was not released (failed burn-in test)
+
 ---
 
 ## 23.4.1

besu/src/main/java/org/hyperledger/besu/RunnerBuilder.java

@@ -87,6 +87,7 @@
 import org.hyperledger.besu.ethereum.p2p.network.P2PNetwork;
 import org.hyperledger.besu.ethereum.p2p.network.ProtocolManager;
 import org.hyperledger.besu.ethereum.p2p.peers.DefaultPeer;
+import org.hyperledger.besu.ethereum.p2p.peers.EnodeDnsConfiguration;
 import org.hyperledger.besu.ethereum.p2p.permissions.PeerPermissions;
 import org.hyperledger.besu.ethereum.p2p.permissions.PeerPermissionsDenylist;
 import org.hyperledger.besu.ethereum.p2p.rlpx.connections.netty.TLSConfiguration;
@@ -191,6 +192,7 @@ public class RunnerBuilder {
   private JsonRpcIpcConfiguration jsonRpcIpcConfiguration;
   private boolean legacyForkIdEnabled;
   private Optional<Long> rpcMaxLogsRange;
+  private Optional<EnodeDnsConfiguration> enodeDnsConfiguration;
 
   /**
    * Add Vertx.
@@ -584,6 +586,18 @@ public RunnerBuilder rpcMaxLogsRange(final Long rpcMaxLogsRange) {
     return this;
   }
 
+  /**
+   * Add enode DNS configuration
+   *
+   * @param enodeDnsConfiguration the DNS configuration for enodes
+   * @return the runner builder
+   */
+  public RunnerBuilder enodeDnsConfiguration(final EnodeDnsConfiguration enodeDnsConfiguration) {
+    this.enodeDnsConfiguration =
+        enodeDnsConfiguration != null ? Optional.of(enodeDnsConfiguration) : Optional.empty();
+    return this;
+  }
+
   /**
    * Build Runner instance.
    *
@@ -1224,7 +1238,8 @@ private Map<String, JsonRpcMethod> jsonRpcMethods(
                 dataDir,
                 besuController.getProtocolManager().ethContext().getEthPeers(),
                 consensusEngineServer,
-                rpcMaxLogsRange);
+                rpcMaxLogsRange,
+                enodeDnsConfiguration);
     methods.putAll(besuController.getAdditionalJsonRpcMethods(jsonRpcApis));
 
     final var pluginMethods =

besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java

@@ -2133,7 +2133,7 @@ private void issueOptionWarnings() {
             "--remote-connections-max-percentage"));
 
     // Check that block producer options work
-    if (!isMergeEnabled()) {
+    if (!isMergeEnabled() && getActualGenesisConfigOptions().isEthHash()) {
       CommandLineUtils.checkOptionDependencies(
           logger,
           commandLine,
@@ -2144,17 +2144,18 @@ private void issueOptionWarnings() {
               "--min-gas-price",
               "--min-block-occupancy-ratio",
               "--miner-extra-data"));
+
+      // Check that mining options are able to work
+      CommandLineUtils.checkOptionDependencies(
+          logger,
+          commandLine,
+          "--miner-enabled",
+          !minerOptionGroup.isMiningEnabled,
+          asList(
+              "--miner-stratum-enabled",
+              "--Xminer-remote-sealers-limit",
+              "--Xminer-remote-sealers-hashrate-ttl"));
     }
-    // Check that mining options are able to work
-    CommandLineUtils.checkOptionDependencies(
-        logger,
-        commandLine,
-        "--miner-enabled",
-        !minerOptionGroup.isMiningEnabled,
-        asList(
-            "--miner-stratum-enabled",
-            "--Xminer-remote-sealers-limit",
-            "--Xminer-remote-sealers-hashrate-ttl"));
 
     CommandLineUtils.failIfOptionDoesntMeetRequirement(
         commandLine,
@@ -3095,6 +3096,7 @@ private Runner synchronize(
             .storageProvider(keyValueStorageProvider(keyValueStorageName))
             .rpcEndpointService(rpcEndpointServiceImpl)
             .rpcMaxLogsRange(rpcMaxLogsRange)
+            .enodeDnsConfiguration(getEnodeDnsConfiguration())
             .build();
 
     addShutdownHook(runner);

besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java

@@ -168,6 +168,20 @@ public class BesuCommandTest extends CommandTestAbstract {
       (new JsonObject()).put("config", new JsonObject().put("ecCurve", "secp256k1"));
   private static final String ENCLAVE_PUBLIC_KEY_PATH =
       BesuCommand.class.getResource("/orion_publickey.pub").getPath();
+  private static final JsonObject VALID_GENESIS_QBFT_POST_LONDON =
+      (new JsonObject())
+          .put(
+              "config",
+              new JsonObject()
+                  .put("londonBlock", 0)
+                  .put("qbft", new JsonObject().put("blockperiodseconds", 5)));
+  private static final JsonObject VALID_GENESIS_IBFT2_POST_LONDON =
+      (new JsonObject())
+          .put(
+              "config",
+              new JsonObject()
+                  .put("londonBlock", 0)
+                  .put("ibft2", new JsonObject().put("blockperiodseconds", 5)));
 
   private static final String[] VALID_ENODE_STRINGS = {
     "enode://" + VALID_NODE_ID + "@192.168.0.1:4567",
@@ -3684,7 +3698,7 @@ public void stratumMiningIsEnabledWhenSpecified() throws Exception {
   @Test
   public void stratumMiningOptionsRequiresServiceToBeEnabled() {
 
-    parseCommand("--miner-stratum-enabled");
+    parseCommand("--network", "dev", "--miner-stratum-enabled");
 
     verifyOptionsConstraintLoggerCall("--miner-enabled", "--miner-stratum-enabled");
 
@@ -3698,7 +3712,7 @@ public void stratumMiningOptionsRequiresServiceToBeEnabled() {
   public void stratumMiningOptionsRequiresServiceToBeEnabledToml() throws IOException {
     final Path toml = createTempFile("toml", "miner-stratum-enabled=true\n");
 
-    parseCommand("--config-file", toml.toString());
+    parseCommand("--network", "dev", "--config-file", toml.toString());
 
     verifyOptionsConstraintLoggerCall("--miner-enabled", "--miner-stratum-enabled");
 
@@ -3753,6 +3767,46 @@ public void blockProducingOptionsWarnsMinerShouldBeEnabledToml() throws IOExcept
     assertThat(commandErrorOutput.toString(UTF_8)).isEmpty();
   }
 
+  @Test
+  public void blockProducingOptionsDoNotWarnWhenPoA() throws IOException {
+
+    final Path genesisFileQBFT = createFakeGenesisFile(VALID_GENESIS_QBFT_POST_LONDON);
+    parseCommand(
+        "--genesis-file",
+        genesisFileQBFT.toString(),
+        "--min-gas-price",
+        "42",
+        "--miner-extra-data",
+        "0x1122334455667788990011223344556677889900112233445566778899001122");
+
+    verify(mockLogger, atMost(0))
+        .warn(
+            stringArgumentCaptor.capture(),
+            stringArgumentCaptor.capture(),
+            stringArgumentCaptor.capture());
+
+    assertThat(commandOutput.toString(UTF_8)).isEmpty();
+    assertThat(commandErrorOutput.toString(UTF_8)).isEmpty();
+
+    final Path genesisFileIBFT2 = createFakeGenesisFile(VALID_GENESIS_IBFT2_POST_LONDON);
+    parseCommand(
+        "--genesis-file",
+        genesisFileIBFT2.toString(),
+        "--min-gas-price",
+        "42",
+        "--miner-extra-data",
+        "0x1122334455667788990011223344556677889900112233445566778899001122");
+
+    verify(mockLogger, atMost(0))
+        .warn(
+            stringArgumentCaptor.capture(),
+            stringArgumentCaptor.capture(),
+            stringArgumentCaptor.capture());
+
+    assertThat(commandOutput.toString(UTF_8)).isEmpty();
+    assertThat(commandErrorOutput.toString(UTF_8)).isEmpty();
+  }
+
   @Test
   public void blockProducingOptionsDoNotWarnWhenMergeEnabled() {
 
@@ -3801,6 +3855,33 @@ public void minGasPriceRequiresMainOptionToml() throws IOException {
     assertThat(commandErrorOutput.toString(UTF_8)).isEmpty();
   }
 
+  @Test
+  public void minGasPriceDoesNotRequireMainOptionWhenPoA() throws IOException {
+    final Path genesisFileQBFT = createFakeGenesisFile(VALID_GENESIS_QBFT_POST_LONDON);
+    parseCommand("--genesis-file", genesisFileQBFT.toString(), "--min-gas-price", "0");
+
+    verify(mockLogger, atMost(0))
+        .warn(
+            stringArgumentCaptor.capture(),
+            stringArgumentCaptor.capture(),
+            stringArgumentCaptor.capture());
+
+    assertThat(commandOutput.toString(UTF_8)).isEmpty();
+    assertThat(commandErrorOutput.toString(UTF_8)).isEmpty();
+
+    final Path genesisFileIBFT2 = createFakeGenesisFile(VALID_GENESIS_IBFT2_POST_LONDON);
+    parseCommand("--genesis-file", genesisFileIBFT2.toString(), "--min-gas-price", "0");
+
+    verify(mockLogger, atMost(0))
+        .warn(
+            stringArgumentCaptor.capture(),
+            stringArgumentCaptor.capture(),
+            stringArgumentCaptor.capture());
+
+    assertThat(commandOutput.toString(UTF_8)).isEmpty();
+    assertThat(commandErrorOutput.toString(UTF_8)).isEmpty();
+  }
+
   @Test
   public void miningParametersAreCaptured() {
     final Address requestedCoinbase = Address.fromHexString("0000011111222223333344444");

besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java

@@ -291,6 +291,7 @@ public void initMocks() throws Exception {
     when(mockRunnerBuilder.rpcEndpointService(any())).thenReturn(mockRunnerBuilder);
     when(mockRunnerBuilder.legacyForkId(anyBoolean())).thenReturn(mockRunnerBuilder);
     when(mockRunnerBuilder.rpcMaxLogsRange(any())).thenReturn(mockRunnerBuilder);
+    when(mockRunnerBuilder.enodeDnsConfiguration(any())).thenReturn(mockRunnerBuilder);
     when(mockRunnerBuilder.build()).thenReturn(mockRunner);
 
     final SignatureAlgorithm signatureAlgorithm = SignatureAlgorithmFactory.getInstance();

config/src/main/java/org/hyperledger/besu/config/GenesisConfigOptions.java

@@ -64,6 +64,13 @@ public interface GenesisConfigOptions {
    */
   boolean isClique();
 
+  /**
+   * Is a Proof of Authority network.
+   *
+   * @return the boolean
+   */
+  boolean isPoa();
+
   /**
    * Is consensus migration boolean.
    *

config/src/main/java/org/hyperledger/besu/config/JsonGenesisConfigOptions.java

@@ -144,6 +144,11 @@ public boolean isQbft() {
     return configRoot.has(QBFT_CONFIG_KEY);
   }
 
+  @Override
+  public boolean isPoa() {
+    return isQbft() || isClique() || isIbft2() || isIbftLegacy();
+  }
+
   @Override
   public BftConfigOptions getBftConfigOptions() {
     final String fieldKey = isIbft2() ? IBFT2_CONFIG_KEY : QBFT_CONFIG_KEY;

config/src/main/java/org/hyperledger/besu/config/StubGenesisConfigOptions.java

@@ -115,6 +115,11 @@ public boolean isQbft() {
     return false;
   }
 
+  @Override
+  public boolean isPoa() {
+    return false;
+  }
+
   @Override
   public CheckpointConfigOptions getCheckpointOptions() {
     return CheckpointConfigOptions.DEFAULT;

config/src/test/java/org/hyperledger/besu/config/GenesisConfigOptionsTest.java

@@ -46,13 +46,22 @@ public void shouldNotUseEthHashIfEthHashNotPresent() {
   public void shouldUseIbft2WhenIbft2InConfig() {
     final GenesisConfigOptions config = fromConfigOptions(singletonMap("ibft2", emptyMap()));
     assertThat(config.isIbft2()).isTrue();
+    assertThat(config.isPoa()).isTrue();
     assertThat(config.getConsensusEngine()).isEqualTo("ibft2");
   }
 
+  public void shouldUseQbftWhenQbftInConfig() {
+    final GenesisConfigOptions config = fromConfigOptions(singletonMap("qbft", emptyMap()));
+    assertThat(config.isQbft()).isTrue();
+    assertThat(config.isPoa()).isTrue();
+    assertThat(config.getConsensusEngine()).isEqualTo("qbft");
+  }
+
   @Test
   public void shouldUseCliqueWhenCliqueInConfig() {
     final GenesisConfigOptions config = fromConfigOptions(singletonMap("clique", emptyMap()));
     assertThat(config.isClique()).isTrue();
+    assertThat(config.isPoa()).isTrue();
     assertThat(config.getCliqueConfigOptions()).isNotSameAs(CliqueConfigOptions.DEFAULT);
     assertThat(config.getConsensusEngine()).isEqualTo("clique");
   }
@@ -61,6 +70,7 @@ public void shouldUseCliqueWhenCliqueInConfig() {
   public void shouldNotUseCliqueIfCliqueNotPresent() {
     final GenesisConfigOptions config = fromConfigOptions(emptyMap());
     assertThat(config.isClique()).isFalse();
+    assertThat(config.isPoa()).isFalse();
     assertThat(config.getCliqueConfigOptions()).isSameAs(CliqueConfigOptions.DEFAULT);
   }
 
@@ -228,6 +238,7 @@ public void shouldSupportEmptyGenesisConfig() {
     final GenesisConfigOptions config = GenesisConfigFile.fromConfig("{}").getConfigOptions();
     assertThat(config.isEthHash()).isFalse();
     assertThat(config.isClique()).isFalse();
+    assertThat(config.isPoa()).isFalse();
     assertThat(config.getHomesteadBlockNumber()).isEmpty();
   }
 

ethereum/api/src/integration-test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcTestMethodsFactory.java

@@ -184,6 +184,7 @@ public Map<String, JsonRpcMethod> methods() {
             dataDir,
             ethPeers,
             Vertx.vertx(new VertxOptions().setWorkerPoolSize(1)),
+            Optional.empty(),
             Optional.empty());
   }
 }

ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/AdminAddPeer.java

@@ -19,19 +19,23 @@
 import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcSuccessResponse;
 import org.hyperledger.besu.ethereum.p2p.network.P2PNetwork;
 import org.hyperledger.besu.ethereum.p2p.peers.DefaultPeer;
+import org.hyperledger.besu.ethereum.p2p.peers.EnodeDnsConfiguration;
 import org.hyperledger.besu.ethereum.p2p.peers.EnodeURLImpl;
 import org.hyperledger.besu.ethereum.p2p.peers.Peer;
 import org.hyperledger.besu.plugin.data.EnodeURL;
 
+import java.util.Optional;
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 public class AdminAddPeer extends AdminModifyPeer {
 
   private static final Logger LOG = LoggerFactory.getLogger(AdminAddPeer.class);
 
-  public AdminAddPeer(final P2PNetwork peerNetwork) {
-    super(peerNetwork);
+  public AdminAddPeer(
+      final P2PNetwork peerNetwork, final Optional<EnodeDnsConfiguration> enodeDnsConfiguration) {
+    super(peerNetwork, enodeDnsConfiguration);
   }
 
   @Override
@@ -42,7 +46,10 @@ public String getName() {
   @Override
   protected JsonRpcResponse performOperation(final Object id, final String enode) {
     LOG.debug("Adding ({}) to peers", enode);
-    final EnodeURL enodeURL = EnodeURLImpl.fromString(enode);
+    final EnodeURL enodeURL =
+        this.enodeDnsConfiguration.isEmpty()
+            ? EnodeURLImpl.fromString(enode)
+            : EnodeURLImpl.fromString(enode, enodeDnsConfiguration.get());
     final Peer peer = DefaultPeer.fromEnodeURL(enodeURL);
     final boolean addedToNetwork = peerNetwork.addMaintainedConnectionPeer(peer);
     return new JsonRpcSuccessResponse(id, addedToNetwork);