Merge branch 'main' into fixGetBlobsV1

.github/workflows/sonarcloud.yml

@@ -38,4 +38,4 @@ jobs:
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
           SONAR_ORGANIZATION: ${{ vars.SONAR_ORGANIZATION }}
           SONAR_PROJECT_KEY: ${{ vars.SONAR_PROJECT_KEY }}
-        run: ./gradlew build sonarqube --continue --info -Dorg.gradle.parallel=true -Dorg.gradle.caching=true
+        run: ./gradlew build sonar --continue -Dorg.gradle.parallel=true -Dorg.gradle.caching=true

CHANGELOG.md

@@ -18,7 +18,8 @@
 - Add configuration of Consolidation Request Contract Address via genesis configuration [#7647](https://github.com/hyperledger/besu/pull/7647)
 - Interrupt pending transaction processing on block creation timeout [#7673](https://github.com/hyperledger/besu/pull/7673)
 - Align gas cap calculation for transaction simulation to Geth approach [#7703](https://github.com/hyperledger/besu/pull/7703)
-- Expose chainId in the `BlockchainService` [#7702](https://github.com/hyperledger/besu/pull/7702)
+- Expose chainId in the `BlockchainService` [7702](https://github.com/hyperledger/besu/pull/7702)
+- Use head block instead of safe block for snap sync [7536](https://github.com/hyperledger/besu/issues/7536)
 - Add support for `chainId` in `CallParameters` [#7720](https://github.com/hyperledger/besu/pull/7720)
 
 ### Bug fixes

besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java

@@ -2701,7 +2701,8 @@ private String generateConfigurationOverview() {
 
     builder
         .setDataStorage(dataStorageOptions.normalizeDataStorageFormat())
-        .setSyncMode(syncMode.normalize());
+        .setSyncMode(syncMode.normalize())
+        .setSyncMinPeers(syncMinPeerCount);
 
     if (jsonRpcConfiguration != null && jsonRpcConfiguration.isEnabled()) {
       builder
@@ -2738,6 +2739,7 @@ && getDataStorageConfiguration().getBonsaiLimitTrieLogsEnabled()) {
 
     builder.setSnapServerEnabled(this.unstableSynchronizerOptions.isSnapsyncServerEnabled());
     builder.setSnapSyncBftEnabled(this.unstableSynchronizerOptions.isSnapSyncBftEnabled());
+    builder.setSnapSyncToHeadEnabled(this.unstableSynchronizerOptions.isSnapSyncToHeadEnabled());
 
     builder.setTxPoolImplementation(buildTransactionPoolConfiguration().getTxPoolImplementation());
     builder.setWorldStateUpdateMode(unstableEvmOptions.toDomainObject().worldUpdaterMode());

besu/src/main/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilder.java

@@ -46,6 +46,7 @@ public class ConfigurationOverviewBuilder {
   private String customGenesisFileName;
   private String dataStorage;
   private String syncMode;
+  private Integer syncMinPeers;
   private Integer rpcPort;
   private Collection<String> rpcHttpApis;
   private Integer enginePort;
@@ -57,6 +58,7 @@ public class ConfigurationOverviewBuilder {
   private Integer trieLogsPruningWindowSize = null;
   private boolean isSnapServerEnabled = false;
   private boolean isSnapSyncBftEnabled = false;
+  private boolean isSnapSyncToHeadEnabled = true;
   private TransactionPoolConfiguration.Implementation txPoolImplementation;
   private EvmConfiguration.WorldUpdaterMode worldStateUpdateMode;
   private Map<String, String> environment;
@@ -148,6 +150,17 @@ public ConfigurationOverviewBuilder setSyncMode(final String syncMode) {
     return this;
   }
 
+  /**
+   * Sets sync min peers.
+   *
+   * @param syncMinPeers number of min peers for sync
+   * @return the builder
+   */
+  public ConfigurationOverviewBuilder setSyncMinPeers(final int syncMinPeers) {
+    this.syncMinPeers = syncMinPeers;
+    return this;
+  }
+
   /**
    * Sets rpc port.
    *
@@ -245,6 +258,18 @@ public ConfigurationOverviewBuilder setSnapSyncBftEnabled(final boolean snapSync
     return this;
   }
 
+  /**
+   * Sets snap sync to head enabled/disabled
+   *
+   * @param snapSyncToHeadEnabled bool to indicate if snap sync to head is enabled
+   * @return the builder
+   */
+  public ConfigurationOverviewBuilder setSnapSyncToHeadEnabled(
+      final boolean snapSyncToHeadEnabled) {
+    isSnapSyncToHeadEnabled = snapSyncToHeadEnabled;
+    return this;
+  }
+
   /**
    * Sets trie logs pruning window size
    *
@@ -340,6 +365,10 @@ public String build() {
       lines.add("Sync mode: " + syncMode);
     }
 
+    if (syncMinPeers != null) {
+      lines.add("Sync min peers: " + syncMinPeers);
+    }
+
     if (rpcHttpApis != null) {
       lines.add("RPC HTTP APIs: " + String.join(",", rpcHttpApis));
     }
@@ -373,6 +402,10 @@ public String build() {
       lines.add("Experimental Snap Sync for BFT enabled");
     }
 
+    if (isSnapSyncToHeadEnabled) {
+      lines.add("Snap Sync to Head enabled");
+    }
+
     if (isBonsaiLimitTrieLogsEnabled) {
       final StringBuilder trieLogPruningString = new StringBuilder();
       trieLogPruningString

besu/src/main/java/org/hyperledger/besu/cli/options/stable/JsonRpcHttpOptions.java

@@ -15,6 +15,7 @@
 package org.hyperledger.besu.cli.options.stable;
 
 import static java.util.Arrays.asList;
+import static org.hyperledger.besu.ethereum.api.jsonrpc.JsonRpcConfiguration.DEFAULT_JSON_RPC_HOST;
 import static org.hyperledger.besu.ethereum.api.jsonrpc.JsonRpcConfiguration.DEFAULT_JSON_RPC_PORT;
 import static org.hyperledger.besu.ethereum.api.jsonrpc.JsonRpcConfiguration.DEFAULT_PRETTY_JSON_ENABLED;
 import static org.hyperledger.besu.ethereum.api.jsonrpc.RpcApis.DEFAULT_RPC_APIS;
@@ -65,7 +66,7 @@ public class JsonRpcHttpOptions {
       paramLabel = DefaultCommandValues.MANDATORY_HOST_FORMAT_HELP,
       description = "Host for JSON-RPC HTTP to listen on (default: ${DEFAULT-VALUE})",
       arity = "1")
-  private String rpcHttpHost;
+  private String rpcHttpHost = DEFAULT_JSON_RPC_HOST;
 
   @CommandLine.Option(
       names = {"--rpc-http-port"},

besu/src/main/java/org/hyperledger/besu/cli/options/unstable/SynchronizerOptions.java

@@ -87,6 +87,8 @@ public class SynchronizerOptions implements CLIOptions<SynchronizerConfiguration
 
   private static final String SNAP_SYNC_BFT_ENABLED_FLAG = "--Xsnapsync-bft-enabled";
 
+  private static final String SNAP_SYNC_TO_HEAD_ENABLED_FLAG = "--Xsnapsync-to-head-enabled";
+
   /**
    * Parse block propagation range.
    *
@@ -314,6 +316,15 @@ public void parseBlockPropagationRange(final String arg) {
       description = "Snap sync enabled for BFT chains (default: ${DEFAULT-VALUE})")
   private Boolean snapsyncBftEnabled = SnapSyncConfiguration.DEFAULT_SNAP_SYNC_BFT_ENABLED;
 
+  @CommandLine.Option(
+      names = SNAP_SYNC_TO_HEAD_ENABLED_FLAG,
+      hidden = true,
+      paramLabel = "<Boolean>",
+      arity = "0..1",
+      description = "Snap sync to head enabled (default: ${DEFAULT-VALUE})")
+  private Boolean snapsyncToHeadEnabled =
+      SnapSyncConfiguration.DEFAULT_SNAP_SYNC_TO_HEAD_ENABLED_FLAG;
+
   @CommandLine.Option(
       names = {"--Xpeertask-system-enabled"},
       hidden = true,
@@ -341,6 +352,15 @@ public boolean isSnapSyncBftEnabled() {
     return snapsyncBftEnabled;
   }
 
+  /**
+   * Flag to know if Snap sync should sync to head instead of safe block
+   *
+   * @return true if snap sync should sync to head
+   */
+  public boolean isSnapSyncToHeadEnabled() {
+    return snapsyncToHeadEnabled;
+  }
+
   /**
    * Flag to indicate whether the peer task system should be used where available
    *
@@ -401,6 +421,7 @@ public static SynchronizerOptions fromConfig(final SynchronizerConfiguration con
     options.checkpointPostMergeSyncEnabled = config.isCheckpointPostMergeEnabled();
     options.snapsyncServerEnabled = config.getSnapSyncConfiguration().isSnapServerEnabled();
     options.snapsyncBftEnabled = config.getSnapSyncConfiguration().isSnapSyncBftEnabled();
+    options.snapsyncToHeadEnabled = config.getSnapSyncConfiguration().isSnapSyncToHeadEnabled();
     return options;
   }
 
@@ -434,6 +455,7 @@ public SynchronizerConfiguration.Builder toDomainObject() {
             .localFlatStorageCountToHealPerRequest(snapsyncFlatStorageHealedCountPerRequest)
             .isSnapServerEnabled(snapsyncServerEnabled)
             .isSnapSyncBftEnabled(snapsyncBftEnabled)
+            .isSnapSyncToHeadEnabled(snapsyncToHeadEnabled)
             .build());
     builder.checkpointPostMergeEnabled(checkpointPostMergeSyncEnabled);
     builder.isPeerTaskSystemEnabled(isPeerTaskSystemEnabled);
@@ -493,7 +515,9 @@ public List<String> getCLIOptions() {
             SNAP_SERVER_ENABLED_FLAG,
             OptionParser.format(snapsyncServerEnabled),
             SNAP_SYNC_BFT_ENABLED_FLAG,
-            OptionParser.format(snapsyncBftEnabled));
+            OptionParser.format(snapsyncBftEnabled),
+            SNAP_SYNC_TO_HEAD_ENABLED_FLAG,
+            OptionParser.format(snapsyncToHeadEnabled));
     return value;
   }
 }

besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java

@@ -65,6 +65,7 @@
 import org.hyperledger.besu.ethereum.eth.sync.PivotBlockSelector;
 import org.hyperledger.besu.ethereum.eth.sync.SyncMode;
 import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration;
+import org.hyperledger.besu.ethereum.eth.sync.fastsync.PivotSelectorFromHeadBlock;
 import org.hyperledger.besu.ethereum.eth.sync.fastsync.PivotSelectorFromPeers;
 import org.hyperledger.besu.ethereum.eth.sync.fastsync.PivotSelectorFromSafeBlock;
 import org.hyperledger.besu.ethereum.eth.sync.fastsync.checkpoint.Checkpoint;
@@ -890,14 +891,27 @@ private PivotBlockSelector createPivotSelector(
             LOG.info("Initial sync done, unsubscribe forkchoice supplier");
           };
 
-      return new PivotSelectorFromSafeBlock(
-          protocolContext,
-          protocolSchedule,
-          ethContext,
-          metricsSystem,
-          genesisConfigOptions,
-          unverifiedForkchoiceSupplier,
-          unsubscribeForkchoiceListener);
+      if (syncConfig.getSnapSyncConfiguration().isSnapSyncToHeadEnabled()) {
+        LOG.info("Using head block for sync.");
+        return new PivotSelectorFromHeadBlock(
+            protocolContext,
+            protocolSchedule,
+            ethContext,
+            metricsSystem,
+            genesisConfigOptions,
+            unverifiedForkchoiceSupplier,
+            unsubscribeForkchoiceListener);
+      } else {
+        LOG.info("Using safe block for sync.");
+        return new PivotSelectorFromSafeBlock(
+            protocolContext,
+            protocolSchedule,
+            ethContext,
+            metricsSystem,
+            genesisConfigOptions,
+            unverifiedForkchoiceSupplier,
+            unsubscribeForkchoiceListener);
+      }
     } else {
       LOG.info("TTD difficulty is not present, creating initial sync phase for PoW");
       return new PivotSelectorFromPeers(ethContext, syncConfig, syncState, metricsSystem);

besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java

@@ -1029,6 +1029,12 @@ public void p2pHostAndPortOptionsAreRespected() {
 
     assertThat(commandOutput.toString(UTF_8)).isEmpty();
     assertThat(commandErrorOutput.toString(UTF_8)).isEmpty();
+
+    // rpc host should remain default ie 127.0.0.1
+    verify(mockRunnerBuilder).jsonRpcConfiguration(jsonRpcConfigArgumentCaptor.capture());
+    verify(mockRunnerBuilder).build();
+
+    assertThat(jsonRpcConfigArgumentCaptor.getValue().getHost()).isEqualTo("127.0.0.1");
   }
 
   @Test

besu/src/test/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilderTest.java

@@ -95,6 +95,16 @@ void setSyncMode() {
     assertThat(syncModeSet).contains("Sync mode: fast");
   }
 
+  @Test
+  void setSyncMinPeers() {
+    final String noSyncMinPeersSet = builder.build();
+    assertThat(noSyncMinPeersSet).doesNotContain("Sync min peers:");
+
+    builder.setSyncMinPeers(3);
+    final String syncMinPeersSet = builder.build();
+    assertThat(syncMinPeersSet).contains("Sync min peers: 3");
+  }
+
   @Test
   void setRpcPort() {
     final String noRpcPortSet = builder.build();

besu/src/test/java/org/hyperledger/besu/controller/MergeBesuControllerBuilderTest.java

@@ -45,6 +45,8 @@
 import org.hyperledger.besu.ethereum.core.PrivacyParameters;
 import org.hyperledger.besu.ethereum.eth.EthProtocolConfiguration;
 import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration;
+import org.hyperledger.besu.ethereum.eth.sync.fastsync.PivotSelectorFromHeadBlock;
+import org.hyperledger.besu.ethereum.eth.sync.fastsync.PivotSelectorFromSafeBlock;
 import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConfiguration;
 import org.hyperledger.besu.ethereum.mainnet.MainnetBlockHeaderFunctions;
 import org.hyperledger.besu.ethereum.mainnet.feemarket.BaseFeeMarket;
@@ -74,12 +76,15 @@
 import org.apache.tuweni.bytes.Bytes;
 import org.apache.tuweni.bytes.Bytes32;
 import org.apache.tuweni.units.bigints.UInt256;
+import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.junit.jupiter.api.io.TempDir;
 import org.mockito.Answers;
 import org.mockito.Mock;
+import org.mockito.MockedConstruction;
+import org.mockito.Mockito;
 import org.mockito.junit.jupiter.MockitoExtension;
 
 @ExtendWith(MockitoExtension.class)
@@ -90,7 +95,10 @@ public class MergeBesuControllerBuilderTest {
 
   @Mock GenesisConfigFile genesisConfigFile;
   @Mock GenesisConfigOptions genesisConfigOptions;
-  @Mock SynchronizerConfiguration synchronizerConfiguration;
+
+  @Mock(answer = Answers.RETURNS_DEEP_STUBS)
+  SynchronizerConfiguration synchronizerConfiguration;
+
   @Mock EthProtocolConfiguration ethProtocolConfiguration;
   @Mock CheckpointConfigOptions checkpointConfigOptions;
 
@@ -146,6 +154,9 @@ public void setup() {
     lenient().when(synchronizerConfiguration.getDownloaderParallelism()).thenReturn(1);
     lenient().when(synchronizerConfiguration.getTransactionsParallelism()).thenReturn(1);
     lenient().when(synchronizerConfiguration.getComputationParallelism()).thenReturn(1);
+    lenient()
+        .when(synchronizerConfiguration.getSnapSyncConfiguration().isSnapSyncToHeadEnabled())
+        .thenReturn(false);
 
     lenient()
         .when(synchronizerConfiguration.getBlockPropagationRange())
@@ -291,6 +302,32 @@ public void assertFinalizedBlockIsPresentWhenStored() {
     assertThat(mergeContext.getFinalized().get()).isEqualTo(finalizedHeader);
   }
 
+  @Test
+  public void assertPivotSelectorFromSafeBlockIsCreated() {
+    MockedConstruction<PivotSelectorFromSafeBlock> mocked =
+        Mockito.mockConstruction(PivotSelectorFromSafeBlock.class);
+    lenient()
+        .when(synchronizerConfiguration.getSnapSyncConfiguration().isSnapSyncToHeadEnabled())
+        .thenReturn(false);
+
+    visitWithMockConfigs(new MergeBesuControllerBuilder()).build();
+
+    Assertions.assertEquals(1, mocked.constructed().size());
+  }
+
+  @Test
+  public void assertPivotSelectorFromHeadBlockIsCreated() {
+    MockedConstruction<PivotSelectorFromHeadBlock> mocked =
+        Mockito.mockConstruction(PivotSelectorFromHeadBlock.class);
+    lenient()
+        .when(synchronizerConfiguration.getSnapSyncConfiguration().isSnapSyncToHeadEnabled())
+        .thenReturn(true);
+
+    visitWithMockConfigs(new MergeBesuControllerBuilder()).build();
+
+    Assertions.assertEquals(1, mocked.constructed().size());
+  }
+
   private BlockHeader finalizedBlockHeader() {
     final long blockNumber = 42;
     final Hash magicHash = Hash.wrap(Bytes32.leftPad(Bytes.ofUnsignedInt(42)));

build.gradle

@@ -31,6 +31,8 @@ plugins {
   id 'me.champeau.jmh' version '0.7.2' apply false
   id 'net.ltgt.errorprone' version '4.0.1'
   id 'maven-publish'
+  id 'jacoco'
+  id 'jacoco-report-aggregation'
   id 'org.sonarqube' version '5.1.0.4882'
 }
 
@@ -40,12 +42,12 @@ sonarqube {
     property "sonar.organization", "$System.env.SONAR_ORGANIZATION"
     property "sonar.gradle.skipCompile", "true"
     property "sonar.host.url", "https://sonarcloud.io"
-    property "sonar.coverage.jacoco.xmlReportPaths", "${buildDir}/reports/jacoco/jacocoRootReport/jacocoRootReport.xml"
+    property "sonar.coverage.jacoco.xmlReportPaths", "${buildDir}/reports/jacoco/testCodeCoverageReport/testCodeCoverageReport.xml"
     property "sonar.coverage.exclusions", "acceptance-tests/**/*"
   }
 }
 
-project.tasks["sonarqube"].dependsOn "jacocoRootReport"
+project.tasks["sonar"].dependsOn "testCodeCoverageReport"
 
 if (!JavaVersion.current().isCompatibleWith(JavaVersion.VERSION_21)) {
   throw new GradleException("Java 21 or later is required to build Besu.\n" +
@@ -360,6 +362,7 @@ allprojects {
       }
     }
     useJUnitPlatform {}
+    finalizedBy jacocoTestReport // report is always generated after tests run
   }
 
   javadoc {
@@ -962,19 +965,6 @@ jacocoTestReport {
   }
 }
 
-task jacocoRootReport(type: org.gradle.testing.jacoco.tasks.JacocoReport) {
-  additionalSourceDirs.from files(subprojects.sourceSets.main.allSource.srcDirs)
-  sourceDirectories.from files(subprojects.sourceSets.main.allSource.srcDirs)
-  classDirectories.from files(subprojects.sourceSets.main.output).asFileTree.matching { exclude 'org/hyperledger/besu/tests/acceptance/**' }
-  executionData.from fileTree(dir: '.', includes: ['**/jacoco/*.exec'])
-  reports {
-    xml.required = true
-    csv.required = true
-    html.destination file("build/reports/jacocoHtml")
-  }
-  onlyIf = { true }
-}
-
 // http://label-schema.org/rc1/
 // using the RFC3339 format "2016-04-12T23:20:50.52Z"
 def buildTime() {