fix: re-sync on drifted head + sentry alert (#1332)

Author: rodrigo-o

Makefile

@@ -166,38 +166,39 @@ test-iex:
 # NODE RUNNERS
 DISCOVERY_PORT ?= 9009
 METRICS_PORT ?= 9568
+MODE ?= full
 
 #▶️ mainnet: @ Run an interactive terminal using checkpoint sync for mainnet.
 mainnet: compile-all
-	iex -S mix run -- --checkpoint-sync-url https://mainnet-checkpoint-sync.stakely.io/ --metrics --metrics-port $(METRICS_PORT) --discovery-port $(DISCOVERY_PORT)
+	iex -S mix run -- --checkpoint-sync-url https://mainnet-checkpoint-sync.stakely.io/ --metrics --metrics-port $(METRICS_PORT) --discovery-port $(DISCOVERY_PORT) --mode $(MODE)
 
 #▶️ mainnet.logfile: @ Run an interactive terminal using checkpoint sync for mainnet with a log file.
 mainnet.logfile: compile-all
-	iex -S mix run -- --checkpoint-sync-url https://mainnet-checkpoint-sync.stakely.io/ --metrics --metrics-port $(METRICS_PORT)  --log-file ./logs/mainnet.log --discovery-port $(DISCOVERY_PORT)
+	iex -S mix run -- --checkpoint-sync-url https://mainnet-checkpoint-sync.stakely.io/ --metrics --metrics-port $(METRICS_PORT)  --log-file ./logs/mainnet.log --discovery-port $(DISCOVERY_PORT) --mode $(MODE)
 
 #▶️ sepolia: @ Run an interactive terminal using sepolia network
 sepolia: compile-all
-	iex -S mix run -- --checkpoint-sync-url https://sepolia.beaconstate.info --network sepolia --metrics --metrics-port $(METRICS_PORT)  --discovery-port $(DISCOVERY_PORT)
+	iex -S mix run -- --checkpoint-sync-url https://sepolia.beaconstate.info --network sepolia --metrics --metrics-port $(METRICS_PORT)  --discovery-port $(DISCOVERY_PORT) --mode $(MODE)
 
 #▶️ sepolia.logfile: @ Run an interactive terminal using sepolia network with a log file
 sepolia.logfile: compile-all
-	iex -S mix run -- --checkpoint-sync-url https://sepolia.beaconstate.info --network sepolia --metrics --metrics-port $(METRICS_PORT)  --log-file ./logs/sepolia.log --discovery-port $(DISCOVERY_PORT)
+	iex -S mix run -- --checkpoint-sync-url https://sepolia.beaconstate.info --network sepolia --metrics --metrics-port $(METRICS_PORT)  --log-file ./logs/sepolia.log --discovery-port $(DISCOVERY_PORT) --mode $(MODE)
 
 #▶️ holesky: @ Run an interactive terminal using holesky network
 holesky: compile-all
-	iex -S mix run -- --checkpoint-sync-url https://checkpoint-sync.holesky.ethpandaops.io --network holesky --metrics --metrics-port $(METRICS_PORT) --discovery-port $(DISCOVERY_PORT)
+	iex -S mix run -- --checkpoint-sync-url https://checkpoint-sync.holesky.ethpandaops.io --network holesky --metrics --metrics-port $(METRICS_PORT) --discovery-port $(DISCOVERY_PORT) --mode $(MODE)
 
 #▶️ holesky.logfile: @ Run an interactive terminal using holesky network with a log file
 holesky.logfile: compile-all
-	iex -S mix run -- --checkpoint-sync-url https://checkpoint-sync.holesky.ethpandaops.io --network holesky --log-file ./logs/holesky.log --metrics --metrics-port $(METRICS_PORT) --discovery-port $(DISCOVERY_PORT)
+	iex -S mix run -- --checkpoint-sync-url https://checkpoint-sync.holesky.ethpandaops.io --network holesky --log-file ./logs/holesky.log --metrics --metrics-port $(METRICS_PORT) --discovery-port $(DISCOVERY_PORT) --mode $(MODE)
 
 #▶️ gnosis: @ Run an interactive terminal using gnosis network
 gnosis: compile-all
-	iex -S mix run -- --checkpoint-sync-url https://checkpoint.gnosischain.com --network gnosis --metrics --metrics-port $(METRICS_PORT) --discovery-port $(DISCOVERY_PORT)
+	iex -S mix run -- --checkpoint-sync-url https://checkpoint.gnosischain.com --network gnosis --metrics --metrics-port $(METRICS_PORT) --discovery-port $(DISCOVERY_PORT) --mode $(MODE)
 
 #▶️ gnosis.logfile: @ Run an interactive terminal using gnosis network with a log file
 gnosis.logfile: compile-all
-	iex -S mix run -- --checkpoint-sync-url https://checkpoint.gnosischain.com --network gnosis --metrics --metrics-port $(METRICS_PORT) --log-file ./logs/gnosis.log --discovery-port $(DISCOVERY_PORT)
+	iex -S mix run -- --checkpoint-sync-url https://checkpoint.gnosischain.com --network gnosis --metrics --metrics-port $(METRICS_PORT) --log-file ./logs/gnosis.log --discovery-port $(DISCOVERY_PORT) --mode $(MODE)
 
 #▶️ checkpoint-sync: @ Run an interactive terminal using checkpoint sync for mainnet.
 checkpoint-sync: mainnet

README.md

@@ -449,7 +449,7 @@ In that case, you can use [eFlambè](https://github.com/Stratus3D/eflambe) to ge
 The following code will capture information from 10 calls to `Handlers.on_block/2`, dumping it in different files named \<timestamp\>-eflambe-output.bggg.
 
 ```elixir
-:eflambe.capture({LambdaEthereumConsensus.ForkChoice.Handlers, :on_block, 2}, 10)
+:eflambe.capture({LambdaEthereumConsensus.ForkChoice, :on_block, 2}, 2)
 ```
 
 The files generated can be processed via common flamegraph tools.

lib/lambda_ethereum_consensus/application.ex

@@ -33,7 +33,7 @@ defmodule LambdaEthereumConsensus.Application do
     :ok
   end
 
-  defp get_children(:db) do
+  defp get_children(:mixed) do
     CheckpointStates.new()
 
     [
@@ -43,8 +43,15 @@ defmodule LambdaEthereumConsensus.Application do
     ]
   end
 
+  defp get_children(:db) do
+    get_children(:mixed) ++
+      [
+        {Task.Supervisor, name: StoreStatesSupervisor}
+      ]
+  end
+
   defp get_children(:full) do
-    get_children(:db) ++
+    get_children(:mixed) ++
       [
         BeaconApi.Endpoint,
         KeyStoreApi.Endpoint,

lib/libp2p_port.ex

@@ -84,6 +84,7 @@ defmodule LambdaEthereumConsensus.Libp2pPort do
 
   @tick_time 1000
   @sync_delay_millis 15_000
+  @head_drift_alert 12
 
   ######################
   ### API
@@ -792,9 +793,34 @@ defmodule LambdaEthereumConsensus.Libp2pPort do
 
     maybe_log_new_slot(slot_data, new_slot_data)
 
-    updated_state |> Map.put(:store, new_store)
+    updated_state
+    |> Map.put(:store, new_store)
+    |> update_syncing_status(new_slot_data, new_store)
   end
 
+  defp update_syncing_status(%{syncing: false} = state, {slot, _third}, %Types.Store{
+         head_slot: head_slot
+       })
+       when slot - head_slot >= @head_drift_alert do
+    Logger.error("[Libp2p] Head slot drifted by #{slot - head_slot} slots.")
+
+    # TODO: (#1194) This is a temporary fix to avoid the drift alert to be triggered and the resync to kick in
+    # when the node is not fully synced. We should have a better way to handle this.
+    Process.send_after(self(), :sync_blocks, 500)
+
+    %{state | syncing: true}
+  end
+
+  defp update_syncing_status(
+         %{syncing: true, blocks_remaining: 0} = state,
+         {slot, _third},
+         %Types.Store{head_slot: head_slot}
+       )
+       when slot - head_slot == 0,
+       do: %{state | syncing: false}
+
+  defp update_syncing_status(state, _slot_data, _), do: state
+
   defp schedule_next_tick() do
     # For millisecond precision
     time_to_next_tick = @tick_time - rem(:os.system_time(:millisecond), @tick_time)