Feat/collapse trie based on size

cmd/node/config/config.toml

@@ -886,9 +886,10 @@
     SnapshotsEnabled = true
     AccountsStatePruningEnabled = false
     PeerStatePruningEnabled = true
-    MaxStateTrieLevelInMemory = 5
-    MaxPeerTrieLevelInMemory = 5
     StateStatisticsEnabled = false
+    MaxUserTrieSizeInMemory = 524288000 #500MB
+    MaxPeerTrieSizeInMemory = 104857600 #100MB
+    DataTriesSizeInMemory = 524288000 #500MB
 
 [TrieLeavesRetrieverConfig]
     Enabled = false

common/common.go

@@ -28,6 +28,9 @@ const (
 	nonceIndex     = 0
 )
 
+// TenMbSize defines the size of 10 megabytes in bytes, used as a constant for memory limits or buffer sizes
+const TenMbSize = uint64(10485760)
+
 type executionResultHandler interface {
 	GetMiniBlockHeadersHandlers() []data.MiniBlockHeaderHandler
 }

common/interface.go

@@ -59,6 +59,7 @@ type Trie interface {
 	VerifyProof(rootHash []byte, key []byte, proof [][]byte) (bool, error)
 	GetStorageManager() StorageManager
 	IsMigratedToLatestVersion() (bool, error)
+	SizeInMemory() int
 	Close() error
 	IsInterfaceNil() bool
 }
@@ -151,7 +152,6 @@ type SnapshotDbHandler interface {
 // TriesHolder is used to store multiple tries
 type TriesHolder interface {
 	Put([]byte, Trie)
-	Replace(key []byte, tr Trie)
 	Get([]byte) Trie
 	GetAll() []Trie
 	Reset()

config/config.go

@@ -457,9 +457,10 @@ type StateTriesConfig struct {
 	SnapshotsEnabled            bool
 	AccountsStatePruningEnabled bool
 	PeerStatePruningEnabled     bool
-	MaxStateTrieLevelInMemory   uint
-	MaxPeerTrieLevelInMemory    uint
 	StateStatisticsEnabled      bool
+	MaxUserTrieSizeInMemory     uint64
+	MaxPeerTrieSizeInMemory     uint64
+	DataTriesSizeInMemory       uint64
 }
 
 // StateAccessesCollectorConfig will hold information about state accesses collector

config/tomlConfig_test.go

@@ -172,8 +172,6 @@ func TestTomlParser(t *testing.T) {
 			SnapshotsEnabled:            true,
 			AccountsStatePruningEnabled: true,
 			PeerStatePruningEnabled:     true,
-			MaxStateTrieLevelInMemory:   38,
-			MaxPeerTrieLevelInMemory:    39,
 		},
 		TxCacheBounds: TxCacheBoundsConfig{
 			MaxNumBytesPerSenderUpperBound: 33_554_432,
@@ -630,8 +628,6 @@ func TestTomlParser(t *testing.T) {
     SnapshotsEnabled = true
     AccountsStatePruningEnabled = true
     PeerStatePruningEnabled = true
-    MaxStateTrieLevelInMemory = 38
-    MaxPeerTrieLevelInMemory = 39
 `
 	cfg := Config{}
 

dataRetriever/factory/resolverscontainer/metaResolversContainerFactory_test.go

@@ -14,7 +14,7 @@ import (
 	"github.com/multiversx/mx-chain-go/dataRetriever/mock"
 	"github.com/multiversx/mx-chain-go/p2p"
 	"github.com/multiversx/mx-chain-go/process/factory"
-	"github.com/multiversx/mx-chain-go/state"
+	"github.com/multiversx/mx-chain-go/state/triesHolder"
 	"github.com/multiversx/mx-chain-go/storage"
 	"github.com/multiversx/mx-chain-go/testscommon"
 	"github.com/multiversx/mx-chain-go/testscommon/cache"
@@ -86,10 +86,10 @@ func createStoreForMeta() dataRetriever.StorageService {
 }
 
 func createTriesHolderForMeta() common.TriesHolder {
-	triesHolder := state.NewDataTriesHolder()
-	triesHolder.Put([]byte(dataRetriever.UserAccountsUnit.String()), &trieMock.TrieStub{})
-	triesHolder.Put([]byte(dataRetriever.PeerAccountsUnit.String()), &trieMock.TrieStub{})
-	return triesHolder
+	triesContainer := triesHolder.NewTriesHolder()
+	triesContainer.Put([]byte(dataRetriever.UserAccountsUnit.String()), &trieMock.TrieStub{})
+	triesContainer.Put([]byte(dataRetriever.PeerAccountsUnit.String()), &trieMock.TrieStub{})
+	return triesContainer
 }
 
 // ------- NewResolversContainerFactory

dataRetriever/factory/resolverscontainer/shardResolversContainerFactory_test.go

@@ -14,7 +14,7 @@ import (
 	"github.com/multiversx/mx-chain-go/dataRetriever/mock"
 	"github.com/multiversx/mx-chain-go/p2p"
 	"github.com/multiversx/mx-chain-go/process/factory"
-	"github.com/multiversx/mx-chain-go/state"
+	"github.com/multiversx/mx-chain-go/state/triesHolder"
 	"github.com/multiversx/mx-chain-go/storage"
 	"github.com/multiversx/mx-chain-go/testscommon"
 	"github.com/multiversx/mx-chain-go/testscommon/cache"
@@ -92,10 +92,10 @@ func createStoreForShard() dataRetriever.StorageService {
 }
 
 func createTriesHolderForShard() common.TriesHolder {
-	triesHolder := state.NewDataTriesHolder()
-	triesHolder.Put([]byte(dataRetriever.UserAccountsUnit.String()), &trieMock.TrieStub{})
-	triesHolder.Put([]byte(dataRetriever.PeerAccountsUnit.String()), &trieMock.TrieStub{})
-	return triesHolder
+	triesContainer := triesHolder.NewTriesHolder()
+	triesContainer.Put([]byte(dataRetriever.UserAccountsUnit.String()), &trieMock.TrieStub{})
+	triesContainer.Put([]byte(dataRetriever.PeerAccountsUnit.String()), &trieMock.TrieStub{})
+	return triesContainer
 }
 
 // ------- NewResolversContainerFactory

dataRetriever/factory/storageRequestersContainer/metaRequestersContainerFactory_test.go

@@ -221,8 +221,6 @@ func getArgumentsMeta() storagerequesterscontainer.FactoryArgs {
 			StateTriesConfig: config.StateTriesConfig{
 				AccountsStatePruningEnabled: false,
 				PeerStatePruningEnabled:     false,
-				MaxStateTrieLevelInMemory:   5,
-				MaxPeerTrieLevelInMemory:    5,
 			},
 		},
 		ShardIDForTries:          0,

dataRetriever/factory/storageRequestersContainer/shardRequestersContainerFactory_test.go

@@ -206,8 +206,6 @@ func getArgumentsShard() storagerequesterscontainer.FactoryArgs {
 			StateTriesConfig: config.StateTriesConfig{
 				AccountsStatePruningEnabled: false,
 				PeerStatePruningEnabled:     false,
-				MaxStateTrieLevelInMemory:   5,
-				MaxPeerTrieLevelInMemory:    5,
 			},
 		},
 		ShardIDForTries:          0,

epochStart/bootstrap/process.go

@@ -15,10 +15,9 @@ import (
 	"github.com/multiversx/mx-chain-core-go/data"
 	"github.com/multiversx/mx-chain-core-go/data/block"
 	"github.com/multiversx/mx-chain-core-go/data/typeConverters/uint64ByteSlice"
+	"github.com/multiversx/mx-chain-go/state/triesHolder"
 	logger "github.com/multiversx/mx-chain-logger-go"
 
-	"github.com/multiversx/mx-chain-go/process/interceptors/processor"
-
 	"github.com/multiversx/mx-chain-go/common"
 	disabledCommon "github.com/multiversx/mx-chain-go/common/disabled"
 	"github.com/multiversx/mx-chain-go/common/ordering"
@@ -42,11 +41,11 @@ import (
 	"github.com/multiversx/mx-chain-go/process/heartbeat/validator"
 	"github.com/multiversx/mx-chain-go/process/interceptors"
 	disabledInterceptors "github.com/multiversx/mx-chain-go/process/interceptors/disabled"
+	"github.com/multiversx/mx-chain-go/process/interceptors/processor"
 	"github.com/multiversx/mx-chain-go/process/peer"
 	"github.com/multiversx/mx-chain-go/redundancy"
 	"github.com/multiversx/mx-chain-go/sharding"
 	"github.com/multiversx/mx-chain-go/sharding/nodesCoordinator"
-	"github.com/multiversx/mx-chain-go/state"
 	"github.com/multiversx/mx-chain-go/state/syncer"
 	"github.com/multiversx/mx-chain-go/storage"
 	"github.com/multiversx/mx-chain-go/storage/cache"
@@ -283,7 +282,7 @@ func NewEpochStartBootstrap(args ArgsEpochStartBootstrap) (*epochStartBootstrap,
 		return nil, err
 	}
 
-	epochStartProvider.trieContainer = state.NewDataTriesHolder()
+	epochStartProvider.trieContainer = triesHolder.NewTriesHolder()
 	epochStartProvider.trieStorageManagers = make(map[string]common.StorageManager)
 
 	if epochStartProvider.generalConfig.Hardfork.AfterHardFork {
@@ -1573,7 +1572,6 @@ func (e *epochStartBootstrap) syncUserAccountsState(rootHash []byte) error {
 			RequestHandler:                    e.requestHandler,
 			Timeout:                           common.TimeoutGettingTrieNodes,
 			Cacher:                            e.dataPool.TrieNodes(),
-			MaxTrieLevelInMemory:              e.generalConfig.StateTriesConfig.MaxStateTrieLevelInMemory,
 			MaxHardCapForMissingNodes:         e.maxHardCapForMissingNodes,
 			TrieSyncerVersion:                 e.trieSyncerVersion,
 			CheckNodesOnDisk:                  e.checkNodesOnDisk,
@@ -1647,7 +1645,6 @@ func (e *epochStartBootstrap) syncValidatorAccountsState(rootHash []byte) error
 			RequestHandler:                    e.requestHandler,
 			Timeout:                           common.TimeoutGettingTrieNodes,
 			Cacher:                            e.dataPool.TrieNodes(),
-			MaxTrieLevelInMemory:              e.generalConfig.StateTriesConfig.MaxPeerTrieLevelInMemory,
 			MaxHardCapForMissingNodes:         e.maxHardCapForMissingNodes,
 			TrieSyncerVersion:                 e.trieSyncerVersion,
 			CheckNodesOnDisk:                  e.checkNodesOnDisk,

epochStart/bootstrap/process_test.go

@@ -182,8 +182,9 @@ func createMockEpochStartBootstrapArgs(
 				AccountsStatePruningEnabled: true,
 				SnapshotsEnabled:            true,
 				PeerStatePruningEnabled:     true,
-				MaxStateTrieLevelInMemory:   5,
-				MaxPeerTrieLevelInMemory:    5,
+				MaxUserTrieSizeInMemory:     generalCfg.StateTriesConfig.MaxUserTrieSizeInMemory,
+				MaxPeerTrieSizeInMemory:     generalCfg.StateTriesConfig.MaxPeerTrieSizeInMemory,
+				DataTriesSizeInMemory:       generalCfg.StateTriesConfig.DataTriesSizeInMemory,
 			},
 			TrieStorageManagerConfig: config.TrieStorageManagerConfig{
 				PruningBufferLen:      1000,

epochStart/metachain/baseRewards_test.go

@@ -29,7 +29,6 @@ import (
 	"github.com/multiversx/mx-chain-go/testscommon/hashingMocks"
 	"github.com/multiversx/mx-chain-go/testscommon/shardingMocks"
 	stateMock "github.com/multiversx/mx-chain-go/testscommon/state"
-	"github.com/multiversx/mx-chain-go/testscommon/storage"
 	"github.com/multiversx/mx-chain-go/trie"
 )
 
@@ -1140,11 +1139,11 @@ func getBaseRewardsArguments() BaseRewardsCreatorArgs {
 	hasher := sha256.NewSha256()
 	marshalizer := &marshal.GogoProtoMarshalizer{}
 
-	storageManagerArgs := storage.GetStorageManagerArgs()
+	storageManagerArgs := txExecOrderStub.GetStorageManagerArgs()
 	storageManagerArgs.Marshalizer = marshalizer
 	storageManagerArgs.Hasher = hasher
 
-	trieFactoryManager, _ := trie.CreateTrieStorageManager(storageManagerArgs, storage.GetStorageManagerOptions())
+	trieFactoryManager, _ := trie.CreateTrieStorageManager(storageManagerArgs, txExecOrderStub.GetStorageManagerOptions())
 	argsAccCreator := factory.ArgsAccountCreator{
 		Hasher:                 hasher,
 		Marshaller:             marshalizer,

epochStart/metachain/systemSCs_test.go

@@ -45,6 +45,7 @@ import (
 	storageFactory "github.com/multiversx/mx-chain-go/storage/factory"
 	"github.com/multiversx/mx-chain-go/storage/storageunit"
 	"github.com/multiversx/mx-chain-go/testscommon"
+	testCommon "github.com/multiversx/mx-chain-go/testscommon/common"
 	"github.com/multiversx/mx-chain-go/testscommon/cryptoMocks"
 	dataRetrieverMock "github.com/multiversx/mx-chain-go/testscommon/dataRetriever"
 	"github.com/multiversx/mx-chain-go/testscommon/enableEpochsHandlerMock"
@@ -748,7 +749,8 @@ func createAccountsDB(
 	trieStorageManager common.StorageManager,
 	enableEpochsHandler common.EnableEpochsHandler,
 ) *state.AccountsDB {
-	tr, _ := trie.NewTrie(trieStorageManager, marshaller, hasher, enableEpochsHandler, 5)
+	tenMbSize := uint64(10485760)
+	tr, _ := trie.NewTrie(trieStorageManager, marshaller, hasher, enableEpochsHandler, tenMbSize)
 	ewlArgs := evictionWaitingList.MemoryEvictionWaitingListArgs{
 		RootHashesSize: 100,
 		HashesSize:     10000,
@@ -757,14 +759,15 @@ func createAccountsDB(
 	spm, _ := storagePruningManager.NewStoragePruningManager(ewl, 10)
 
 	args := state.ArgsAccountsDB{
-		Trie:                   tr,
-		Hasher:                 hasher,
-		Marshaller:             marshaller,
-		AccountFactory:         accountFactory,
-		StoragePruningManager:  spm,
-		AddressConverter:       &testscommon.PubkeyConverterMock{},
-		SnapshotsManager:       disabledState.NewDisabledSnapshotsManager(),
-		StateAccessesCollector: disabledState.NewDisabledStateAccessesCollector(),
+		Trie:                     tr,
+		Hasher:                   hasher,
+		Marshaller:               marshaller,
+		AccountFactory:           accountFactory,
+		StoragePruningManager:    spm,
+		AddressConverter:         &testscommon.PubkeyConverterMock{},
+		SnapshotsManager:         disabledState.NewDisabledSnapshotsManager(),
+		StateAccessesCollector:   disabledState.NewDisabledStateAccessesCollector(),
+		MaxDataTriesSizeInMemory: tenMbSize,
 	}
 	adb, _ := state.NewAccountsDB(args)
 	return adb
@@ -773,12 +776,12 @@ func createAccountsDB(
 func createFullArgumentsForSystemSCProcessing(enableEpochsConfig config.EnableEpochs, trieStorer storage.Storer) (ArgsNewEpochStartSystemSCProcessing, vm.SystemSCContainer) {
 	hasher := sha256.NewSha256()
 	marshalizer := &marshal.GogoProtoMarshalizer{}
-	storageManagerArgs := storageMock.GetStorageManagerArgs()
+	storageManagerArgs := testCommon.GetStorageManagerArgs()
 	storageManagerArgs.Marshalizer = marshalizer
 	storageManagerArgs.Hasher = hasher
 	storageManagerArgs.MainStorer = trieStorer
 
-	trieFactoryManager, _ := trie.CreateTrieStorageManager(storageManagerArgs, storageMock.GetStorageManagerOptions())
+	trieFactoryManager, _ := trie.CreateTrieStorageManager(storageManagerArgs, testCommon.GetStorageManagerOptions())
 	argsAccCreator := factory.ArgsAccountCreator{
 		Hasher:                 hasher,
 		Marshaller:             marshalizer,

factory/api/apiResolverFactory.go

@@ -589,27 +589,29 @@ func createNewAccountsAdapterApi(args scQueryElementArgs, chainHandler data.Chai
 	trieCreatorArgs := trieFactory.TrieCreateArgs{
 		MainStorer:          trieStorer,
 		PruningEnabled:      args.generalConfig.StateTriesConfig.AccountsStatePruningEnabled,
-		MaxTrieLevelInMem:   args.generalConfig.StateTriesConfig.MaxStateTrieLevelInMemory,
 		SnapshotsEnabled:    args.generalConfig.StateTriesConfig.SnapshotsEnabled,
 		IdleProvider:        args.coreComponents.ProcessStatusHandler(),
 		Identifier:          dataRetriever.UserAccountsUnit.String(),
 		EnableEpochsHandler: args.coreComponents.EnableEpochsHandler(),
 		StatsCollector:      args.statusCoreComponents.StateStatsHandler(),
+		MaxSizeInMemory:     args.generalConfig.StateTriesConfig.MaxUserTrieSizeInMemory,
 	}
 	trieStorageManager, merkleTrie, err := trFactory.Create(trieCreatorArgs)
 	if err != nil {
 		return nil, nil, err
 	}
 
 	argsAPIAccountsDB := state.ArgsAccountsDB{
-		Trie:                   merkleTrie,
-		Hasher:                 args.coreComponents.Hasher(),
-		Marshaller:             args.coreComponents.InternalMarshalizer(),
-		AccountFactory:         accountFactory,
-		StoragePruningManager:  storagePruning,
-		AddressConverter:       args.coreComponents.AddressPubKeyConverter(),
-		SnapshotsManager:       disabledState.NewDisabledSnapshotsManager(),
-		StateAccessesCollector: disabledState.NewDisabledStateAccessesCollector(),
+		Trie:                     merkleTrie,
+		Hasher:                   args.coreComponents.Hasher(),
+		Marshaller:               args.coreComponents.InternalMarshalizer(),
+		AccountFactory:           accountFactory,
+		StoragePruningManager:    storagePruning,
+		AddressConverter:         args.coreComponents.AddressPubKeyConverter(),
+		SnapshotsManager:         disabledState.NewDisabledSnapshotsManager(),
+		StateAccessesCollector:   disabledState.NewDisabledStateAccessesCollector(),
+		// TODO check if this should be lower than in the processing adb
+		MaxDataTriesSizeInMemory: args.generalConfig.StateTriesConfig.DataTriesSizeInMemory,
 	}
 
 	provider, err := blockInfoProviders.NewCurrentBlockInfo(chainHandler)

factory/api/apiResolverFactory_test.go

@@ -37,7 +37,10 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-const unreachableStep = 10000
+const (
+	unreachableStep = 10000
+	tenMBSize       = uint64(10485760)
+)
 
 type failingSteps struct {
 	marshallerStepCounter int
@@ -306,7 +309,9 @@ func createMockSCQueryElementArgs() api.SCQueryElementArgs {
 				SnapshotsGoroutineNum: 1,
 			},
 			StateTriesConfig: config.StateTriesConfig{
-				MaxStateTrieLevelInMemory: 5,
+				MaxUserTrieSizeInMemory: tenMBSize,
+				MaxPeerTrieSizeInMemory: tenMBSize,
+				DataTriesSizeInMemory:   tenMBSize,
 			},
 			VirtualMachine: config.VirtualMachineServicesConfig{
 				Querying: config.QueryVirtualMachineConfig{

factory/consensus/consensusComponents.go

@@ -544,7 +544,6 @@ func (ccf *consensusComponentsFactory) createArgsBaseAccountsSyncer(trieStorageM
 		RequestHandler:                    ccf.processComponents.RequestHandler(),
 		Timeout:                           common.TimeoutGettingTrieNodes,
 		Cacher:                            ccf.dataComponents.Datapool().TrieNodes(),
-		MaxTrieLevelInMemory:              ccf.config.StateTriesConfig.MaxStateTrieLevelInMemory,
 		MaxHardCapForMissingNodes:         ccf.config.TrieSync.MaxHardCapForMissingNodes,
 		TrieSyncerVersion:                 ccf.config.TrieSync.TrieSyncerVersion,
 		CheckNodesOnDisk:                  ccf.config.TrieSync.CheckNodesOnDisk,