diff --git a/cmd/node/config/config.toml b/cmd/node/config/config.toml index 4f22a66494b..7df6f631998 100644 --- a/cmd/node/config/config.toml +++ b/cmd/node/config/config.toml @@ -886,9 +886,10 @@ SnapshotsEnabled = true AccountsStatePruningEnabled = false PeerStatePruningEnabled = true - MaxStateTrieLevelInMemory = 5 - MaxPeerTrieLevelInMemory = 5 StateStatisticsEnabled = false + MaxUserTrieSizeInMemory = 524288000 #500MB + MaxPeerTrieSizeInMemory = 104857600 #100MB + DataTriesSizeInMemory = 524288000 #500MB [TrieLeavesRetrieverConfig] Enabled = false diff --git a/common/common.go b/common/common.go index 0ee094ccab1..b5bec600b2b 100644 --- a/common/common.go +++ b/common/common.go @@ -28,6 +28,9 @@ const ( nonceIndex = 0 ) +// TenMbSize defines the size of 10 megabytes in bytes, used as a constant for memory limits or buffer sizes +const TenMbSize = uint64(10485760) + type executionResultHandler interface { GetMiniBlockHeadersHandlers() []data.MiniBlockHeaderHandler } diff --git a/common/interface.go b/common/interface.go index c55e8c73c3d..6b564102261 100644 --- a/common/interface.go +++ b/common/interface.go @@ -59,6 +59,7 @@ type Trie interface { VerifyProof(rootHash []byte, key []byte, proof [][]byte) (bool, error) GetStorageManager() StorageManager IsMigratedToLatestVersion() (bool, error) + SizeInMemory() int Close() error IsInterfaceNil() bool } @@ -151,7 +152,6 @@ type SnapshotDbHandler interface { // TriesHolder is used to store multiple tries type TriesHolder interface { Put([]byte, Trie) - Replace(key []byte, tr Trie) Get([]byte) Trie GetAll() []Trie Reset() diff --git a/config/config.go b/config/config.go index d7aa89c959f..426f025475c 100644 --- a/config/config.go +++ b/config/config.go @@ -457,9 +457,10 @@ type StateTriesConfig struct { SnapshotsEnabled bool AccountsStatePruningEnabled bool PeerStatePruningEnabled bool - MaxStateTrieLevelInMemory uint - MaxPeerTrieLevelInMemory uint StateStatisticsEnabled bool + MaxUserTrieSizeInMemory uint64 + MaxPeerTrieSizeInMemory uint64 + DataTriesSizeInMemory uint64 } // StateAccessesCollectorConfig will hold information about state accesses collector diff --git a/config/tomlConfig_test.go b/config/tomlConfig_test.go index 20ae9006984..17c5fac0638 100644 --- a/config/tomlConfig_test.go +++ b/config/tomlConfig_test.go @@ -172,8 +172,6 @@ func TestTomlParser(t *testing.T) { SnapshotsEnabled: true, AccountsStatePruningEnabled: true, PeerStatePruningEnabled: true, - MaxStateTrieLevelInMemory: 38, - MaxPeerTrieLevelInMemory: 39, }, TxCacheBounds: TxCacheBoundsConfig{ MaxNumBytesPerSenderUpperBound: 33_554_432, @@ -630,8 +628,6 @@ func TestTomlParser(t *testing.T) { SnapshotsEnabled = true AccountsStatePruningEnabled = true PeerStatePruningEnabled = true - MaxStateTrieLevelInMemory = 38 - MaxPeerTrieLevelInMemory = 39 ` cfg := Config{} diff --git a/dataRetriever/factory/resolverscontainer/metaResolversContainerFactory_test.go b/dataRetriever/factory/resolverscontainer/metaResolversContainerFactory_test.go index 003d20a37b6..ed8037542a2 100644 --- a/dataRetriever/factory/resolverscontainer/metaResolversContainerFactory_test.go +++ b/dataRetriever/factory/resolverscontainer/metaResolversContainerFactory_test.go @@ -14,7 +14,7 @@ import ( "github.com/multiversx/mx-chain-go/dataRetriever/mock" "github.com/multiversx/mx-chain-go/p2p" "github.com/multiversx/mx-chain-go/process/factory" - "github.com/multiversx/mx-chain-go/state" + "github.com/multiversx/mx-chain-go/state/triesHolder" "github.com/multiversx/mx-chain-go/storage" "github.com/multiversx/mx-chain-go/testscommon" "github.com/multiversx/mx-chain-go/testscommon/cache" @@ -86,10 +86,10 @@ func createStoreForMeta() dataRetriever.StorageService { } func createTriesHolderForMeta() common.TriesHolder { - triesHolder := state.NewDataTriesHolder() - triesHolder.Put([]byte(dataRetriever.UserAccountsUnit.String()), &trieMock.TrieStub{}) - triesHolder.Put([]byte(dataRetriever.PeerAccountsUnit.String()), &trieMock.TrieStub{}) - return triesHolder + triesContainer := triesHolder.NewTriesHolder() + triesContainer.Put([]byte(dataRetriever.UserAccountsUnit.String()), &trieMock.TrieStub{}) + triesContainer.Put([]byte(dataRetriever.PeerAccountsUnit.String()), &trieMock.TrieStub{}) + return triesContainer } // ------- NewResolversContainerFactory diff --git a/dataRetriever/factory/resolverscontainer/shardResolversContainerFactory_test.go b/dataRetriever/factory/resolverscontainer/shardResolversContainerFactory_test.go index e2a26be3733..74f99260ec9 100644 --- a/dataRetriever/factory/resolverscontainer/shardResolversContainerFactory_test.go +++ b/dataRetriever/factory/resolverscontainer/shardResolversContainerFactory_test.go @@ -14,7 +14,7 @@ import ( "github.com/multiversx/mx-chain-go/dataRetriever/mock" "github.com/multiversx/mx-chain-go/p2p" "github.com/multiversx/mx-chain-go/process/factory" - "github.com/multiversx/mx-chain-go/state" + "github.com/multiversx/mx-chain-go/state/triesHolder" "github.com/multiversx/mx-chain-go/storage" "github.com/multiversx/mx-chain-go/testscommon" "github.com/multiversx/mx-chain-go/testscommon/cache" @@ -92,10 +92,10 @@ func createStoreForShard() dataRetriever.StorageService { } func createTriesHolderForShard() common.TriesHolder { - triesHolder := state.NewDataTriesHolder() - triesHolder.Put([]byte(dataRetriever.UserAccountsUnit.String()), &trieMock.TrieStub{}) - triesHolder.Put([]byte(dataRetriever.PeerAccountsUnit.String()), &trieMock.TrieStub{}) - return triesHolder + triesContainer := triesHolder.NewTriesHolder() + triesContainer.Put([]byte(dataRetriever.UserAccountsUnit.String()), &trieMock.TrieStub{}) + triesContainer.Put([]byte(dataRetriever.PeerAccountsUnit.String()), &trieMock.TrieStub{}) + return triesContainer } // ------- NewResolversContainerFactory diff --git a/dataRetriever/factory/storageRequestersContainer/metaRequestersContainerFactory_test.go b/dataRetriever/factory/storageRequestersContainer/metaRequestersContainerFactory_test.go index 6a44b37b153..1186184fc84 100644 --- a/dataRetriever/factory/storageRequestersContainer/metaRequestersContainerFactory_test.go +++ b/dataRetriever/factory/storageRequestersContainer/metaRequestersContainerFactory_test.go @@ -221,8 +221,6 @@ func getArgumentsMeta() storagerequesterscontainer.FactoryArgs { StateTriesConfig: config.StateTriesConfig{ AccountsStatePruningEnabled: false, PeerStatePruningEnabled: false, - MaxStateTrieLevelInMemory: 5, - MaxPeerTrieLevelInMemory: 5, }, }, ShardIDForTries: 0, diff --git a/dataRetriever/factory/storageRequestersContainer/shardRequestersContainerFactory_test.go b/dataRetriever/factory/storageRequestersContainer/shardRequestersContainerFactory_test.go index ecf848b35ac..9e5e9645ac8 100644 --- a/dataRetriever/factory/storageRequestersContainer/shardRequestersContainerFactory_test.go +++ b/dataRetriever/factory/storageRequestersContainer/shardRequestersContainerFactory_test.go @@ -206,8 +206,6 @@ func getArgumentsShard() storagerequesterscontainer.FactoryArgs { StateTriesConfig: config.StateTriesConfig{ AccountsStatePruningEnabled: false, PeerStatePruningEnabled: false, - MaxStateTrieLevelInMemory: 5, - MaxPeerTrieLevelInMemory: 5, }, }, ShardIDForTries: 0, diff --git a/epochStart/bootstrap/process.go b/epochStart/bootstrap/process.go index 6fa278b77e7..70aa5ccf873 100644 --- a/epochStart/bootstrap/process.go +++ b/epochStart/bootstrap/process.go @@ -15,10 +15,9 @@ import ( "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/typeConverters/uint64ByteSlice" + "github.com/multiversx/mx-chain-go/state/triesHolder" logger "github.com/multiversx/mx-chain-logger-go" - "github.com/multiversx/mx-chain-go/process/interceptors/processor" - "github.com/multiversx/mx-chain-go/common" disabledCommon "github.com/multiversx/mx-chain-go/common/disabled" "github.com/multiversx/mx-chain-go/common/ordering" @@ -42,11 +41,11 @@ import ( "github.com/multiversx/mx-chain-go/process/heartbeat/validator" "github.com/multiversx/mx-chain-go/process/interceptors" disabledInterceptors "github.com/multiversx/mx-chain-go/process/interceptors/disabled" + "github.com/multiversx/mx-chain-go/process/interceptors/processor" "github.com/multiversx/mx-chain-go/process/peer" "github.com/multiversx/mx-chain-go/redundancy" "github.com/multiversx/mx-chain-go/sharding" "github.com/multiversx/mx-chain-go/sharding/nodesCoordinator" - "github.com/multiversx/mx-chain-go/state" "github.com/multiversx/mx-chain-go/state/syncer" "github.com/multiversx/mx-chain-go/storage" "github.com/multiversx/mx-chain-go/storage/cache" @@ -283,7 +282,7 @@ func NewEpochStartBootstrap(args ArgsEpochStartBootstrap) (*epochStartBootstrap, return nil, err } - epochStartProvider.trieContainer = state.NewDataTriesHolder() + epochStartProvider.trieContainer = triesHolder.NewTriesHolder() epochStartProvider.trieStorageManagers = make(map[string]common.StorageManager) if epochStartProvider.generalConfig.Hardfork.AfterHardFork { @@ -1573,7 +1572,6 @@ func (e *epochStartBootstrap) syncUserAccountsState(rootHash []byte) error { RequestHandler: e.requestHandler, Timeout: common.TimeoutGettingTrieNodes, Cacher: e.dataPool.TrieNodes(), - MaxTrieLevelInMemory: e.generalConfig.StateTriesConfig.MaxStateTrieLevelInMemory, MaxHardCapForMissingNodes: e.maxHardCapForMissingNodes, TrieSyncerVersion: e.trieSyncerVersion, CheckNodesOnDisk: e.checkNodesOnDisk, @@ -1647,7 +1645,6 @@ func (e *epochStartBootstrap) syncValidatorAccountsState(rootHash []byte) error RequestHandler: e.requestHandler, Timeout: common.TimeoutGettingTrieNodes, Cacher: e.dataPool.TrieNodes(), - MaxTrieLevelInMemory: e.generalConfig.StateTriesConfig.MaxPeerTrieLevelInMemory, MaxHardCapForMissingNodes: e.maxHardCapForMissingNodes, TrieSyncerVersion: e.trieSyncerVersion, CheckNodesOnDisk: e.checkNodesOnDisk, diff --git a/epochStart/bootstrap/process_test.go b/epochStart/bootstrap/process_test.go index 49843c2bc8b..7cabfa4e0bd 100644 --- a/epochStart/bootstrap/process_test.go +++ b/epochStart/bootstrap/process_test.go @@ -182,8 +182,9 @@ func createMockEpochStartBootstrapArgs( AccountsStatePruningEnabled: true, SnapshotsEnabled: true, PeerStatePruningEnabled: true, - MaxStateTrieLevelInMemory: 5, - MaxPeerTrieLevelInMemory: 5, + MaxUserTrieSizeInMemory: generalCfg.StateTriesConfig.MaxUserTrieSizeInMemory, + MaxPeerTrieSizeInMemory: generalCfg.StateTriesConfig.MaxPeerTrieSizeInMemory, + DataTriesSizeInMemory: generalCfg.StateTriesConfig.DataTriesSizeInMemory, }, TrieStorageManagerConfig: config.TrieStorageManagerConfig{ PruningBufferLen: 1000, diff --git a/epochStart/metachain/baseRewards_test.go b/epochStart/metachain/baseRewards_test.go index b0e1118b84e..61eb184c5b9 100644 --- a/epochStart/metachain/baseRewards_test.go +++ b/epochStart/metachain/baseRewards_test.go @@ -29,7 +29,6 @@ import ( "github.com/multiversx/mx-chain-go/testscommon/hashingMocks" "github.com/multiversx/mx-chain-go/testscommon/shardingMocks" stateMock "github.com/multiversx/mx-chain-go/testscommon/state" - "github.com/multiversx/mx-chain-go/testscommon/storage" "github.com/multiversx/mx-chain-go/trie" ) @@ -1140,11 +1139,11 @@ func getBaseRewardsArguments() BaseRewardsCreatorArgs { hasher := sha256.NewSha256() marshalizer := &marshal.GogoProtoMarshalizer{} - storageManagerArgs := storage.GetStorageManagerArgs() + storageManagerArgs := txExecOrderStub.GetStorageManagerArgs() storageManagerArgs.Marshalizer = marshalizer storageManagerArgs.Hasher = hasher - trieFactoryManager, _ := trie.CreateTrieStorageManager(storageManagerArgs, storage.GetStorageManagerOptions()) + trieFactoryManager, _ := trie.CreateTrieStorageManager(storageManagerArgs, txExecOrderStub.GetStorageManagerOptions()) argsAccCreator := factory.ArgsAccountCreator{ Hasher: hasher, Marshaller: marshalizer, diff --git a/epochStart/metachain/systemSCs_test.go b/epochStart/metachain/systemSCs_test.go index aac10d69d01..1aff5eb3e0e 100644 --- a/epochStart/metachain/systemSCs_test.go +++ b/epochStart/metachain/systemSCs_test.go @@ -45,6 +45,7 @@ import ( storageFactory "github.com/multiversx/mx-chain-go/storage/factory" "github.com/multiversx/mx-chain-go/storage/storageunit" "github.com/multiversx/mx-chain-go/testscommon" + testCommon "github.com/multiversx/mx-chain-go/testscommon/common" "github.com/multiversx/mx-chain-go/testscommon/cryptoMocks" dataRetrieverMock "github.com/multiversx/mx-chain-go/testscommon/dataRetriever" "github.com/multiversx/mx-chain-go/testscommon/enableEpochsHandlerMock" @@ -748,7 +749,8 @@ func createAccountsDB( trieStorageManager common.StorageManager, enableEpochsHandler common.EnableEpochsHandler, ) *state.AccountsDB { - tr, _ := trie.NewTrie(trieStorageManager, marshaller, hasher, enableEpochsHandler, 5) + tenMbSize := uint64(10485760) + tr, _ := trie.NewTrie(trieStorageManager, marshaller, hasher, enableEpochsHandler, tenMbSize) ewlArgs := evictionWaitingList.MemoryEvictionWaitingListArgs{ RootHashesSize: 100, HashesSize: 10000, @@ -757,14 +759,15 @@ func createAccountsDB( spm, _ := storagePruningManager.NewStoragePruningManager(ewl, 10) args := state.ArgsAccountsDB{ - Trie: tr, - Hasher: hasher, - Marshaller: marshaller, - AccountFactory: accountFactory, - StoragePruningManager: spm, - AddressConverter: &testscommon.PubkeyConverterMock{}, - SnapshotsManager: disabledState.NewDisabledSnapshotsManager(), - StateAccessesCollector: disabledState.NewDisabledStateAccessesCollector(), + Trie: tr, + Hasher: hasher, + Marshaller: marshaller, + AccountFactory: accountFactory, + StoragePruningManager: spm, + AddressConverter: &testscommon.PubkeyConverterMock{}, + SnapshotsManager: disabledState.NewDisabledSnapshotsManager(), + StateAccessesCollector: disabledState.NewDisabledStateAccessesCollector(), + MaxDataTriesSizeInMemory: tenMbSize, } adb, _ := state.NewAccountsDB(args) return adb @@ -773,12 +776,12 @@ func createAccountsDB( func createFullArgumentsForSystemSCProcessing(enableEpochsConfig config.EnableEpochs, trieStorer storage.Storer) (ArgsNewEpochStartSystemSCProcessing, vm.SystemSCContainer) { hasher := sha256.NewSha256() marshalizer := &marshal.GogoProtoMarshalizer{} - storageManagerArgs := storageMock.GetStorageManagerArgs() + storageManagerArgs := testCommon.GetStorageManagerArgs() storageManagerArgs.Marshalizer = marshalizer storageManagerArgs.Hasher = hasher storageManagerArgs.MainStorer = trieStorer - trieFactoryManager, _ := trie.CreateTrieStorageManager(storageManagerArgs, storageMock.GetStorageManagerOptions()) + trieFactoryManager, _ := trie.CreateTrieStorageManager(storageManagerArgs, testCommon.GetStorageManagerOptions()) argsAccCreator := factory.ArgsAccountCreator{ Hasher: hasher, Marshaller: marshalizer, diff --git a/factory/api/apiResolverFactory.go b/factory/api/apiResolverFactory.go index f60e73c162a..9ef03e5bef1 100644 --- a/factory/api/apiResolverFactory.go +++ b/factory/api/apiResolverFactory.go @@ -589,12 +589,12 @@ func createNewAccountsAdapterApi(args scQueryElementArgs, chainHandler data.Chai trieCreatorArgs := trieFactory.TrieCreateArgs{ MainStorer: trieStorer, PruningEnabled: args.generalConfig.StateTriesConfig.AccountsStatePruningEnabled, - MaxTrieLevelInMem: args.generalConfig.StateTriesConfig.MaxStateTrieLevelInMemory, SnapshotsEnabled: args.generalConfig.StateTriesConfig.SnapshotsEnabled, IdleProvider: args.coreComponents.ProcessStatusHandler(), Identifier: dataRetriever.UserAccountsUnit.String(), EnableEpochsHandler: args.coreComponents.EnableEpochsHandler(), StatsCollector: args.statusCoreComponents.StateStatsHandler(), + MaxSizeInMemory: args.generalConfig.StateTriesConfig.MaxUserTrieSizeInMemory, } trieStorageManager, merkleTrie, err := trFactory.Create(trieCreatorArgs) if err != nil { @@ -602,14 +602,16 @@ func createNewAccountsAdapterApi(args scQueryElementArgs, chainHandler data.Chai } argsAPIAccountsDB := state.ArgsAccountsDB{ - Trie: merkleTrie, - Hasher: args.coreComponents.Hasher(), - Marshaller: args.coreComponents.InternalMarshalizer(), - AccountFactory: accountFactory, - StoragePruningManager: storagePruning, - AddressConverter: args.coreComponents.AddressPubKeyConverter(), - SnapshotsManager: disabledState.NewDisabledSnapshotsManager(), - StateAccessesCollector: disabledState.NewDisabledStateAccessesCollector(), + Trie: merkleTrie, + Hasher: args.coreComponents.Hasher(), + Marshaller: args.coreComponents.InternalMarshalizer(), + AccountFactory: accountFactory, + StoragePruningManager: storagePruning, + AddressConverter: args.coreComponents.AddressPubKeyConverter(), + SnapshotsManager: disabledState.NewDisabledSnapshotsManager(), + StateAccessesCollector: disabledState.NewDisabledStateAccessesCollector(), + // TODO check if this should be lower than in the processing adb + MaxDataTriesSizeInMemory: args.generalConfig.StateTriesConfig.DataTriesSizeInMemory, } provider, err := blockInfoProviders.NewCurrentBlockInfo(chainHandler) diff --git a/factory/api/apiResolverFactory_test.go b/factory/api/apiResolverFactory_test.go index 79329267cf5..dd2af1535cc 100644 --- a/factory/api/apiResolverFactory_test.go +++ b/factory/api/apiResolverFactory_test.go @@ -37,7 +37,10 @@ import ( "github.com/stretchr/testify/require" ) -const unreachableStep = 10000 +const ( + unreachableStep = 10000 + tenMBSize = uint64(10485760) +) type failingSteps struct { marshallerStepCounter int @@ -306,7 +309,9 @@ func createMockSCQueryElementArgs() api.SCQueryElementArgs { SnapshotsGoroutineNum: 1, }, StateTriesConfig: config.StateTriesConfig{ - MaxStateTrieLevelInMemory: 5, + MaxUserTrieSizeInMemory: tenMBSize, + MaxPeerTrieSizeInMemory: tenMBSize, + DataTriesSizeInMemory: tenMBSize, }, VirtualMachine: config.VirtualMachineServicesConfig{ Querying: config.QueryVirtualMachineConfig{ diff --git a/factory/consensus/consensusComponents.go b/factory/consensus/consensusComponents.go index 7410a695a30..6c821239b1b 100644 --- a/factory/consensus/consensusComponents.go +++ b/factory/consensus/consensusComponents.go @@ -544,7 +544,6 @@ func (ccf *consensusComponentsFactory) createArgsBaseAccountsSyncer(trieStorageM RequestHandler: ccf.processComponents.RequestHandler(), Timeout: common.TimeoutGettingTrieNodes, Cacher: ccf.dataComponents.Datapool().TrieNodes(), - MaxTrieLevelInMemory: ccf.config.StateTriesConfig.MaxStateTrieLevelInMemory, MaxHardCapForMissingNodes: ccf.config.TrieSync.MaxHardCapForMissingNodes, TrieSyncerVersion: ccf.config.TrieSync.TrieSyncerVersion, CheckNodesOnDisk: ccf.config.TrieSync.CheckNodesOnDisk, diff --git a/factory/processing/blockProcessorCreator_test.go b/factory/processing/blockProcessorCreator_test.go index 008f5e5a4e7..aa139f9ae51 100644 --- a/factory/processing/blockProcessorCreator_test.go +++ b/factory/processing/blockProcessorCreator_test.go @@ -8,6 +8,7 @@ import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/hashing" "github.com/multiversx/mx-chain-core-go/marshal" + common2 "github.com/multiversx/mx-chain-go/testscommon/common" vmcommon "github.com/multiversx/mx-chain-vm-common-go" "github.com/stretchr/testify/require" @@ -26,7 +27,6 @@ import ( "github.com/multiversx/mx-chain-go/testscommon/hashingMocks" "github.com/multiversx/mx-chain-go/testscommon/processMocks" stateMock "github.com/multiversx/mx-chain-go/testscommon/state" - storageManager "github.com/multiversx/mx-chain-go/testscommon/storage" trieMock "github.com/multiversx/mx-chain-go/testscommon/trie" "github.com/multiversx/mx-chain-go/trie" ) @@ -91,13 +91,13 @@ func Test_newBlockProcessorCreatorForMeta(t *testing.T) { cryptoComponents := componentsMock.GetCryptoComponents(coreComponents) networkComponents := componentsMock.GetNetworkComponents(cryptoComponents) - storageManagerArgs := storageManager.GetStorageManagerArgs() + storageManagerArgs := common2.GetStorageManagerArgs() storageManagerArgs.Marshalizer = coreComponents.InternalMarshalizer() storageManagerArgs.Hasher = coreComponents.Hasher() - storageManagerUser, _ := trie.CreateTrieStorageManager(storageManagerArgs, storageManager.GetStorageManagerOptions()) + storageManagerUser, _ := trie.CreateTrieStorageManager(storageManagerArgs, common2.GetStorageManagerOptions()) storageManagerArgs.MainStorer = mock.NewMemDbMock() - storageManagerPeer, _ := trie.CreateTrieStorageManager(storageManagerArgs, storageManager.GetStorageManagerOptions()) + storageManagerPeer, _ := trie.CreateTrieStorageManager(storageManagerArgs, common2.GetStorageManagerOptions()) trieStorageManagers := make(map[string]common.StorageManager) trieStorageManagers[dataRetriever.UserAccountsUnit.String()] = storageManagerUser @@ -209,20 +209,22 @@ func createAccountAdapter( trieStorage common.StorageManager, handler common.EnableEpochsHandler, ) (state.AccountsAdapter, error) { - tr, err := trie.NewTrie(trieStorage, marshaller, hasher, handler, 5) + tenMbSize := uint64(10485760) + tr, err := trie.NewTrie(trieStorage, marshaller, hasher, handler, tenMbSize) if err != nil { return nil, err } args := state.ArgsAccountsDB{ - Trie: tr, - Hasher: hasher, - Marshaller: marshaller, - AccountFactory: accountFactory, - StoragePruningManager: disabled.NewDisabledStoragePruningManager(), - AddressConverter: &testscommon.PubkeyConverterMock{}, - SnapshotsManager: disabledState.NewDisabledSnapshotsManager(), - StateAccessesCollector: disabledState.NewDisabledStateAccessesCollector(), + Trie: tr, + Hasher: hasher, + Marshaller: marshaller, + AccountFactory: accountFactory, + StoragePruningManager: disabled.NewDisabledStoragePruningManager(), + AddressConverter: &testscommon.PubkeyConverterMock{}, + SnapshotsManager: disabledState.NewDisabledSnapshotsManager(), + StateAccessesCollector: disabledState.NewDisabledStateAccessesCollector(), + MaxDataTriesSizeInMemory: tenMbSize, } adb, err := state.NewAccountsDB(args) if err != nil { diff --git a/factory/processing/processComponents.go b/factory/processing/processComponents.go index f5050230e1c..d4e30c29e0b 100644 --- a/factory/processing/processComponents.go +++ b/factory/processing/processComponents.go @@ -1952,7 +1952,6 @@ func (pcf *processComponentsFactory) createExportFactoryHandler( ExportTriesStorageConfig: hardforkConfig.ExportTriesStorageConfig, ExportStateStorageConfig: hardforkConfig.ExportStateStorageConfig, ExportStateKeysConfig: hardforkConfig.ExportKeysStorageConfig, - MaxTrieLevelInMemory: pcf.config.StateTriesConfig.MaxStateTrieLevelInMemory, WhiteListHandler: pcf.whiteListHandler, WhiteListerVerifiedTxs: pcf.whiteListerVerifiedTxs, MainInterceptorsContainer: mainInterceptorsContainer, diff --git a/factory/state/stateComponents.go b/factory/state/stateComponents.go index 0a2e4f278df..29aa97f51e4 100644 --- a/factory/state/stateComponents.go +++ b/factory/state/stateComponents.go @@ -253,15 +253,17 @@ func (scf *stateComponentsFactory) createAccountsAdapters(triesContainer common. } argsProcessingAccountsDB := state.ArgsAccountsDB{ - Trie: merkleTrie, - Hasher: scf.core.Hasher(), - Marshaller: scf.core.InternalMarshalizer(), - AccountFactory: accountFactory, - StoragePruningManager: storagePruning, - AddressConverter: scf.core.AddressPubKeyConverter(), - SnapshotsManager: snapshotsManager, - StateAccessesCollector: StateAccessesCollector, - PruningEnabled: scf.config.StateTriesConfig.AccountsStatePruningEnabled, + Trie: merkleTrie, + Hasher: scf.core.Hasher(), + Marshaller: scf.core.InternalMarshalizer(), + AccountFactory: accountFactory, + StoragePruningManager: storagePruning, + AddressConverter: scf.core.AddressPubKeyConverter(), + SnapshotsManager: snapshotsManager, + StateAccessesCollector: StateAccessesCollector, + PruningEnabled: scf.config.StateTriesConfig.AccountsStatePruningEnabled, + MaxDataTriesSizeInMemory: scf.config.StateTriesConfig.DataTriesSizeInMemory, + } accountsAdapter, err := state.NewAccountsDB(argsProcessingAccountsDB) if err != nil { @@ -280,14 +282,16 @@ func (scf *stateComponentsFactory) createAccountsAdapters(triesContainer common. } argsAPIAccountsDB := state.ArgsAccountsDB{ - Trie: merkleTrie, - Hasher: scf.core.Hasher(), - Marshaller: scf.core.InternalMarshalizer(), - AccountFactory: accountFactoryAPI, - StoragePruningManager: storagePruning, - AddressConverter: scf.core.AddressPubKeyConverter(), - SnapshotsManager: disabled.NewDisabledSnapshotsManager(), - StateAccessesCollector: disabled.NewDisabledStateAccessesCollector(), + Trie: merkleTrie, + Hasher: scf.core.Hasher(), + Marshaller: scf.core.InternalMarshalizer(), + AccountFactory: accountFactoryAPI, + StoragePruningManager: storagePruning, + AddressConverter: scf.core.AddressPubKeyConverter(), + SnapshotsManager: disabled.NewDisabledSnapshotsManager(), + StateAccessesCollector: disabled.NewDisabledStateAccessesCollector(), + // TODO check if this should be lower than in the processing adb + MaxDataTriesSizeInMemory: scf.config.StateTriesConfig.DataTriesSizeInMemory, } accountsAdapterApiOnFinal, err := factoryState.CreateAccountsAdapterAPIOnFinal(argsAPIAccountsDB, scf.chainHandler) @@ -357,15 +361,17 @@ func (scf *stateComponentsFactory) createPeerAdapter(triesContainer common.Tries // TODO: also collect state accesses for the peer trie argsProcessingPeerAccountsDB := state.ArgsAccountsDB{ - Trie: merkleTrie, - Hasher: scf.core.Hasher(), - Marshaller: scf.core.InternalMarshalizer(), - AccountFactory: accountFactory, - StoragePruningManager: storagePruning, - AddressConverter: scf.core.AddressPubKeyConverter(), - SnapshotsManager: snapshotManager, - StateAccessesCollector: disabled.NewDisabledStateAccessesCollector(), - PruningEnabled: scf.config.StateTriesConfig.PeerStatePruningEnabled, + Trie: merkleTrie, + Hasher: scf.core.Hasher(), + Marshaller: scf.core.InternalMarshalizer(), + AccountFactory: accountFactory, + StoragePruningManager: storagePruning, + AddressConverter: scf.core.AddressPubKeyConverter(), + SnapshotsManager: snapshotManager, + StateAccessesCollector: disabled.NewDisabledStateAccessesCollector(), + PruningEnabled: scf.config.StateTriesConfig.PeerStatePruningEnabled, + // TODO check if this should be different from the user adb + MaxDataTriesSizeInMemory: scf.config.StateTriesConfig.DataTriesSizeInMemory, } peerAdapter, err := state.NewPeerAccountsDB(argsProcessingPeerAccountsDB) if err != nil { diff --git a/genesis/process/genesisBlockCreator_test.go b/genesis/process/genesisBlockCreator_test.go index d5dada9caec..5c523d1d50a 100644 --- a/genesis/process/genesisBlockCreator_test.go +++ b/genesis/process/genesisBlockCreator_test.go @@ -57,8 +57,8 @@ func createMockArgument( entireSupply *big.Int, ) ArgsGenesisBlockCreator { - storageManagerArgs := storageCommon.GetStorageManagerArgs() - storageManager, _ := trie.CreateTrieStorageManager(storageManagerArgs, storageCommon.GetStorageManagerOptions()) + storageManagerArgs := commonMocks.GetStorageManagerArgs() + storageManager, _ := trie.CreateTrieStorageManager(storageManagerArgs, commonMocks.GetStorageManagerOptions()) trieStorageManagers := make(map[string]common.StorageManager) trieStorageManagers[dataRetriever.UserAccountsUnit.String()] = storageManager diff --git a/genesis/process/memoryComponents.go b/genesis/process/memoryComponents.go index 91c32211740..d2bf3e3bb50 100644 --- a/genesis/process/memoryComponents.go +++ b/genesis/process/memoryComponents.go @@ -11,8 +11,6 @@ import ( "github.com/multiversx/mx-chain-go/trie" ) -const maxTrieLevelInMemory = uint(5) - func createAccountAdapter( marshaller marshal.Marshalizer, hasher hashing.Hasher, @@ -21,20 +19,21 @@ func createAccountAdapter( addressConverter core.PubkeyConverter, enableEpochsHandler common.EnableEpochsHandler, ) (state.AccountsAdapter, error) { - tr, err := trie.NewTrie(trieStorage, marshaller, hasher, enableEpochsHandler, maxTrieLevelInMemory) + tr, err := trie.NewTrie(trieStorage, marshaller, hasher, enableEpochsHandler, common.TenMbSize) if err != nil { return nil, err } args := state.ArgsAccountsDB{ - Trie: tr, - Hasher: hasher, - Marshaller: marshaller, - AccountFactory: accountFactory, - StoragePruningManager: disabled.NewDisabledStoragePruningManager(), - AddressConverter: addressConverter, - SnapshotsManager: disabledState.NewDisabledSnapshotsManager(), - StateAccessesCollector: disabledState.NewDisabledStateAccessesCollector(), + Trie: tr, + Hasher: hasher, + Marshaller: marshaller, + AccountFactory: accountFactory, + StoragePruningManager: disabled.NewDisabledStoragePruningManager(), + AddressConverter: addressConverter, + SnapshotsManager: disabledState.NewDisabledSnapshotsManager(), + StateAccessesCollector: disabledState.NewDisabledStateAccessesCollector(), + MaxDataTriesSizeInMemory: common.TenMbSize, } adb, err := state.NewAccountsDB(args) diff --git a/integrationTests/benchmarks/loadFromTrie_test.go b/integrationTests/benchmarks/loadFromTrie_test.go index 8b2d2736b1a..668fd5256d2 100644 --- a/integrationTests/benchmarks/loadFromTrie_test.go +++ b/integrationTests/benchmarks/loadFromTrie_test.go @@ -15,8 +15,8 @@ import ( "github.com/multiversx/mx-chain-go/storage" "github.com/multiversx/mx-chain-go/storage/database" "github.com/multiversx/mx-chain-go/storage/storageunit" + testStorage "github.com/multiversx/mx-chain-go/testscommon/common" "github.com/multiversx/mx-chain-go/testscommon/enableEpochsHandlerMock" - testStorage "github.com/multiversx/mx-chain-go/testscommon/storage" "github.com/multiversx/mx-chain-go/trie" "github.com/stretchr/testify/require" ) @@ -101,7 +101,7 @@ func generateTriesWithMaxDepth( ) []*keyForTrie { tries := make([]*keyForTrie, numTries) for i := 0; i < numTries; i++ { - tr, _ := trie.NewTrie(storage, marshaller, hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, 2) + tr, _ := trie.NewTrie(storage, marshaller, hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, integrationTests.TenMbSize) key := insertKeysIntoTrie(t, tr, numTrieLevels, numChildrenPerBranch) rootHash, _ := tr.RootHash() diff --git a/integrationTests/longTests/storage/storage_test.go b/integrationTests/longTests/storage/storage_test.go index bea274856d8..b6a68d5bd19 100644 --- a/integrationTests/longTests/storage/storage_test.go +++ b/integrationTests/longTests/storage/storage_test.go @@ -9,8 +9,8 @@ import ( "github.com/multiversx/mx-chain-core-go/hashing/blake2b" "github.com/multiversx/mx-chain-core-go/marshal" "github.com/multiversx/mx-chain-go/integrationTests" + testCommon "github.com/multiversx/mx-chain-go/testscommon/common" "github.com/multiversx/mx-chain-go/testscommon/enableEpochsHandlerMock" - "github.com/multiversx/mx-chain-go/testscommon/storage" "github.com/multiversx/mx-chain-go/trie" "github.com/stretchr/testify/assert" ) @@ -106,18 +106,17 @@ func TestWriteContinuouslyInTree(t *testing.T) { nbTxsWrite := 1000000 testStorage := integrationTests.NewTestStorage() store := testStorage.CreateStorageLevelDB() - storageManagerArgs := storage.GetStorageManagerArgs() + storageManagerArgs := testCommon.GetStorageManagerArgs() storageManagerArgs.MainStorer = store storageManagerArgs.Marshalizer = &marshal.JsonMarshalizer{} storageManagerArgs.Hasher = blake2b.NewBlake2b() - options := storage.GetStorageManagerOptions() + options := testCommon.GetStorageManagerOptions() options.PruningEnabled = false trieStorage, _ := trie.CreateTrieStorageManager(storageManagerArgs, options) - maxTrieLevelInMemory := uint(5) - tr, _ := trie.NewTrie(trieStorage, &marshal.JsonMarshalizer{}, blake2b.NewBlake2b(), &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, maxTrieLevelInMemory) + tr, _ := trie.NewTrie(trieStorage, &marshal.JsonMarshalizer{}, blake2b.NewBlake2b(), &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, integrationTests.TenMbSize) defer func() { _ = store.DestroyUnit() diff --git a/integrationTests/multiShard/hardFork/hardFork_test.go b/integrationTests/multiShard/hardFork/hardFork_test.go index 234dd81f063..06477abd3a2 100644 --- a/integrationTests/multiShard/hardFork/hardFork_test.go +++ b/integrationTests/multiShard/hardFork/hardFork_test.go @@ -656,7 +656,6 @@ func createHardForkExporter( }, ExportStateStorageConfig: exportConfig, ExportStateKeysConfig: keysConfig, - MaxTrieLevelInMemory: uint(5), WhiteListHandler: node.WhiteListHandler, WhiteListerVerifiedTxs: node.WhiteListerVerifiedTxs, MainInterceptorsContainer: node.MainInterceptorsContainer, diff --git a/integrationTests/state/stateTrie/stateTrie_test.go b/integrationTests/state/stateTrie/stateTrie_test.go index befb38a5d57..005125a2ea4 100644 --- a/integrationTests/state/stateTrie/stateTrie_test.go +++ b/integrationTests/state/stateTrie/stateTrie_test.go @@ -25,6 +25,7 @@ import ( "github.com/multiversx/mx-chain-core-go/hashing/sha256" crypto "github.com/multiversx/mx-chain-crypto-go" "github.com/multiversx/mx-chain-go/epochStart/notifier" + testCommon "github.com/multiversx/mx-chain-go/testscommon/common" trieMock "github.com/multiversx/mx-chain-go/testscommon/trie" "github.com/multiversx/mx-chain-storage-go/types" vmcommon "github.com/multiversx/mx-chain-vm-common-go" @@ -54,7 +55,6 @@ import ( "github.com/multiversx/mx-chain-go/testscommon/enableEpochsHandlerMock" "github.com/multiversx/mx-chain-go/testscommon/marshallerMock" stateMock "github.com/multiversx/mx-chain-go/testscommon/state" - testStorage "github.com/multiversx/mx-chain-go/testscommon/storage" "github.com/multiversx/mx-chain-go/trie" ) @@ -271,13 +271,12 @@ func TestAccountsDB_CommitTwoOkAccountsShouldWork(t *testing.T) { func TestTrieDB_RecreateFromStorageShouldWork(t *testing.T) { hasher := integrationTests.TestHasher store := integrationTests.CreateMemUnit() - args := testStorage.GetStorageManagerArgs() + args := testCommon.GetStorageManagerArgs() args.MainStorer = store args.Hasher = hasher trieStorage, _ := trie.NewTrieStorageManager(args) - maxTrieLevelInMemory := uint(5) - tr1, _ := trie.NewTrie(trieStorage, integrationTests.TestMarshalizer, hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, maxTrieLevelInMemory) + tr1, _ := trie.NewTrie(trieStorage, integrationTests.TestMarshalizer, hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, integrationTests.TenMbSize) key := hasher.Compute("key") value := hasher.Compute("value") @@ -1056,11 +1055,10 @@ func createAccounts( HashesSize: evictionWaitListSize * 100, } ewl, _ := evictionWaitingList.NewMemoryEvictionWaitingList(ewlArgs) - args := testStorage.GetStorageManagerArgs() + args := testCommon.GetStorageManagerArgs() args.MainStorer = store trieStorage, _ := trie.NewTrieStorageManager(args) - maxTrieLevelInMemory := uint(5) - tr, _ := trie.NewTrie(trieStorage, integrationTests.TestMarshalizer, integrationTests.TestHasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, maxTrieLevelInMemory) + tr, _ := trie.NewTrie(trieStorage, integrationTests.TestMarshalizer, integrationTests.TestHasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, integrationTests.TenMbSize) spm, _ := storagePruningManager.NewStoragePruningManager(ewl, 10) argsAccCreator := factory.ArgsAccountCreator{ Hasher: integrationTests.TestHasher, @@ -1081,14 +1079,15 @@ func createAccounts( StateStatsHandler: statistics.NewStateStatistics(), }) argsAccountsDB := state.ArgsAccountsDB{ - Trie: tr, - Hasher: integrationTests.TestHasher, - Marshaller: integrationTests.TestMarshalizer, - AccountFactory: accCreator, - StoragePruningManager: spm, - AddressConverter: &testscommon.PubkeyConverterMock{}, - SnapshotsManager: snapshotsManager, - StateAccessesCollector: &stateMock.StateAccessesCollectorStub{}, + Trie: tr, + Hasher: integrationTests.TestHasher, + Marshaller: integrationTests.TestMarshalizer, + AccountFactory: accCreator, + StoragePruningManager: spm, + AddressConverter: &testscommon.PubkeyConverterMock{}, + SnapshotsManager: snapshotsManager, + StateAccessesCollector: &stateMock.StateAccessesCollectorStub{}, + MaxDataTriesSizeInMemory: integrationTests.TenMbSize, } adb, _ := state.NewAccountsDB(argsAccountsDB) @@ -2733,11 +2732,10 @@ func createAccountsDBTestSetup() *state.AccountsDB { HashesSize: evictionWaitListSize * 100, } ewl, _ := evictionWaitingList.NewMemoryEvictionWaitingList(ewlArgs) - args := testStorage.GetStorageManagerArgs() + args := testCommon.GetStorageManagerArgs() args.GeneralConfig = generalCfg trieStorage, _ := trie.NewTrieStorageManager(args) - maxTrieLevelInMemory := uint(5) - tr, _ := trie.NewTrie(trieStorage, integrationTests.TestMarshalizer, integrationTests.TestHasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, maxTrieLevelInMemory) + tr, _ := trie.NewTrie(trieStorage, integrationTests.TestMarshalizer, integrationTests.TestHasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, integrationTests.TenMbSize) spm, _ := storagePruningManager.NewStoragePruningManager(ewl, 10) argsAccCreator := factory.ArgsAccountCreator{ Hasher: integrationTests.TestHasher, @@ -2760,14 +2758,15 @@ func createAccountsDBTestSetup() *state.AccountsDB { }) argsAccountsDB := state.ArgsAccountsDB{ - Trie: tr, - Hasher: integrationTests.TestHasher, - Marshaller: integrationTests.TestMarshalizer, - AccountFactory: accCreator, - StoragePruningManager: spm, - AddressConverter: &testscommon.PubkeyConverterMock{}, - SnapshotsManager: snapshotsManager, - StateAccessesCollector: &stateMock.StateAccessesCollectorStub{}, + Trie: tr, + Hasher: integrationTests.TestHasher, + Marshaller: integrationTests.TestMarshalizer, + AccountFactory: accCreator, + StoragePruningManager: spm, + AddressConverter: &testscommon.PubkeyConverterMock{}, + SnapshotsManager: snapshotsManager, + StateAccessesCollector: &stateMock.StateAccessesCollectorStub{}, + MaxDataTriesSizeInMemory: integrationTests.TenMbSize, } adb, _ := state.NewAccountsDB(argsAccountsDB) @@ -2789,11 +2788,10 @@ func TestStateSnapshot_MultipleEpochsWithoutCompleteSnapshot(t *testing.T) { numEpochs := 5 getCounters := make([]int, numEpochs) - args := testStorage.GetStorageManagerArgs() + args := testCommon.GetStorageManagerArgs() args.MainStorer = mainStorer trieStorage, _ := trie.NewTrieStorageManager(args) - maxTrieLevelInMemory := uint(5) - tr, _ := trie.NewTrie(trieStorage, args.Marshalizer, args.Hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, maxTrieLevelInMemory) + tr, _ := trie.NewTrie(trieStorage, args.Marshalizer, args.Hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, integrationTests.TenMbSize) defer func() { _ = trieStorage.Close() }() diff --git a/integrationTests/state/stateTrieClose/stateTrieClose_test.go b/integrationTests/state/stateTrieClose/stateTrieClose_test.go index 9d99a178484..8a638c8a32a 100644 --- a/integrationTests/state/stateTrieClose/stateTrieClose_test.go +++ b/integrationTests/state/stateTrieClose/stateTrieClose_test.go @@ -12,9 +12,9 @@ import ( "github.com/multiversx/mx-chain-go/common/errChan" "github.com/multiversx/mx-chain-go/integrationTests" "github.com/multiversx/mx-chain-go/state/parsers" + testCommon "github.com/multiversx/mx-chain-go/testscommon/common" "github.com/multiversx/mx-chain-go/testscommon/enableEpochsHandlerMock" "github.com/multiversx/mx-chain-go/testscommon/goroutines" - "github.com/multiversx/mx-chain-go/testscommon/storage" "github.com/multiversx/mx-chain-go/trie" "github.com/multiversx/mx-chain-go/trie/keyBuilder" "github.com/stretchr/testify/assert" @@ -23,7 +23,7 @@ import ( func TestPatriciaMerkleTrie_Close(t *testing.T) { numLeavesToAdd := 200 trieStorage, _ := integrationTests.CreateTrieStorageManager(integrationTests.CreateMemUnit()) - tr, _ := trie.NewTrie(trieStorage, integrationTests.TestMarshalizer, integrationTests.TestHasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, 5) + tr, _ := trie.NewTrie(trieStorage, integrationTests.TestMarshalizer, integrationTests.TestHasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, integrationTests.TenMbSize) for i := 0; i < numLeavesToAdd; i++ { _ = tr.Update([]byte(strconv.Itoa(i)), []byte(strconv.Itoa(i))) @@ -139,7 +139,7 @@ func TestPatriciaMerkleTrie_Close(t *testing.T) { } func TestTrieStorageManager_Close(t *testing.T) { - args := storage.GetStorageManagerArgs() + args := testCommon.GetStorageManagerArgs() gc := goroutines.NewGoCounter(goroutines.TestsRelevantGoRoutines) idxInitial, _ := gc.Snapshot() diff --git a/integrationTests/state/stateTrieSync/stateTrieSync_test.go b/integrationTests/state/stateTrieSync/stateTrieSync_test.go index cd416f663c5..2c7729f9058 100644 --- a/integrationTests/state/stateTrieSync/stateTrieSync_test.go +++ b/integrationTests/state/stateTrieSync/stateTrieSync_test.go @@ -609,7 +609,6 @@ func getUserAccountSyncerArgs(node *integrationTests.TestProcessorNode, version RequestHandler: node.RequestHandler, Timeout: common.TimeoutGettingTrieNodes, Cacher: node.DataPool.TrieNodes(), - MaxTrieLevelInMemory: 200, MaxHardCapForMissingNodes: 5000, TrieSyncerVersion: version, UserAccountsSyncStatisticsHandler: statistics.NewTrieSyncStatistics(), diff --git a/integrationTests/testInitializer.go b/integrationTests/testInitializer.go index 8e1d8969632..cd19b7dff18 100644 --- a/integrationTests/testInitializer.go +++ b/integrationTests/testInitializer.go @@ -81,7 +81,6 @@ import ( "github.com/multiversx/mx-chain-go/testscommon/stakingcommon" testStorage "github.com/multiversx/mx-chain-go/testscommon/state" statusHandlerMock "github.com/multiversx/mx-chain-go/testscommon/statusHandler" - testcommonStorage "github.com/multiversx/mx-chain-go/testscommon/storage" "github.com/multiversx/mx-chain-go/testscommon/txDataBuilder" "github.com/multiversx/mx-chain-go/trie" "github.com/multiversx/mx-chain-go/vm" @@ -104,6 +103,9 @@ var InitialRating = uint32(50) // AdditionalGasLimit is the value that can be added on a transaction in the GasLimit var AdditionalGasLimit = uint64(999000) +// TenMbSize represents 10 MB in bytes +const TenMbSize = uint64(10485760) + // GasSchedulePath -- const GasSchedulePath = "../../../../cmd/node/config/gasSchedules/gasScheduleV4.toml" @@ -114,7 +116,6 @@ const ( shuffleBetweenShards = false adaptivity = false hysteresis = float32(0.2) - maxTrieLevelInMemory = uint(5) delegationContractsList = "delegationContracts" ) @@ -431,7 +432,7 @@ func CreateTrieStorageManagerWithPruningStorer(coordinator sharding.Coordinator, fmt.Println("err creating main storer" + err.Error()) } - args := testcommonStorage.GetStorageManagerArgs() + args := commonMocks.GetStorageManagerArgs() args.MainStorer = mainStorer args.Marshalizer = TestMarshalizer args.Hasher = TestHasher @@ -443,7 +444,7 @@ func CreateTrieStorageManagerWithPruningStorer(coordinator sharding.Coordinator, // CreateTrieStorageManager creates the trie storage manager for the tests func CreateTrieStorageManager(store storage.Storer) (common.StorageManager, storage.Storer) { - args := testcommonStorage.GetStorageManagerArgs() + args := commonMocks.GetStorageManagerArgs() args.MainStorer = store args.Marshalizer = TestMarshalizer args.Hasher = TestHasher @@ -467,7 +468,7 @@ func CreateAccountsDBWithEnableEpochsHandler( trieStorageManager common.StorageManager, enableEpochsHandler common.EnableEpochsHandler, ) (*state.AccountsDB, common.Trie) { - tr, _ := trie.NewTrie(trieStorageManager, TestMarshalizer, TestHasher, enableEpochsHandler, maxTrieLevelInMemory) + tr, _ := trie.NewTrie(trieStorageManager, TestMarshalizer, TestHasher, enableEpochsHandler, TenMbSize) ewlArgs := evictionWaitingList.MemoryEvictionWaitingListArgs{ RootHashesSize: 100, @@ -491,15 +492,16 @@ func CreateAccountsDBWithEnableEpochsHandler( _ = snapshotsManager.SetSyncer(&mock.AccountsDBSyncerStub{}) args := state.ArgsAccountsDB{ - Trie: tr, - Hasher: sha256.NewSha256(), - Marshaller: TestMarshalizer, - AccountFactory: accountFactory, - StoragePruningManager: spm, - AddressConverter: &testscommon.PubkeyConverterMock{}, - SnapshotsManager: snapshotsManager, - StateAccessesCollector: disabled.NewDisabledStateAccessesCollector(), - PruningEnabled: trieStorageManager.IsPruningEnabled(), + Trie: tr, + Hasher: sha256.NewSha256(), + Marshaller: TestMarshalizer, + AccountFactory: accountFactory, + StoragePruningManager: spm, + AddressConverter: &testscommon.PubkeyConverterMock{}, + SnapshotsManager: snapshotsManager, + StateAccessesCollector: disabled.NewDisabledStateAccessesCollector(), + PruningEnabled: trieStorageManager.IsPruningEnabled(), + MaxDataTriesSizeInMemory: TenMbSize, } adb, _ := state.NewAccountsDB(args) @@ -1158,13 +1160,13 @@ func CreateSimpleTxProcessor(accnts state.AccountsAdapter) process.TransactionPr // CreateNewDefaultTrie returns a new trie with test hasher and marsahalizer func CreateNewDefaultTrie() common.Trie { - args := testcommonStorage.GetStorageManagerArgs() + args := commonMocks.GetStorageManagerArgs() args.Marshalizer = TestMarshalizer args.Hasher = TestHasher trieStorage, _ := trie.NewTrieStorageManager(args) - tr, _ := trie.NewTrie(trieStorage, TestMarshalizer, TestHasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, maxTrieLevelInMemory) + tr, _ := trie.NewTrie(trieStorage, TestMarshalizer, TestHasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, TenMbSize) return tr } diff --git a/integrationTests/testProcessorNode.go b/integrationTests/testProcessorNode.go index 0a4f06b6d14..8b510a6e9d6 100644 --- a/integrationTests/testProcessorNode.go +++ b/integrationTests/testProcessorNode.go @@ -31,6 +31,7 @@ import ( ed25519SingleSig "github.com/multiversx/mx-chain-crypto-go/signing/ed25519/singlesig" "github.com/multiversx/mx-chain-crypto-go/signing/mcl" mclsig "github.com/multiversx/mx-chain-crypto-go/signing/mcl/singlesig" + "github.com/multiversx/mx-chain-go/state/triesHolder" vmcommon "github.com/multiversx/mx-chain-vm-common-go" "github.com/multiversx/mx-chain-vm-common-go/parsers" wasmConfig "github.com/multiversx/mx-chain-vm-go/config" @@ -705,7 +706,7 @@ func (tpn *TestProcessorNode) initAccountDBsWithPruningStorer() { tpn.EpochStartNotifier = notifier.NewEpochStartSubscriptionHandler() } trieStorageManager := CreateTrieStorageManagerWithPruningStorer(tpn.ShardCoordinator, tpn.EpochStartNotifier) - tpn.TrieContainer = state.NewDataTriesHolder() + tpn.TrieContainer = triesHolder.NewTriesHolder() var stateTrie common.Trie tpn.AccntState, stateTrie = CreateAccountsDBWithEnableEpochsHandler(UserAccount, trieStorageManager, tpn.EnableEpochsHandler) tpn.AccntStateProposal, _ = CreateAccountsDBWithEnableEpochsHandler(UserAccount, trieStorageManager, tpn.EnableEpochsHandler) @@ -722,7 +723,7 @@ func (tpn *TestProcessorNode) initAccountDBsWithPruningStorer() { func (tpn *TestProcessorNode) initAccountDBs(store storage.Storer) { trieStorageManager, _ := CreateTrieStorageManager(store) - tpn.TrieContainer = state.NewDataTriesHolder() + tpn.TrieContainer = triesHolder.NewTriesHolder() var stateTrie common.Trie tpn.AccntState, stateTrie = CreateAccountsDBWithEnableEpochsHandler(UserAccount, trieStorageManager, tpn.EnableEpochsHandler) tpn.AccntStateProposal, _ = CreateAccountsDBWithEnableEpochsHandler(UserAccount, trieStorageManager, tpn.EnableEpochsHandler) diff --git a/integrationTests/vm/staking/componentsHolderCreator.go b/integrationTests/vm/staking/componentsHolderCreator.go index d46283b8d8a..226556b1b96 100644 --- a/integrationTests/vm/staking/componentsHolderCreator.go +++ b/integrationTests/vm/staking/componentsHolderCreator.go @@ -209,7 +209,7 @@ func createAccountsDB( coreComponents.InternalMarshalizer(), coreComponents.Hasher(), coreComponents.EnableEpochsHandler(), - 5, + integrationTests.TenMbSize, ) argsEvictionWaitingList := evictionWaitingList.MemoryEvictionWaitingListArgs{ @@ -219,14 +219,15 @@ func createAccountsDB( ewl, _ := evictionWaitingList.NewMemoryEvictionWaitingList(argsEvictionWaitingList) spm, _ := storagePruningManager.NewStoragePruningManager(ewl, 10) argsAccountsDb := state.ArgsAccountsDB{ - Trie: tr, - Hasher: coreComponents.Hasher(), - Marshaller: coreComponents.InternalMarshalizer(), - AccountFactory: accountFactory, - StoragePruningManager: spm, - AddressConverter: coreComponents.AddressPubKeyConverter(), - SnapshotsManager: &stateTests.SnapshotsManagerStub{}, - StateAccessesCollector: disabledState.NewDisabledStateAccessesCollector(), + Trie: tr, + Hasher: coreComponents.Hasher(), + Marshaller: coreComponents.InternalMarshalizer(), + AccountFactory: accountFactory, + StoragePruningManager: spm, + AddressConverter: coreComponents.AddressPubKeyConverter(), + SnapshotsManager: &stateTests.SnapshotsManagerStub{}, + StateAccessesCollector: disabledState.NewDisabledStateAccessesCollector(), + MaxDataTriesSizeInMemory: integrationTests.TenMbSize, } adb, _ := state.NewAccountsDB(argsAccountsDb) return adb diff --git a/node/chainSimulator/components/testOnlyProcessingNode_test.go b/node/chainSimulator/components/testOnlyProcessingNode_test.go index 65f93c45c18..9c63b210613 100644 --- a/node/chainSimulator/components/testOnlyProcessingNode_test.go +++ b/node/chainSimulator/components/testOnlyProcessingNode_test.go @@ -150,7 +150,7 @@ func TestNewTestOnlyProcessingNode(t *testing.T) { t.Run("CreateStateComponents failure should error", func(t *testing.T) { args := createMockArgsTestOnlyProcessingNode(t) args.ShardIDStr = common.MetachainShardName // coverage only - args.Configs.GeneralConfig.StateTriesConfig.MaxStateTrieLevelInMemory = 0 + args.Configs.GeneralConfig.TrieStorageManagerConfig = config.TrieStorageManagerConfig{} node, err := NewTestOnlyProcessingNode(args) require.Error(t, err) require.Nil(t, node) diff --git a/node/nodeRunner.go b/node/nodeRunner.go index 73f37114f61..941b5f5a26f 100644 --- a/node/nodeRunner.go +++ b/node/nodeRunner.go @@ -647,7 +647,6 @@ func getUserAccountSyncer( bootstrapComponents mainFactory.BootstrapComponentsHolder, processComponents mainFactory.ProcessComponentsHolder, ) (process.AccountsDBSyncer, error) { - maxTrieLevelInMemory := config.StateTriesConfig.MaxStateTrieLevelInMemory userTrie := stateComponents.TriesContainer().Get([]byte(dataRetriever.UserAccountsUnit.String())) storageManager := userTrie.GetStorageManager() @@ -663,7 +662,6 @@ func getUserAccountSyncer( dataComponents, processComponents, storageManager, - maxTrieLevelInMemory, ), ShardId: bootstrapComponents.ShardCoordinator().SelfId(), Throttler: thr, @@ -680,7 +678,6 @@ func getValidatorAccountSyncer( stateComponents mainFactory.StateComponentsHolder, processComponents mainFactory.ProcessComponentsHolder, ) (process.AccountsDBSyncer, error) { - maxTrieLevelInMemory := config.StateTriesConfig.MaxPeerTrieLevelInMemory peerTrie := stateComponents.TriesContainer().Get([]byte(dataRetriever.PeerAccountsUnit.String())) storageManager := peerTrie.GetStorageManager() @@ -691,7 +688,6 @@ func getValidatorAccountSyncer( dataComponents, processComponents, storageManager, - maxTrieLevelInMemory, ), } @@ -704,7 +700,6 @@ func getBaseAccountSyncerArgs( dataComponents mainFactory.DataComponentsHolder, processComponents mainFactory.ProcessComponentsHolder, storageManager common.StorageManager, - maxTrieLevelInMemory uint, ) syncer.ArgsNewBaseAccountsSyncer { return syncer.ArgsNewBaseAccountsSyncer{ Hasher: coreComponents.Hasher(), @@ -713,7 +708,6 @@ func getBaseAccountSyncerArgs( RequestHandler: processComponents.RequestHandler(), Timeout: common.TimeoutGettingTrieNodes, Cacher: dataComponents.Datapool().TrieNodes(), - MaxTrieLevelInMemory: maxTrieLevelInMemory, MaxHardCapForMissingNodes: config.TrieSync.MaxHardCapForMissingNodes, TrieSyncerVersion: config.TrieSync.TrieSyncerVersion, CheckNodesOnDisk: true, diff --git a/process/sync/baseForkDetector_test.go b/process/sync/baseForkDetector_test.go index 2fa476de1df..c140471b48f 100644 --- a/process/sync/baseForkDetector_test.go +++ b/process/sync/baseForkDetector_test.go @@ -5,9 +5,6 @@ import ( "testing" "time" - "github.com/multiversx/mx-chain-go/testscommon/chainParameters" - "github.com/multiversx/mx-chain-go/testscommon/processMocks" - "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-core-go/data/block" @@ -17,8 +14,10 @@ import ( "github.com/multiversx/mx-chain-go/process/mock" "github.com/multiversx/mx-chain-go/process/sync" "github.com/multiversx/mx-chain-go/testscommon" + "github.com/multiversx/mx-chain-go/testscommon/chainParameters" "github.com/multiversx/mx-chain-go/testscommon/dataRetriever" "github.com/multiversx/mx-chain-go/testscommon/enableEpochsHandlerMock" + "github.com/multiversx/mx-chain-go/testscommon/processMocks" "github.com/stretchr/testify/assert" ) diff --git a/state/accountsDB.go b/state/accountsDB.go index b412bac97d6..093005a9bf8 100644 --- a/state/accountsDB.go +++ b/state/accountsDB.go @@ -23,6 +23,7 @@ import ( "github.com/multiversx/mx-chain-go/common/errChan" "github.com/multiversx/mx-chain-go/common/holders" "github.com/multiversx/mx-chain-go/state/parsers" + "github.com/multiversx/mx-chain-go/state/triesHolder" "github.com/multiversx/mx-chain-go/trie/keyBuilder" "github.com/multiversx/mx-chain-go/trie/statistics" ) @@ -98,15 +99,16 @@ var log = logger.GetOrCreate("state") // ArgsAccountsDB is the arguments DTO for the AccountsDB instance type ArgsAccountsDB struct { - Trie common.Trie - Hasher hashing.Hasher - Marshaller marshal.Marshalizer - AccountFactory AccountFactory - StoragePruningManager StoragePruningManager - AddressConverter core.PubkeyConverter - SnapshotsManager SnapshotsManager - StateAccessesCollector StateAccessesCollector - PruningEnabled bool + Trie common.Trie + Hasher hashing.Hasher + Marshaller marshal.Marshalizer + AccountFactory AccountFactory + StoragePruningManager StoragePruningManager + AddressConverter core.PubkeyConverter + SnapshotsManager SnapshotsManager + StateAccessesCollector StateAccessesCollector + PruningEnabled bool + MaxDataTriesSizeInMemory uint64 } // NewAccountsDB creates a new account manager @@ -116,10 +118,15 @@ func NewAccountsDB(args ArgsAccountsDB) (*AccountsDB, error) { return nil, err } - return createAccountsDb(args), nil + return createAccountsDb(args) } -func createAccountsDb(args ArgsAccountsDB) *AccountsDB { +func createAccountsDb(args ArgsAccountsDB) (*AccountsDB, error) { + dth, err := triesHolder.NewDataTriesHolder(args.MaxDataTriesSizeInMemory) + if err != nil { + return nil, fmt.Errorf("cannot create data tries holder: %w", err) + } + return &AccountsDB{ mainTrie: args.Trie, hasher: args.Hasher, @@ -128,7 +135,7 @@ func createAccountsDb(args ArgsAccountsDB) *AccountsDB { storagePruningManager: args.StoragePruningManager, entries: make([]JournalEntry, 0), mutOp: sync.RWMutex{}, - dataTries: NewDataTriesHolder(), + dataTries: dth, obsoleteDataTrieHashes: make(map[string][][]byte), loadCodeMeasurements: &loadingMeasurements{ identifier: "load code", @@ -137,7 +144,7 @@ func createAccountsDb(args ArgsAccountsDB) *AccountsDB { snapshotsManger: args.SnapshotsManager, stateAccessesCollector: args.StateAccessesCollector, pruningEnabled: args.PruningEnabled, - } + }, nil } func checkArgsAccountsDB(args ArgsAccountsDB) error { diff --git a/state/accountsDB_test.go b/state/accountsDB_test.go index 2f2125c7737..103952a18b3 100644 --- a/state/accountsDB_test.go +++ b/state/accountsDB_test.go @@ -41,17 +41,20 @@ import ( "github.com/multiversx/mx-chain-go/state/storagePruningManager/disabled" "github.com/multiversx/mx-chain-go/state/storagePruningManager/evictionWaitingList" "github.com/multiversx/mx-chain-go/testscommon" + testCommon "github.com/multiversx/mx-chain-go/testscommon/common" "github.com/multiversx/mx-chain-go/testscommon/enableEpochsHandlerMock" "github.com/multiversx/mx-chain-go/testscommon/hashingMocks" "github.com/multiversx/mx-chain-go/testscommon/marshallerMock" stateMock "github.com/multiversx/mx-chain-go/testscommon/state" - "github.com/multiversx/mx-chain-go/testscommon/storage" "github.com/multiversx/mx-chain-go/testscommon/storageManager" trieMock "github.com/multiversx/mx-chain-go/testscommon/trie" "github.com/multiversx/mx-chain-go/trie" ) -const trieDbOperationDelay = time.Second +const ( + trieDbOperationDelay = time.Second + tenMbSize = uint64(10485760) +) func createMockAccountsDBArgs() state.ArgsAccountsDB { accCreator := &stateMock.AccountsFactoryStub{ @@ -78,13 +81,14 @@ func createMockAccountsDBArgs() state.ArgsAccountsDB { return &storageManager.StorageManagerStub{} }, }, - Hasher: &hashingMocks.HasherMock{}, - Marshaller: &marshallerMock.MarshalizerMock{}, - AccountFactory: accCreator, - StoragePruningManager: disabled.NewDisabledStoragePruningManager(), - AddressConverter: &testscommon.PubkeyConverterMock{}, - SnapshotsManager: snapshotsManager, - StateAccessesCollector: &stateMock.StateAccessesCollectorStub{}, + Hasher: &hashingMocks.HasherMock{}, + Marshaller: &marshallerMock.MarshalizerMock{}, + AccountFactory: accCreator, + StoragePruningManager: disabled.NewDisabledStoragePruningManager(), + AddressConverter: &testscommon.PubkeyConverterMock{}, + SnapshotsManager: snapshotsManager, + StateAccessesCollector: &stateMock.StateAccessesCollectorStub{}, + MaxDataTriesSizeInMemory: tenMbSize, } } @@ -141,10 +145,10 @@ func getDefaultStateComponents( marshaller := &marshallerMock.MarshalizerMock{} hasher := &hashingMocks.HasherMock{} - args := storage.GetStorageManagerArgs() + args := testCommon.GetStorageManagerArgs() args.MainStorer = db trieStorage, _ := trie.NewTrieStorageManager(args) - tr, _ := trie.NewTrie(trieStorage, marshaller, hasher, enableEpochsHandler, 5) + tr, _ := trie.NewTrie(trieStorage, marshaller, hasher, enableEpochsHandler, tenMbSize) ewlArgs := evictionWaitingList.MemoryEvictionWaitingListArgs{ RootHashesSize: 100, HashesSize: 10000, @@ -173,15 +177,16 @@ func getDefaultStateComponents( collector, _ := stateAccesses.NewCollector(stateDisabled.NewDisabledStateAccessesStorer(), stateAccesses.WithCollectWrite()) argsAccountsDB := state.ArgsAccountsDB{ - Trie: tr, - Hasher: hasher, - Marshaller: marshaller, - AccountFactory: accCreator, - StoragePruningManager: spm, - AddressConverter: &testscommon.PubkeyConverterMock{}, - SnapshotsManager: snapshotsManager, - StateAccessesCollector: collector, - PruningEnabled: true, + Trie: tr, + Hasher: hasher, + Marshaller: marshaller, + AccountFactory: accCreator, + StoragePruningManager: spm, + AddressConverter: &testscommon.PubkeyConverterMock{}, + SnapshotsManager: snapshotsManager, + StateAccessesCollector: collector, + PruningEnabled: true, + MaxDataTriesSizeInMemory: tenMbSize, } adb, _ := state.NewAccountsDB(argsAccountsDB) @@ -2126,10 +2131,9 @@ func TestAccountsDB_MainTrieAutomaticallyMarksCodeUpdatesForEviction(t *testing. marshaller := &marshallerMock.MarshalizerMock{} hasher := &hashingMocks.HasherMock{} ewl := stateMock.NewEvictionWaitingListMock(100) - args := storage.GetStorageManagerArgs() + args := testCommon.GetStorageManagerArgs() tsm, _ := trie.NewTrieStorageManager(args) - maxTrieLevelInMemory := uint(5) - tr, _ := trie.NewTrie(tsm, marshaller, hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, maxTrieLevelInMemory) + tr, _ := trie.NewTrie(tsm, marshaller, hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, tenMbSize) spm, _ := storagePruningManager.NewStoragePruningManager(ewl, 5) argsAccountsDB := createMockAccountsDBArgs() @@ -2205,14 +2209,13 @@ func TestAccountsDB_RemoveAccountSetsObsoleteHashes(t *testing.T) { func TestAccountsDB_RemoveAccountMarksObsoleteHashesForEviction(t *testing.T) { t.Parallel() - maxTrieLevelInMemory := uint(5) marshaller := &marshallerMock.MarshalizerMock{} hasher := &hashingMocks.HasherMock{} ewl := stateMock.NewEvictionWaitingListMock(100) - args := storage.GetStorageManagerArgs() + args := testCommon.GetStorageManagerArgs() tsm, _ := trie.NewTrieStorageManager(args) - tr, _ := trie.NewTrie(tsm, marshaller, hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, maxTrieLevelInMemory) + tr, _ := trie.NewTrie(tsm, marshaller, hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, tenMbSize) spm, _ := storagePruningManager.NewStoragePruningManager(ewl, 5) argsAccountsDB := createMockAccountsDBArgs() @@ -2411,13 +2414,12 @@ func modifyDataTries(t *testing.T, accountsAddresses [][]byte, adb *state.Accoun func TestAccountsDB_GetCode(t *testing.T) { t.Parallel() - maxTrieLevelInMemory := uint(5) marshaller := &marshallerMock.MarshalizerMock{} hasher := &hashingMocks.HasherMock{} - args := storage.GetStorageManagerArgs() + args := testCommon.GetStorageManagerArgs() tsm, _ := trie.NewTrieStorageManager(args) - tr, _ := trie.NewTrie(tsm, marshaller, hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, maxTrieLevelInMemory) + tr, _ := trie.NewTrie(tsm, marshaller, hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, tenMbSize) spm := disabled.NewDisabledStoragePruningManager() argsAccountsDB := createMockAccountsDBArgs() argsAccountsDB.Trie = tr @@ -2867,13 +2869,12 @@ func TestAccountsDB_NewAccountsDbStartsSnapshotAfterRestart(t *testing.T) { } func BenchmarkAccountsDb_GetCodeEntry(b *testing.B) { - maxTrieLevelInMemory := uint(5) marshaller := &marshallerMock.MarshalizerMock{} hasher := &hashingMocks.HasherMock{} - args := storage.GetStorageManagerArgs() + args := testCommon.GetStorageManagerArgs() tsm, _ := trie.NewTrieStorageManager(args) - tr, _ := trie.NewTrie(tsm, marshaller, hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, maxTrieLevelInMemory) + tr, _ := trie.NewTrie(tsm, marshaller, hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, tenMbSize) spm := disabled.NewDisabledStoragePruningManager() argsAccountsDB := createMockAccountsDBArgs() @@ -3196,8 +3197,8 @@ func TestAccountsDB_RevertTxWhichMigratesDataRemovesMigratedData(t *testing.T) { marshaller := &marshallerMock.MarshalizerMock{} hasher := &hashingMocks.HasherMock{} enableEpochsHandler := enableEpochsHandlerMock.NewEnableEpochsHandlerStub() - tsm, _ := trie.NewTrieStorageManager(storage.GetStorageManagerArgs()) - tr, _ := trie.NewTrie(tsm, marshaller, hasher, enableEpochsHandler, uint(5)) + tsm, _ := trie.NewTrieStorageManager(testCommon.GetStorageManagerArgs()) + tr, _ := trie.NewTrie(tsm, marshaller, hasher, enableEpochsHandler, tenMbSize) spm := &stateMock.StoragePruningManagerStub{} argsAccountsDB := createMockAccountsDBArgs() argsAccountsDB.PruningEnabled = true diff --git a/state/dataTriesHolder_test.go b/state/dataTriesHolder_test.go deleted file mode 100644 index 8e65f1bb3b2..00000000000 --- a/state/dataTriesHolder_test.go +++ /dev/null @@ -1,119 +0,0 @@ -package state_test - -import ( - "strconv" - "sync" - "testing" - - "github.com/multiversx/mx-chain-core-go/core/check" - "github.com/multiversx/mx-chain-go/state" - trieMock "github.com/multiversx/mx-chain-go/testscommon/trie" - "github.com/stretchr/testify/assert" -) - -func TestNewDataTriesHolder(t *testing.T) { - t.Parallel() - - dth := state.NewDataTriesHolder() - assert.False(t, check.IfNil(dth)) -} - -func TestDataTriesHolder_PutAndGet(t *testing.T) { - t.Parallel() - - tr1 := &trieMock.TrieStub{} - - dth := state.NewDataTriesHolder() - dth.Put([]byte("trie1"), tr1) - tr := dth.Get([]byte("trie1")) - - assert.True(t, tr == tr1) -} - -func TestDataTriesHolder_Replace(t *testing.T) { - t.Parallel() - - tr1 := &trieMock.TrieStub{} - tr2 := &trieMock.TrieStub{} - - dth := state.NewDataTriesHolder() - dth.Put([]byte("trie1"), tr1) - dth.Replace([]byte("trie1"), tr2) - retrievedTrie := dth.Get([]byte("trie1")) - - assert.True(t, retrievedTrie == tr2) - assert.True(t, retrievedTrie != tr1) -} - -func TestDataTriesHolder_GetAll(t *testing.T) { - t.Parallel() - - tr1 := &trieMock.TrieStub{} - tr2 := &trieMock.TrieStub{} - tr3 := &trieMock.TrieStub{} - - dth := state.NewDataTriesHolder() - dth.Put([]byte("trie1"), tr1) - dth.Put([]byte("trie2"), tr2) - dth.Put([]byte("trie3"), tr3) - tries := dth.GetAll() - - assert.Equal(t, 3, len(tries)) -} - -func TestDataTriesHolder_Reset(t *testing.T) { - t.Parallel() - - tr1 := &trieMock.TrieStub{} - - dth := state.NewDataTriesHolder() - dth.Put([]byte("trie1"), tr1) - dth.Reset() - - tr := dth.Get([]byte("trie1")) - assert.Nil(t, tr) -} - -func TestDataTriesHolder_Concurrency(t *testing.T) { - t.Parallel() - - dth := state.NewDataTriesHolder() - numTries := 50 - - wg := sync.WaitGroup{} - wg.Add(numTries) - - for i := 0; i < numTries; i++ { - go func(key int) { - dth.Put([]byte(strconv.Itoa(key)), &trieMock.TrieStub{}) - wg.Done() - }(i) - } - - wg.Wait() - - tries := dth.GetAll() - assert.Equal(t, numTries, len(tries)) -} - -func TestDataTriesHolder_GetAllTries(t *testing.T) { - t.Parallel() - - dth := state.NewDataTriesHolder() - numTries := 50 - - wg := sync.WaitGroup{} - wg.Add(numTries) - - for i := 0; i < numTries; i++ { - go func(key int) { - dth.Put([]byte(strconv.Itoa(key)), &trieMock.TrieStub{}) - wg.Done() - }(i) - } - - wg.Wait() - - tries := dth.GetAllTries() - assert.Equal(t, numTries, len(tries)) -} diff --git a/state/factory/accountsAdapterAPICreator_test.go b/state/factory/accountsAdapterAPICreator_test.go index a735fc7f9ce..0fd272719cb 100644 --- a/state/factory/accountsAdapterAPICreator_test.go +++ b/state/factory/accountsAdapterAPICreator_test.go @@ -17,19 +17,21 @@ import ( ) func createMockAccountsArgs() state.ArgsAccountsDB { + tenMbSize := uint64(10 * 1024 * 1024) return state.ArgsAccountsDB{ Trie: &mockTrie.TrieStub{ GetStorageManagerCalled: func() common.StorageManager { return &storageManager.StorageManagerStub{} }, }, - Hasher: &testscommon.HasherStub{}, - Marshaller: &marshallerMock.MarshalizerMock{}, - AccountFactory: &mockState.AccountsFactoryStub{}, - StoragePruningManager: &mockState.StoragePruningManagerStub{}, - AddressConverter: &testscommon.PubkeyConverterMock{}, - SnapshotsManager: &mockState.SnapshotsManagerStub{}, - StateAccessesCollector: &mockState.StateAccessesCollectorStub{}, + Hasher: &testscommon.HasherStub{}, + Marshaller: &marshallerMock.MarshalizerMock{}, + AccountFactory: &mockState.AccountsFactoryStub{}, + StoragePruningManager: &mockState.StoragePruningManagerStub{}, + AddressConverter: &testscommon.PubkeyConverterMock{}, + SnapshotsManager: &mockState.SnapshotsManagerStub{}, + StateAccessesCollector: &mockState.StateAccessesCollectorStub{}, + MaxDataTriesSizeInMemory: tenMbSize, } } diff --git a/state/peerAccountsDB.go b/state/peerAccountsDB.go index 093e6d3b6e2..8545bdb9ff7 100644 --- a/state/peerAccountsDB.go +++ b/state/peerAccountsDB.go @@ -17,8 +17,13 @@ func NewPeerAccountsDB(args ArgsAccountsDB) (*PeerAccountsDB, error) { return nil, err } + originalAccountsDb, err := createAccountsDb(args) + if err != nil { + return nil, err + } + adb := &PeerAccountsDB{ - AccountsDB: createAccountsDb(args), + AccountsDB: originalAccountsDb, } return adb, nil diff --git a/state/storagePruningManager/storagePruningManager_test.go b/state/storagePruningManager/storagePruningManager_test.go index 006a88e4d70..cc06ec8024b 100644 --- a/state/storagePruningManager/storagePruningManager_test.go +++ b/state/storagePruningManager/storagePruningManager_test.go @@ -15,11 +15,11 @@ import ( "github.com/multiversx/mx-chain-go/state/lastSnapshotMarker" "github.com/multiversx/mx-chain-go/state/storagePruningManager/evictionWaitingList" "github.com/multiversx/mx-chain-go/testscommon" + testCommon "github.com/multiversx/mx-chain-go/testscommon/common" "github.com/multiversx/mx-chain-go/testscommon/enableEpochsHandlerMock" "github.com/multiversx/mx-chain-go/testscommon/hashingMocks" "github.com/multiversx/mx-chain-go/testscommon/marshallerMock" stateMock "github.com/multiversx/mx-chain-go/testscommon/state" - "github.com/multiversx/mx-chain-go/testscommon/storage" "github.com/multiversx/mx-chain-go/trie" ) @@ -31,9 +31,10 @@ func getDefaultTrieAndAccountsDbAndStoragePruningManager() (common.Trie, *state. } marshaller := &marshallerMock.MarshalizerMock{} hasher := &hashingMocks.HasherMock{} - args := storage.GetStorageManagerArgs() + args := testCommon.GetStorageManagerArgs() trieStorage, _ := trie.NewTrieStorageManager(args) - tr, _ := trie.NewTrie(trieStorage, marshaller, hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, 5) + tenMBSize := uint64(10485760) + tr, _ := trie.NewTrie(trieStorage, marshaller, hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, tenMBSize) ewlArgs := evictionWaitingList.MemoryEvictionWaitingListArgs{ RootHashesSize: 100, HashesSize: 10000, @@ -62,14 +63,15 @@ func getDefaultTrieAndAccountsDbAndStoragePruningManager() (common.Trie, *state. }) argsAccountsDB := state.ArgsAccountsDB{ - Trie: tr, - Hasher: hasher, - Marshaller: marshaller, - AccountFactory: accCreator, - StoragePruningManager: spm, - AddressConverter: &testscommon.PubkeyConverterMock{}, - SnapshotsManager: snapshotsManager, - StateAccessesCollector: &stateMock.StateAccessesCollectorStub{}, + Trie: tr, + Hasher: hasher, + Marshaller: marshaller, + AccountFactory: accCreator, + StoragePruningManager: spm, + AddressConverter: &testscommon.PubkeyConverterMock{}, + SnapshotsManager: snapshotsManager, + StateAccessesCollector: &stateMock.StateAccessesCollectorStub{}, + MaxDataTriesSizeInMemory: tenMBSize, } adb, _ := state.NewAccountsDB(argsAccountsDB) diff --git a/state/syncer/baseAccountsSyncer.go b/state/syncer/baseAccountsSyncer.go index 3cee93d7325..ee84ac1f352 100644 --- a/state/syncer/baseAccountsSyncer.go +++ b/state/syncer/baseAccountsSyncer.go @@ -28,7 +28,6 @@ type baseAccountsSyncer struct { timeoutHandler trie.TimeoutHandler shardId uint32 cacher storage.Cacher - maxTrieLevelInMemory uint name string maxHardCapForMissingNodes int checkNodesOnDisk bool @@ -54,7 +53,6 @@ type ArgsNewBaseAccountsSyncer struct { UserAccountsSyncStatisticsHandler common.SizeSyncStatisticsHandler AppStatusHandler core.AppStatusHandler EnableEpochsHandler common.EnableEpochsHandler - MaxTrieLevelInMemory uint MaxHardCapForMissingNodes int TrieSyncerVersion int CheckNodesOnDisk bool @@ -217,7 +215,8 @@ func (b *baseAccountsSyncer) GetSyncedTries() map[string]common.Trie { b.mutex.Lock() defer b.mutex.Unlock() - dataTrie, err := trie.NewTrie(b.trieStorageManager, b.marshalizer, b.hasher, b.enableEpochsHandler, b.maxTrieLevelInMemory) + tenMBSize := uint64(10485760) + dataTrie, err := trie.NewTrie(b.trieStorageManager, b.marshalizer, b.hasher, b.enableEpochsHandler, tenMBSize) if err != nil { log.Warn("error creating a new trie in baseAccountsSyncer.GetSyncedTries", "error", err) return make(map[string]common.Trie) diff --git a/state/syncer/baseAccoutnsSyncer_test.go b/state/syncer/baseAccoutnsSyncer_test.go index e2fcf5336f0..4a7ec3cb1ab 100644 --- a/state/syncer/baseAccoutnsSyncer_test.go +++ b/state/syncer/baseAccoutnsSyncer_test.go @@ -28,7 +28,6 @@ func getDefaultBaseAccSyncerArgs() syncer.ArgsNewBaseAccountsSyncer { UserAccountsSyncStatisticsHandler: &testscommon.SizeSyncStatisticsHandlerStub{}, AppStatusHandler: &statusHandler.AppStatusHandlerStub{}, EnableEpochsHandler: &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, - MaxTrieLevelInMemory: 5, MaxHardCapForMissingNodes: 100, TrieSyncerVersion: 3, CheckNodesOnDisk: false, diff --git a/state/syncer/userAccountSyncer_test.go b/state/syncer/userAccountSyncer_test.go index 3ecdf5cd178..6bbd82b6847 100644 --- a/state/syncer/userAccountSyncer_test.go +++ b/state/syncer/userAccountSyncer_test.go @@ -29,7 +29,6 @@ func getDefaultBaseAccSyncerArgs() ArgsNewBaseAccountsSyncer { Cacher: cache.NewCacherMock(), UserAccountsSyncStatisticsHandler: &testscommon.SizeSyncStatisticsHandlerStub{}, AppStatusHandler: &statusHandler.AppStatusHandlerStub{}, - MaxTrieLevelInMemory: 0, MaxHardCapForMissingNodes: 100, TrieSyncerVersion: 2, CheckNodesOnDisk: false, @@ -90,7 +89,8 @@ func TestUserAccountsSyncer_MissingDataTrieNodeFound(t *testing.T) { }, } - tr, _ := trie.NewTrie(tsm, args.Marshalizer, args.Hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, 5) + tenMBSize := uint64(10485760) + tr, _ := trie.NewTrie(tsm, args.Marshalizer, args.Hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, tenMBSize) key := []byte("key") value := []byte("value") _ = tr.Update(key, value) diff --git a/state/syncer/userAccountsSyncer.go b/state/syncer/userAccountsSyncer.go index a63745aa387..626b8457b7c 100644 --- a/state/syncer/userAccountsSyncer.go +++ b/state/syncer/userAccountsSyncer.go @@ -85,7 +85,6 @@ func NewUserAccountsSyncer(args ArgsNewUserAccountsSyncer) (*userAccountsSyncer, timeoutHandler: timeoutHandler, shardId: args.ShardId, cacher: args.Cacher, - maxTrieLevelInMemory: args.MaxTrieLevelInMemory, name: fmt.Sprintf("user accounts for shard %s", core.GetShardIDString(args.ShardId)), maxHardCapForMissingNodes: args.MaxHardCapForMissingNodes, trieSyncerVersion: args.TrieSyncerVersion, diff --git a/state/syncer/userAccountsSyncer_test.go b/state/syncer/userAccountsSyncer_test.go index 5d7252d3b2e..3a7e108b3f0 100644 --- a/state/syncer/userAccountsSyncer_test.go +++ b/state/syncer/userAccountsSyncer_test.go @@ -32,6 +32,8 @@ import ( "github.com/multiversx/mx-chain-go/trie/storageMarker" ) +const tenMBSize = uint64(10485760) + func getDefaultUserAccountsSyncerArgs() syncer.ArgsNewUserAccountsSyncer { return syncer.ArgsNewUserAccountsSyncer{ ArgsNewBaseAccountsSyncer: getDefaultBaseAccSyncerArgs(), @@ -111,7 +113,7 @@ func getSerializedTrieNode( }, } - tr, _ := trie.NewTrie(tsm, marshaller, hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, 5) + tr, _ := trie.NewTrie(tsm, marshaller, hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, tenMBSize) _ = tr.Update(key, []byte("value")) _ = tr.Commit() @@ -162,7 +164,7 @@ func TestUserAccountsSyncer_SyncAccounts(t *testing.T) { }) } -func getDefaultTrieParameters() (common.StorageManager, marshal.Marshalizer, hashing.Hasher, common.EnableEpochsHandler, uint) { +func getDefaultTrieParameters() (common.StorageManager, marshal.Marshalizer, hashing.Hasher, common.EnableEpochsHandler, uint64) { marshalizer := &testscommon.ProtobufMarshalizerMock{} hasher := &testscommon.KeccakMock{} @@ -183,9 +185,8 @@ func getDefaultTrieParameters() (common.StorageManager, marshal.Marshalizer, has } trieStorageManager, _ := trie.NewTrieStorageManager(args) - maxTrieLevelInMemory := uint(1) - return trieStorageManager, args.Marshalizer, args.Hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, maxTrieLevelInMemory + return trieStorageManager, args.Marshalizer, args.Hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, tenMBSize } func emptyTrie() common.Trie { @@ -237,7 +238,7 @@ func TestUserAccountsSyncer_SyncAccountDataTries(t *testing.T) { s, err := syncer.NewUserAccountsSyncer(args) require.Nil(t, err) - _, _ = trie.NewTrie(args.TrieStorageManager, args.Marshalizer, args.Hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, 5) + _, _ = trie.NewTrie(args.TrieStorageManager, args.Marshalizer, args.Hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, tenMBSize) tr := emptyTrie() account, err := accounts.NewUserAccount(testscommon.TestPubKeyAlice, &trieMock.DataTrieTrackerStub{}, &trieMock.TrieLeafParserStub{}) @@ -294,7 +295,7 @@ func TestUserAccountsSyncer_SyncAccountDataTries(t *testing.T) { s, err := syncer.NewUserAccountsSyncer(args) require.Nil(t, err) - _, _ = trie.NewTrie(args.TrieStorageManager, args.Marshalizer, args.Hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, 5) + _, _ = trie.NewTrie(args.TrieStorageManager, args.Marshalizer, args.Hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, tenMBSize) tr := emptyTrie() account, err := accounts.NewUserAccount(testscommon.TestPubKeyAlice, &trieMock.DataTrieTrackerStub{}, &trieMock.TrieLeafParserStub{}) @@ -361,7 +362,7 @@ func TestUserAccountsSyncer_MissingDataTrieNodeFound(t *testing.T) { }, } - tr, _ := trie.NewTrie(tsm, args.Marshalizer, args.Hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, 5) + tr, _ := trie.NewTrie(tsm, args.Marshalizer, args.Hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, tenMBSize) key := []byte("key") value := []byte("value") _ = tr.Update(key, value) diff --git a/state/syncer/validatorAccountsSyncer.go b/state/syncer/validatorAccountsSyncer.go index e436bde8e8c..8463f65dfca 100644 --- a/state/syncer/validatorAccountsSyncer.go +++ b/state/syncer/validatorAccountsSyncer.go @@ -43,7 +43,6 @@ func NewValidatorAccountsSyncer(args ArgsNewValidatorAccountsSyncer) (*validator timeoutHandler: timeoutHandler, shardId: core.MetachainShardId, cacher: args.Cacher, - maxTrieLevelInMemory: args.MaxTrieLevelInMemory, name: "peer accounts", maxHardCapForMissingNodes: args.MaxHardCapForMissingNodes, trieSyncerVersion: args.TrieSyncerVersion, diff --git a/state/triesHolder/dataTriesHolder.go b/state/triesHolder/dataTriesHolder.go new file mode 100644 index 00000000000..cc1cba71807 --- /dev/null +++ b/state/triesHolder/dataTriesHolder.go @@ -0,0 +1,213 @@ +package triesHolder + +import ( + "fmt" + "math" + "sync" + + "github.com/multiversx/mx-chain-core-go/core/check" + "github.com/multiversx/mx-chain-go/common" + "github.com/multiversx/mx-chain-go/storage" + logger "github.com/multiversx/mx-chain-logger-go" + "github.com/multiversx/mx-chain-storage-go/lrucache/capacity" +) + +const maxTrieSizeMinValue = 1 * 1024 * 1024 // 1 MB + +var log = logger.GetOrCreate("state/dataTriesHolder") + +// dataTriesHolder is a structure that holds a map of tries and manages their memory usage +// It uses a doubly linked list to keep track of the order in which the tries were used +// and evicts the oldest used tries when the total size exceeds a maximum limit. +type dataTriesHolder struct { + cacher storage.AdaptedSizedLRUCache + dirtyTries map[string]struct{} // These are the tries that have been modified and need to be persisted + touchedTries map[string]struct{} // These are needed to compute an accurate totalTriesSize + evictedBuffer map[string]common.Trie // in case eviction happens for a dirty trie, we keep it here until we commit it + + mutex sync.RWMutex +} + +// NewDataTriesHolder creates a new instance of dataTriesHolder +func NewDataTriesHolder(maxTriesSize uint64) (*dataTriesHolder, error) { + if maxTriesSize < maxTrieSizeMinValue { + return nil, fmt.Errorf("%w, provided %d, minimum %d", ErrInvalidMaxTrieSizeValue, maxTriesSize, maxTrieSizeMinValue) + } + log.Trace("creating new data tries holder", "maxTriesSize", maxTriesSize) + + c, err := capacity.NewCapacityLRU(math.MaxInt, int64(maxTriesSize)) + if err != nil { + return nil, err + } + + return &dataTriesHolder{ + cacher: c, + dirtyTries: make(map[string]struct{}), + touchedTries: make(map[string]struct{}), + evictedBuffer: make(map[string]common.Trie), + mutex: sync.RWMutex{}, + }, nil +} + +// Put adds a trie pointer to the tries map +func (dth *dataTriesHolder) Put(key []byte, tr common.Trie) { + dth.mutex.Lock() + defer dth.mutex.Unlock() + + dth.putNoLock(key, tr) +} + +func (dth *dataTriesHolder) putNoLock(key []byte, tr common.Trie) { + if check.IfNil(tr) || len(key) == 0 { + log.Warn("trying to put nil trie or empty key in dataTriesHolder", "key", key, "trie", tr) + return + } + + keyString := string(key) + + if len(dth.evictedBuffer) > 0 { + // this means that this trie was evicted while being dirty + delete(dth.evictedBuffer, keyString) + } + + evicted := dth.cacher.AddSizedAndReturnEvicted(keyString, tr, int64(tr.SizeInMemory())) + dth.dirtyTries[keyString] = struct{}{} + dth.touchedTries[keyString] = struct{}{} + + if len(evicted) == 0 { + return + } + + for evictedKey, evictedValue := range evicted { + evictedKeyString, ok := evictedKey.(string) + if !ok { + log.Warn("invalid data in dataTriesHolder cache", "entry type", fmt.Sprintf("%T", evictedKey)) + continue + } + _, ok = dth.dirtyTries[evictedKeyString] + if !ok { + continue + } + + evictedTrie, ok := evictedValue.(common.Trie) + if !ok { + log.Warn("invalid data in dataTriesHolder cache", "entry type", fmt.Sprintf("%T", evictedValue)) + continue + } + dth.evictedBuffer[evictedKeyString] = evictedTrie + } +} + +// Get returns the trie pointer that is stored in the map at the given key +func (dth *dataTriesHolder) Get(key []byte) common.Trie { + if len(key) == 0 { + return nil + } + + dth.mutex.Lock() + defer dth.mutex.Unlock() + + keyString := string(key) + + val, ok := dth.cacher.Get(keyString) + if !ok { + // maybe it was evicted while being dirty + evictedTr, exists := dth.evictedBuffer[keyString] + if !exists { + return nil + } + + delete(dth.evictedBuffer, keyString) + dth.putNoLock(key, evictedTr) + return evictedTr + } + + dth.touchedTries[keyString] = struct{}{} + tr, ok := val.(common.Trie) + if !ok { + log.Warn("invalid data in dataTriesHolder cache", "entry type", fmt.Sprintf("%T", val)) + return nil + } + + return tr +} + +// GetAll returns all the tries that are marked as dirty for this implementation. +// It also resets their dirty flag and recomputes the total size. +func (dth *dataTriesHolder) GetAll() []common.Trie { + dth.mutex.Lock() + defer dth.mutex.Unlock() + + tries := make([]common.Trie, 0) + for keyString := range dth.dirtyTries { + tr := dth.getDirtyTrieNoLock(keyString) + if check.IfNil(tr) { + continue + } + tries = append(tries, tr) + } + dth.dirtyTries = make(map[string]struct{}) + dth.evictedBuffer = make(map[string]common.Trie) + dth.recomputeTotalSize() + return tries +} + +func (dth *dataTriesHolder) getDirtyTrieNoLock(key string) common.Trie { + entry, exists := dth.cacher.Get(key) + if exists { + tr, ok := entry.(common.Trie) + if !ok { + log.Warn("invalid data in dataTriesHolder cache", "entry type", fmt.Sprintf("%T", entry)) + return nil + } + + return tr + } + + tr, ok := dth.evictedBuffer[key] + if !ok { + return nil + } + return tr +} + +func (dth *dataTriesHolder) recomputeTotalSize() { + for keyString := range dth.touchedTries { + entry, exists := dth.cacher.Get(keyString) + if !exists { + continue + } + tr, ok := entry.(common.Trie) + if !ok { + log.Warn("invalid data in dataTriesHolder cache", "entry type", fmt.Sprintf("%T", entry)) + continue + } + + evicted := dth.cacher.AddSized(keyString, tr, int64(tr.SizeInMemory())) + if evicted { + log.Warn("unexpected eviction while recomputing total size in dataTriesHolder") + } + } + dth.touchedTries = make(map[string]struct{}) +} + +// Reset clears the tries map +func (dth *dataTriesHolder) Reset() { + dth.mutex.Lock() + dth.reset() + dth.mutex.Unlock() +} + +func (dth *dataTriesHolder) reset() { + log.Trace("reset data tries holder") + + dth.cacher.Purge() + dth.dirtyTries = make(map[string]struct{}) + dth.touchedTries = make(map[string]struct{}) + dth.evictedBuffer = make(map[string]common.Trie) +} + +// IsInterfaceNil returns true if underlying object is nil +func (dth *dataTriesHolder) IsInterfaceNil() bool { + return dth == nil +} diff --git a/state/triesHolder/dataTriesHolder_test.go b/state/triesHolder/dataTriesHolder_test.go new file mode 100644 index 00000000000..20217f470fa --- /dev/null +++ b/state/triesHolder/dataTriesHolder_test.go @@ -0,0 +1,474 @@ +package triesHolder + +import ( + "errors" + "strconv" + "sync" + "testing" + + "github.com/multiversx/mx-chain-core-go/core/check" + "github.com/multiversx/mx-chain-go/common" + trieMock "github.com/multiversx/mx-chain-go/testscommon/trie" + "github.com/stretchr/testify/assert" +) + +const ( + dthSize = 2 * 1024 * 1024 // 2MB + oneKB = 1 * 1024 +) + +type testTries struct { + key []byte + trie common.Trie +} + +func getTestTries(numTries int) []testTries { + tries := make([]testTries, 0) + for i := 0; i < numTries; i++ { + tr := &trieMock.TrieStub{ + SizeInMemoryCalled: func() int { + return oneKB + }, + } + key := []byte("trie" + strconv.Itoa(i)) + tries = append(tries, testTries{key: key, trie: tr}) + } + return tries +} + +func TestNewDataTriesHolder(t *testing.T) { + t.Parallel() + + t.Run(" invalid max size", func(t *testing.T) { + t.Parallel() + + dth, err := NewDataTriesHolder(512 * 1024) // less than 1MB + assert.True(t, errors.Is(err, ErrInvalidMaxTrieSizeValue)) + assert.True(t, check.IfNil(dth)) + }) + + t.Run("should create new instance", func(t *testing.T) { + t.Parallel() + + dth, err := NewDataTriesHolder(dthSize) + assert.Nil(t, err) + assert.False(t, check.IfNil(dth)) + assert.NotNil(t, dth.evictedBuffer) + assert.NotNil(t, dth.dirtyTries) + assert.NotNil(t, dth.touchedTries) + assert.Equal(t, uint64(0), dth.cacher.SizeInBytesContained()) + assert.Equal(t, 0, dth.cacher.Len()) + }) +} + +func TestDataTriesHolder_Put(t *testing.T) { + t.Parallel() + + t.Run("put invalid data", func(t *testing.T) { + t.Parallel() + + dth, _ := NewDataTriesHolder(dthSize) + + dth.Put([]byte("key"), nil) + dth.Put(nil, &trieMock.TrieStub{}) + assert.Equal(t, uint64(0), dth.cacher.SizeInBytesContained()) + assert.Equal(t, 0, dth.cacher.Len()) + assert.Equal(t, 0, len(dth.dirtyTries)) + assert.Equal(t, 0, len(dth.touchedTries)) + assert.Equal(t, 0, len(dth.evictedBuffer)) + }) + t.Run("put in empty tries holder", func(t *testing.T) { + t.Parallel() + + dth, _ := NewDataTriesHolder(dthSize) + entry := getTestTries(1)[0] + + dth.Put(entry.key, entry.trie) + + assert.Equal(t, 1, dth.cacher.Len()) + assert.Equal(t, 1, len(dth.dirtyTries)) + retrievedEntry, ok := dth.cacher.Get(string(entry.key)) + assert.True(t, ok) + tr, ok := retrievedEntry.(common.Trie) + assert.True(t, ok) + assert.Equal(t, tr, entry.trie) + assert.Equal(t, uint64(oneKB), dth.cacher.SizeInBytesContained()) + assert.Equal(t, 1, len(dth.dirtyTries)) + assert.Equal(t, 1, len(dth.touchedTries)) + assert.Equal(t, 0, len(dth.evictedBuffer)) + }) + t.Run("put in populated tries holder", func(t *testing.T) { + t.Parallel() + + dth, _ := NewDataTriesHolder(dthSize) + numEntries := 5 + entries := getTestTries(numEntries) + for i := 0; i < numEntries; i++ { + dth.Put(entries[i].key, entries[i].trie) + } + + assert.Equal(t, numEntries, dth.cacher.Len()) + assert.Equal(t, numEntries, len(dth.dirtyTries)) + assert.Equal(t, numEntries, len(dth.touchedTries)) + assert.Equal(t, 0, len(dth.evictedBuffer)) + assert.Equal(t, uint64(numEntries*oneKB), dth.cacher.SizeInBytesContained()) + cacherKeys := dth.cacher.Keys() + assert.Equal(t, numEntries, len(cacherKeys)) + for i := 0; i < numEntries; i++ { + retrievedEntry, ok := dth.cacher.Get(cacherKeys[i]) + assert.True(t, ok) + tr, ok := retrievedEntry.(common.Trie) + assert.True(t, ok) + assert.Equal(t, tr, entries[i].trie) + assert.Equal(t, cacherKeys[i], string(entries[i].key)) + } + }) + t.Run("put oldest used trie moves to newest used", func(t *testing.T) { + t.Parallel() + + dth, _ := NewDataTriesHolder(dthSize) + numEntries := 5 + entries := getTestTries(numEntries) + for i := 0; i < numEntries; i++ { + dth.Put(entries[i].key, entries[i].trie) + } + + dth.Put(entries[0].key, entries[0].trie) + + assert.Equal(t, numEntries, dth.cacher.Len()) + assert.Equal(t, numEntries, len(dth.dirtyTries)) + assert.Equal(t, numEntries, len(dth.touchedTries)) + assert.Equal(t, 0, len(dth.evictedBuffer)) + assert.Equal(t, uint64(numEntries*oneKB), dth.cacher.SizeInBytesContained()) + keys := dth.cacher.Keys() + assert.Equal(t, string(entries[0].key), keys[numEntries-1]) + }) + t.Run("put existing trie moves to newest used", func(t *testing.T) { + t.Parallel() + + dth, _ := NewDataTriesHolder(dthSize) + numEntries := 5 + entries := getTestTries(numEntries) + for i := 0; i < numEntries; i++ { + dth.Put(entries[i].key, entries[i].trie) + } + + triePos := 2 + dth.Put(entries[triePos].key, entries[triePos].trie) + + assert.Equal(t, numEntries, dth.cacher.Len()) + assert.Equal(t, numEntries, len(dth.dirtyTries)) + assert.Equal(t, numEntries, len(dth.touchedTries)) + assert.Equal(t, 0, len(dth.evictedBuffer)) + assert.Equal(t, uint64(numEntries*oneKB), dth.cacher.SizeInBytesContained()) + keys := dth.cacher.Keys() + assert.Equal(t, string(entries[triePos].key), keys[numEntries-1]) + }) + t.Run("put with eviction - evicted dirty tries should be in eviction buffer", func(t *testing.T) { + t.Parallel() + + dth, _ := NewDataTriesHolder(dthSize) + numEntries := 5 + entries := getTestTries(numEntries) + for i := 0; i < numEntries; i++ { + dth.Put(entries[i].key, entries[i].trie) + } + + tr := &trieMock.TrieStub{ + SizeInMemoryCalled: func() int { + return dthSize + }, + } + key := []byte("trieEvict") + + dth.Put(key, tr) + + assert.Equal(t, 1, dth.cacher.Len()) + assert.Equal(t, numEntries+1, len(dth.dirtyTries)) + assert.Equal(t, numEntries+1, len(dth.touchedTries)) + assert.Equal(t, numEntries, len(dth.evictedBuffer)) + assert.Equal(t, uint64(dthSize), dth.cacher.SizeInBytesContained()) + }) + t.Run("put with eviction - not dirty tries should evict", func(t *testing.T) { + t.Parallel() + + dth, _ := NewDataTriesHolder(dthSize) + numEntries := 5 + entries := getTestTries(numEntries) + for i := 0; i < numEntries; i++ { + dth.Put(entries[i].key, entries[i].trie) + } + dth.dirtyTries = make(map[string]struct{}) // reset dirty tries + + sizeToEvictTwoTries := dthSize - (3 * oneKB) + tr := &trieMock.TrieStub{ + SizeInMemoryCalled: func() int { + return sizeToEvictTwoTries + }, + } + key := []byte("trieEvict") + + dth.Put(key, tr) + numEvictedTries := 2 + + assert.Equal(t, numEntries-numEvictedTries+1, dth.cacher.Len()) + assert.Equal(t, 1, len(dth.dirtyTries)) + assert.Equal(t, numEntries+1, len(dth.touchedTries)) + assert.Equal(t, 0, len(dth.evictedBuffer)) + assert.Equal(t, uint64(dthSize), dth.cacher.SizeInBytesContained()) + }) +} + +func TestDataTriesHolder_Get(t *testing.T) { + t.Parallel() + + t.Run("get not existing trie should return nil", func(t *testing.T) { + t.Parallel() + + dth, _ := NewDataTriesHolder(dthSize) + tr := dth.Get([]byte("notExistingKey")) + assert.Nil(t, tr) + assert.Equal(t, 0, dth.cacher.Len()) + }) + t.Run("get existing trie should move to newest used", func(t *testing.T) { + t.Parallel() + + dth, _ := NewDataTriesHolder(dthSize) + numEntries := 5 + entries := getTestTries(numEntries) + for i := 0; i < numEntries; i++ { + dth.Put(entries[i].key, entries[i].trie) + } + + tr := dth.Get(entries[1].key) + assert.Equal(t, entries[1].trie, tr) + assert.Equal(t, numEntries, dth.cacher.Len()) + keys := dth.cacher.Keys() + assert.Equal(t, string(entries[1].key), keys[numEntries-1]) + assert.Equal(t, uint64(numEntries*oneKB), dth.cacher.SizeInBytesContained()) + }) + t.Run("get from evicted buffer should put back in cache", func(t *testing.T) { + t.Parallel() + + dth, _ := NewDataTriesHolder(dthSize) + numEntries := 5 + entries := getTestTries(numEntries) + for i := 0; i < numEntries; i++ { + dth.Put(entries[i].key, entries[i].trie) + } + + assert.Equal(t, 0, len(dth.evictedBuffer)) + tr := &trieMock.TrieStub{ + SizeInMemoryCalled: func() int { + return dthSize + }, + } + key := []byte("trieEvict") + dth.Put(key, tr) + assert.Equal(t, numEntries, len(dth.evictedBuffer)) + assert.Equal(t, 1, dth.cacher.Len()) + + numGetFromTrie := 3 + for i := 0; i < numGetFromTrie; i++ { + _ = dth.Get(entries[i].key) + } + + assert.Equal(t, 3, len(dth.evictedBuffer)) // 2 original tries + trieEvict + assert.Equal(t, numGetFromTrie, dth.cacher.Len()) + }) +} + +func TestDataTriesHolder_GetAll(t *testing.T) { + t.Parallel() + + t.Run("dirty trie not found in tries map does not panic", func(t *testing.T) { + t.Parallel() + + dth, _ := NewDataTriesHolder(dthSize) + numEntries := 5 + entries := getTestTries(numEntries) + for i := 0; i < numEntries; i++ { + dth.Put(entries[i].key, entries[i].trie) + } + dth.cacher.Remove(string(entries[0].key)) + + dirtyTries := dth.GetAll() + assert.Equal(t, numEntries-1, len(dirtyTries)) + assert.Equal(t, 0, len(dth.dirtyTries)) + }) + t.Run("trie size is correctly computed after GetAll and eviction", func(t *testing.T) { + t.Parallel() + + dth, _ := NewDataTriesHolder(dthSize) + numEntries := 5 + entries := getTestTries(numEntries) + for i := 0; i < numEntries; i++ { + dth.Put(entries[i].key, entries[i].trie) + } + + assert.Equal(t, uint64(numEntries*oneKB), dth.cacher.SizeInBytesContained()) + + dirtyTries := dth.GetAll() + assert.Equal(t, numEntries, len(dirtyTries)) + assert.Equal(t, 0, len(dth.dirtyTries)) + assert.Equal(t, uint64(numEntries*oneKB), dth.cacher.SizeInBytesContained()) + + trieSize := dthSize - ((numEntries + 1) * oneKB) + tr := &trieMock.TrieStub{ + SizeInMemoryCalled: func() int { + return trieSize + }, + } + key := []byte("newTrie") + dth.Put(key, tr) + assert.Equal(t, numEntries+1, dth.cacher.Len()) + assert.Equal(t, uint64(numEntries*oneKB+trieSize), dth.cacher.SizeInBytesContained()) + dth.dirtyTries = make(map[string]struct{}) // reset dirty tries + + // get a trie and "resolve" some nodes, thus increasing its size in memory + _ = dth.Get(key) + originalSize := trieSize + trieSize = trieSize + oneKB + assert.Equal(t, uint64(numEntries*oneKB+originalSize), dth.cacher.SizeInBytesContained()) // size is not updated + assert.Equal(t, numEntries+1, dth.cacher.Len()) // no eviction + assert.Equal(t, 1, len(dth.touchedTries)) + + dirtyTries = dth.GetAll() + assert.Equal(t, 0, len(dirtyTries)) + assert.Equal(t, 0, len(dth.dirtyTries)) + assert.Equal(t, uint64(dthSize), dth.cacher.SizeInBytesContained()) // size is updated + assert.Equal(t, numEntries+1, dth.cacher.Len()) // no eviction + assert.Equal(t, 0, len(dth.touchedTries)) + + // put again the same trie, now with increased size triggering eviction + _ = dth.Get(key) + assert.Equal(t, 1, len(dth.touchedTries)) + trieSize = trieSize + oneKB/2 + assert.Equal(t, 0, len(dth.dirtyTries)) + assert.Equal(t, uint64(dthSize), dth.cacher.SizeInBytesContained()) // size is not updated + assert.Equal(t, numEntries+1, dth.cacher.Len()) // no eviction + + dirtyTries = dth.GetAll() + assert.Equal(t, 0, len(dirtyTries)) + assert.Equal(t, 0, len(dth.dirtyTries)) + assert.Equal(t, uint64(dthSize-oneKB/2), dth.cacher.SizeInBytesContained()) // size is updated + assert.Equal(t, numEntries, dth.cacher.Len()) // eviction + assert.Equal(t, 0, len(dth.touchedTries)) + }) + t.Run("should work", func(t *testing.T) { + t.Parallel() + + dth, _ := NewDataTriesHolder(dthSize) + numEntries := 5 + entries := getTestTries(numEntries) + for i := 0; i < numEntries; i++ { + dth.Put(entries[i].key, entries[i].trie) + } + + dirtyTries := dth.GetAll() + assert.Equal(t, numEntries, len(dirtyTries)) + assert.Equal(t, 0, len(dth.dirtyTries)) + }) +} + +func TestDataTriesHolder_Reset(t *testing.T) { + t.Parallel() + + dth, _ := NewDataTriesHolder(dthSize) + numEntries := 5 + entries := getTestTries(numEntries) + for i := 0; i < numEntries; i++ { + dth.Put(entries[i].key, entries[i].trie) + } + + dth.Reset() + assert.Equal(t, 0, dth.cacher.Len()) + assert.Equal(t, uint64(0), dth.cacher.SizeInBytesContained()) + assert.Equal(t, 0, len(dth.dirtyTries)) + assert.Equal(t, 0, len(dth.touchedTries)) + assert.Equal(t, 0, len(dth.evictedBuffer)) +} + +func TestDataTriesHolder_Concurrency(t *testing.T) { + t.Parallel() + + dth, _ := NewDataTriesHolder(dthSize) + numEntries := 5 + entries := getTestTries(numEntries) + + wg := sync.WaitGroup{} + wg.Add(numEntries) + + for i := 0; i < numEntries; i++ { + go func(key int) { + dth.Put(entries[key].key, entries[key].trie) + wg.Done() + }(i) + } + + wg.Wait() + + assert.Equal(t, numEntries, dth.cacher.Len()) + assert.Equal(t, numEntries, len(dth.dirtyTries)) + assert.Equal(t, uint64(numEntries*oneKB), dth.cacher.SizeInBytesContained()) +} + +func BenchmarkDataTriesHolder_PutWithEviction(b *testing.B) { + numEntries := 100000 + dth, _ := NewDataTriesHolder(uint64(numEntries * oneKB / 10)) // set max size to 10% of total size to force evictions + entries := getTestTries(numEntries) + b.ResetTimer() + for i := 0; i < b.N; i++ { + entry := entries[i%numEntries] + dth.Put(entry.key, entry.trie) + if i%1000 == 0 { + dth.dirtyTries = make(map[string]struct{}) // reset dirty tries to allow evictions + } + } +} + +func BenchmarkDataTriesHolder_PutNoEviction(b *testing.B) { + numEntries := 100000 + dth, _ := NewDataTriesHolder(uint64(numEntries * oneKB * 2)) // set max size to 200% of total size to avoid evictions + entries := getTestTries(numEntries) + b.ResetTimer() + for i := 0; i < b.N; i++ { + entry := entries[i%numEntries] + dth.Put(entry.key, entry.trie) + } +} + +func BenchmarkDataTriesHolder_Get(b *testing.B) { + numEntries := 100000 + dth, _ := NewDataTriesHolder(uint64(numEntries * oneKB * 2)) + entries := getTestTries(numEntries) + for i := 0; i < numEntries; i++ { + dth.Put(entries[i].key, entries[i].trie) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + entry := entries[i%numEntries] + dth.Get(entry.key) + } +} + +func BenchmarkDataTriesHolder_GetAll(b *testing.B) { + numEntries := 10000 + dth, _ := NewDataTriesHolder(uint64(numEntries * oneKB * 2)) + entries := getTestTries(numEntries) + for i := 0; i < numEntries; i++ { + dth.Put(entries[i].key, entries[i].trie) + } + dth.dirtyTries = make(map[string]struct{}) + dth.touchedTries = make(map[string]struct{}) + numDirty := numEntries / 10 + for i := 0; i < numDirty; i++ { + dth.dirtyTries[string(entries[i].key)] = struct{}{} + dth.touchedTries[string(entries[i].key)] = struct{}{} + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + _ = dth.GetAll() + } +} diff --git a/state/triesHolder/errors.go b/state/triesHolder/errors.go new file mode 100644 index 00000000000..539ef76c3f5 --- /dev/null +++ b/state/triesHolder/errors.go @@ -0,0 +1,6 @@ +package triesHolder + +import "errors" + +// ErrInvalidMaxTrieSizeValue signals that the provided max trie size value is invalid +var ErrInvalidMaxTrieSizeValue = errors.New("invalid max trie size value") diff --git a/state/dataTriesHolder.go b/state/triesHolder/triesHolder.go similarity index 51% rename from state/dataTriesHolder.go rename to state/triesHolder/triesHolder.go index 8333b875fce..75f1023a5f2 100644 --- a/state/dataTriesHolder.go +++ b/state/triesHolder/triesHolder.go @@ -1,4 +1,4 @@ -package state +package triesHolder import ( "sync" @@ -7,34 +7,29 @@ import ( logger "github.com/multiversx/mx-chain-logger-go" ) -type dataTriesHolder struct { +type triesHolder struct { tries map[string]common.Trie mutex sync.RWMutex } -// NewDataTriesHolder creates a new instance of dataTriesHolder -func NewDataTriesHolder() *dataTriesHolder { - return &dataTriesHolder{ +// NewTriesHolder creates a new instance of triesHolder +func NewTriesHolder() *triesHolder { + return &triesHolder{ tries: make(map[string]common.Trie), } } // Put adds a trie pointer to the tries map -func (dth *dataTriesHolder) Put(key []byte, tr common.Trie) { - log.Trace("put trie in data tries holder", "key", key) +func (dth *triesHolder) Put(key []byte, tr common.Trie) { + log.Trace("put trie in tries holder", "key", key) dth.mutex.Lock() dth.tries[string(key)] = tr dth.mutex.Unlock() } -// Replace changes a trie pointer to the tries map -func (dth *dataTriesHolder) Replace(key []byte, tr common.Trie) { - dth.Put(key, tr) -} - // Get returns the trie pointer that is stored in the map at the given key -func (dth *dataTriesHolder) Get(key []byte) common.Trie { +func (dth *triesHolder) Get(key []byte) common.Trie { dth.mutex.Lock() defer dth.mutex.Unlock() @@ -42,7 +37,7 @@ func (dth *dataTriesHolder) Get(key []byte) common.Trie { } // GetAll returns all trie pointers from the map -func (dth *dataTriesHolder) GetAll() []common.Trie { +func (dth *triesHolder) GetAll() []common.Trie { dth.mutex.Lock() defer dth.mutex.Unlock() @@ -54,21 +49,8 @@ func (dth *dataTriesHolder) GetAll() []common.Trie { return tries } -// GetAllTries returns the tries with key value map -func (dth *dataTriesHolder) GetAllTries() map[string]common.Trie { - dth.mutex.Lock() - defer dth.mutex.Unlock() - - copyTries := make(map[string]common.Trie, len(dth.tries)) - for key, trie := range dth.tries { - copyTries[key] = trie - } - - return copyTries -} - // Reset clears the tries map -func (dth *dataTriesHolder) Reset() { +func (dth *triesHolder) Reset() { dth.mutex.Lock() if log.GetLevel() == logger.LogTrace { @@ -82,6 +64,6 @@ func (dth *dataTriesHolder) Reset() { } // IsInterfaceNil returns true if underlying object is nil -func (dth *dataTriesHolder) IsInterfaceNil() bool { +func (dth *triesHolder) IsInterfaceNil() bool { return dth == nil } diff --git a/state/triesHolder/triesHolder_test.go b/state/triesHolder/triesHolder_test.go new file mode 100644 index 00000000000..f34f0ef6dfe --- /dev/null +++ b/state/triesHolder/triesHolder_test.go @@ -0,0 +1,88 @@ +package triesHolder + +import ( + "strconv" + "sync" + "testing" + + "github.com/multiversx/mx-chain-core-go/core/check" + trieMock "github.com/multiversx/mx-chain-go/testscommon/trie" + "github.com/stretchr/testify/assert" +) + +func TestNewTriesHolder(t *testing.T) { + t.Parallel() + + dth := NewTriesHolder() + assert.False(t, check.IfNil(dth)) +} + +func TestTriesHolder_PutAndGet(t *testing.T) { + t.Parallel() + + tr1 := &trieMock.TrieStub{} + + dth := NewTriesHolder() + dth.Put([]byte("trie1"), tr1) + tr := dth.Get([]byte("trie1")) + + assert.True(t, tr == tr1) +} + +func TestTriesHolder_GetAll(t *testing.T) { + t.Parallel() + + tr1 := &trieMock.TrieStub{} + tr2 := &trieMock.TrieStub{} + tr3 := &trieMock.TrieStub{} + + dth := NewTriesHolder() + dth.Put([]byte("trie1"), tr1) + dth.Put([]byte("trie2"), tr2) + dth.Put([]byte("trie3"), tr3) + tries := dth.GetAll() + + assert.Equal(t, 3, len(tries)) +} + +func TestTriesHolder_Reset(t *testing.T) { + t.Parallel() + + tr1 := &trieMock.TrieStub{} + + dth := NewTriesHolder() + dth.Put([]byte("trie1"), tr1) + dth.Reset() + + tr := dth.Get([]byte("trie1")) + assert.Nil(t, tr) +} + +func TestTriesHolder_Concurrency(t *testing.T) { + t.Parallel() + + dth := NewTriesHolder() + numCalls := 5000 + + wg := sync.WaitGroup{} + wg.Add(numCalls) + + for i := 0; i < numCalls; i++ { + go func(key int) { + defer wg.Done() + + switch key % 4 { + case 0: + dth.Put([]byte(strconv.Itoa(key)), &trieMock.TrieStub{}) + case 1: + dth.Get([]byte(strconv.Itoa(key))) + case 2: + dth.GetAll() + case 3: + dth.Reset() + } + }(i) + } + + wg.Wait() +} diff --git a/testscommon/storage/storageManagerArgs.go b/testscommon/common/storageManagerArgs.go similarity index 98% rename from testscommon/storage/storageManagerArgs.go rename to testscommon/common/storageManagerArgs.go index 1f32e18f0d0..6e244bfb162 100644 --- a/testscommon/storage/storageManagerArgs.go +++ b/testscommon/common/storageManagerArgs.go @@ -1,4 +1,4 @@ -package storage +package common import ( "github.com/multiversx/mx-chain-go/common/statistics/disabled" diff --git a/testscommon/components/components.go b/testscommon/components/components.go index a8d7fb25b50..e12700267db 100644 --- a/testscommon/components/components.go +++ b/testscommon/components/components.go @@ -9,6 +9,7 @@ import ( "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/endProcess" "github.com/multiversx/mx-chain-core-go/data/outport" + "github.com/multiversx/mx-chain-go/state/triesHolder" logger "github.com/multiversx/mx-chain-logger-go" wasmConfig "github.com/multiversx/mx-chain-vm-go/config" "github.com/stretchr/testify/require" @@ -37,13 +38,11 @@ import ( p2pConfig "github.com/multiversx/mx-chain-go/p2p/config" "github.com/multiversx/mx-chain-go/sharding" "github.com/multiversx/mx-chain-go/sharding/nodesCoordinator" - "github.com/multiversx/mx-chain-go/state" "github.com/multiversx/mx-chain-go/testscommon" commonMocks "github.com/multiversx/mx-chain-go/testscommon/common" "github.com/multiversx/mx-chain-go/testscommon/dblookupext" "github.com/multiversx/mx-chain-go/testscommon/shardingMocks" statusHandlerMock "github.com/multiversx/mx-chain-go/testscommon/statusHandler" - "github.com/multiversx/mx-chain-go/testscommon/storage" "github.com/multiversx/mx-chain-go/trie" ) @@ -364,20 +363,21 @@ func GetNetworkFactoryArgs() networkComp.NetworkComponentsFactoryArgs { // GetStateFactoryArgs - func GetStateFactoryArgs(coreComponents factory.CoreComponentsHolder, statusCoreComp factory.StatusCoreComponentsHolder) stateComp.StateComponentsFactoryArgs { - tsm, _ := trie.NewTrieStorageManager(storage.GetStorageManagerArgs()) + tsm, _ := trie.NewTrieStorageManager(commonMocks.GetStorageManagerArgs()) storageManagerUser, _ := trie.NewTrieStorageManagerWithoutPruning(tsm) - tsm, _ = trie.NewTrieStorageManager(storage.GetStorageManagerArgs()) + tsm, _ = trie.NewTrieStorageManager(commonMocks.GetStorageManagerArgs()) storageManagerPeer, _ := trie.NewTrieStorageManagerWithoutPruning(tsm) trieStorageManagers := make(map[string]common.StorageManager) trieStorageManagers[dataRetriever.UserAccountsUnit.String()] = storageManagerUser trieStorageManagers[dataRetriever.PeerAccountsUnit.String()] = storageManagerPeer - triesHolder := state.NewDataTriesHolder() - trieUsers, _ := trie.NewTrie(storageManagerUser, coreComponents.InternalMarshalizer(), coreComponents.Hasher(), coreComponents.EnableEpochsHandler(), 5) - triePeers, _ := trie.NewTrie(storageManagerPeer, coreComponents.InternalMarshalizer(), coreComponents.Hasher(), coreComponents.EnableEpochsHandler(), 5) - triesHolder.Put([]byte(dataRetriever.UserAccountsUnit.String()), trieUsers) - triesHolder.Put([]byte(dataRetriever.PeerAccountsUnit.String()), triePeers) + triesContainer := triesHolder.NewTriesHolder() + tenMBSize := uint64(10485760) + trieUsers, _ := trie.NewTrie(storageManagerUser, coreComponents.InternalMarshalizer(), coreComponents.Hasher(), coreComponents.EnableEpochsHandler(), tenMBSize) + triePeers, _ := trie.NewTrie(storageManagerPeer, coreComponents.InternalMarshalizer(), coreComponents.Hasher(), coreComponents.EnableEpochsHandler(), tenMBSize) + triesContainer.Put([]byte(dataRetriever.UserAccountsUnit.String()), trieUsers) + triesContainer.Put([]byte(dataRetriever.PeerAccountsUnit.String()), triePeers) stateComponentsFactoryArgs := stateComp.StateComponentsFactoryArgs{ Config: GetGeneralConfig(), diff --git a/testscommon/components/configs.go b/testscommon/components/configs.go index b3672ef1d32..c5d08a3ba3b 100644 --- a/testscommon/components/configs.go +++ b/testscommon/components/configs.go @@ -5,6 +5,8 @@ import ( "github.com/multiversx/mx-chain-go/testscommon" ) +const tenMBSize = uint64(10485760) + // GetGeneralConfig - func GetGeneralConfig() config.Config { return config.Config{ @@ -22,8 +24,9 @@ func GetGeneralConfig() config.Config { StateTriesConfig: config.StateTriesConfig{ AccountsStatePruningEnabled: true, PeerStatePruningEnabled: true, - MaxStateTrieLevelInMemory: 5, - MaxPeerTrieLevelInMemory: 5, + MaxUserTrieSizeInMemory: tenMBSize, + MaxPeerTrieSizeInMemory: tenMBSize, + DataTriesSizeInMemory: tenMBSize, }, EvictionWaitingList: config.EvictionWaitingListConfig{ HashesSize: 100, diff --git a/testscommon/generalConfig.go b/testscommon/generalConfig.go index 75b26eef432..7e7b5f1e3b5 100644 --- a/testscommon/generalConfig.go +++ b/testscommon/generalConfig.go @@ -5,6 +5,8 @@ import ( "github.com/multiversx/mx-chain-go/storage/storageunit" ) +const tenMbSize = uint64(10485760) + // GetGeneralConfig returns the common configuration used for testing func GetGeneralConfig() config.Config { return config.Config{ @@ -152,8 +154,9 @@ func GetGeneralConfig() config.Config { SnapshotsEnabled: true, AccountsStatePruningEnabled: false, PeerStatePruningEnabled: false, - MaxStateTrieLevelInMemory: 5, - MaxPeerTrieLevelInMemory: 5, + MaxUserTrieSizeInMemory: tenMbSize, + MaxPeerTrieSizeInMemory: tenMbSize, + DataTriesSizeInMemory: tenMbSize, }, TrieStorageManagerConfig: config.TrieStorageManagerConfig{ PruningBufferLen: 1000, diff --git a/testscommon/integrationtests/factory.go b/testscommon/integrationtests/factory.go index 80d65d1c703..aca3bdfdd34 100644 --- a/testscommon/integrationtests/factory.go +++ b/testscommon/integrationtests/factory.go @@ -18,9 +18,9 @@ import ( "github.com/multiversx/mx-chain-go/storage/factory" "github.com/multiversx/mx-chain-go/storage/storageunit" "github.com/multiversx/mx-chain-go/testscommon" + testCommon "github.com/multiversx/mx-chain-go/testscommon/common" "github.com/multiversx/mx-chain-go/testscommon/enableEpochsHandlerMock" testStorage "github.com/multiversx/mx-chain-go/testscommon/state" - testcommonStorage "github.com/multiversx/mx-chain-go/testscommon/storage" "github.com/multiversx/mx-chain-go/trie" ) @@ -30,9 +30,6 @@ var TestMarshalizer = &marshal.GogoProtoMarshalizer{} // TestHasher - var TestHasher = sha256.NewSha256() -// MaxTrieLevelInMemory - -const MaxTrieLevelInMemory = uint(5) - // CreateMemUnit - func CreateMemUnit() storage.Storer { capacity := uint32(10) @@ -94,14 +91,15 @@ func CreateAccountsDB(db storage.Storer, enableEpochs common.EnableEpochsHandler } ewl, _ := evictionWaitingList.NewMemoryEvictionWaitingList(ewlArgs) - args := testcommonStorage.GetStorageManagerArgs() + args := testCommon.GetStorageManagerArgs() args.MainStorer = db args.Marshalizer = TestMarshalizer args.Hasher = TestHasher trieStorage, _ := trie.NewTrieStorageManager(args) + tenMBSize := uint64(10485760) - tr, _ := trie.NewTrie(trieStorage, TestMarshalizer, TestHasher, enableEpochs, MaxTrieLevelInMemory) + tr, _ := trie.NewTrie(trieStorage, TestMarshalizer, TestHasher, enableEpochs, tenMBSize) spm, _ := storagePruningManager.NewStoragePruningManager(ewl, 10) argsAccCreator := accountFactory.ArgsAccountCreator{ @@ -125,14 +123,15 @@ func CreateAccountsDB(db storage.Storer, enableEpochs common.EnableEpochsHandler }) argsAccountsDB := state.ArgsAccountsDB{ - Trie: tr, - Hasher: TestHasher, - Marshaller: TestMarshalizer, - AccountFactory: accCreator, - StoragePruningManager: spm, - AddressConverter: &testscommon.PubkeyConverterMock{}, - SnapshotsManager: snapshotsManager, - StateAccessesCollector: disabled.NewDisabledStateAccessesCollector(), + Trie: tr, + Hasher: TestHasher, + Marshaller: TestMarshalizer, + AccountFactory: accCreator, + StoragePruningManager: spm, + AddressConverter: &testscommon.PubkeyConverterMock{}, + SnapshotsManager: snapshotsManager, + StateAccessesCollector: disabled.NewDisabledStateAccessesCollector(), + MaxDataTriesSizeInMemory: tenMBSize, } adb, _ := state.NewAccountsDB(argsAccountsDB) diff --git a/testscommon/state/testTrie.go b/testscommon/state/testTrie.go index 8744009aa18..81e4cc77117 100644 --- a/testscommon/state/testTrie.go +++ b/testscommon/state/testTrie.go @@ -40,7 +40,8 @@ func GetDefaultTrieParameters() (common.StorageManager, marshal.Marshalizer, has // GetNewTrie - func GetNewTrie() common.Trie { tsm, marshaller, hasher := GetDefaultTrieParameters() - tr, _ := trie.NewTrie(tsm, marshaller, hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, 5) + tenMBSize := uint64(10485760) + tr, _ := trie.NewTrie(tsm, marshaller, hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, tenMBSize) return tr } diff --git a/testscommon/trie/snapshotPruningStorerStub.go b/testscommon/storage/snapshotPruningStorerStub.go similarity index 99% rename from testscommon/trie/snapshotPruningStorerStub.go rename to testscommon/storage/snapshotPruningStorerStub.go index 8de709ab2cd..f6186eb49dc 100644 --- a/testscommon/trie/snapshotPruningStorerStub.go +++ b/testscommon/storage/snapshotPruningStorerStub.go @@ -1,4 +1,4 @@ -package trie +package storage import ( "github.com/multiversx/mx-chain-core-go/core" diff --git a/testscommon/trie/trieStub.go b/testscommon/trie/trieStub.go index 30e0ba6066e..cd078b5a428 100644 --- a/testscommon/trie/trieStub.go +++ b/testscommon/trie/trieStub.go @@ -33,6 +33,7 @@ type TrieStub struct { CloseCalled func() error CollectLeavesForMigrationCalled func(args vmcommon.ArgsMigrateDataTrieLeaves) error IsMigratedToLatestVersionCalled func() (bool, error) + SizeInMemoryCalled func() int } // GetStorageManager - @@ -215,6 +216,14 @@ func (ts *TrieStub) IsMigratedToLatestVersion() (bool, error) { return false, nil } +// SizeInMemory - +func (ts *TrieStub) SizeInMemory() int { + if ts.SizeInMemoryCalled != nil { + return ts.SizeInMemoryCalled() + } + return 0 +} + // Close - func (ts *TrieStub) Close() error { if ts.CloseCalled != nil { diff --git a/testscommon/trie/triesHolderStub.go b/testscommon/trie/triesHolderStub.go index 42eab41d7f5..4d5a57e5738 100644 --- a/testscommon/trie/triesHolderStub.go +++ b/testscommon/trie/triesHolderStub.go @@ -20,13 +20,6 @@ func (ths *TriesHolderStub) Put(key []byte, trie common.Trie) { } } -// Replace - -func (ths *TriesHolderStub) Replace(key []byte, trie common.Trie) { - if ths.RemoveCalled != nil { - ths.RemoveCalled(key, trie) - } -} - // Get - func (ths *TriesHolderStub) Get(key []byte) common.Trie { if ths.GetCalled != nil { diff --git a/trie/baseIterator.go b/trie/baseIterator.go index 8ff558790d8..5f34106ab92 100644 --- a/trie/baseIterator.go +++ b/trie/baseIterator.go @@ -3,11 +3,13 @@ package trie import ( "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-go/common" + "github.com/multiversx/mx-chain-go/trie/trieMetricsCollector" ) type baseIterator struct { currentNode node nextNodes []node + tmc MetricsCollector db common.TrieStorageInteractor } @@ -23,7 +25,8 @@ func newBaseIterator(trie common.Trie) (*baseIterator, error) { } trieStorage := trie.GetStorageManager() - nextNodes, err := pmt.root.getChildren(trieStorage) + tmc := trieMetricsCollector.NewDisabledTrieMetricsCollector() + nextNodes, err := pmt.root.getChildren(tmc, trieStorage) if err != nil { return nil, err } @@ -32,6 +35,7 @@ func newBaseIterator(trie common.Trie) (*baseIterator, error) { currentNode: pmt.root, nextNodes: nextNodes, db: trieStorage, + tmc: tmc, }, nil } @@ -50,7 +54,7 @@ func (it *baseIterator) next() ([]node, error) { } it.currentNode = n - return it.currentNode.getChildren(it.db) + return it.currentNode.getChildren(it.tmc, it.db) } // MarshalizedNode marshalizes the current node, and then returns the serialized node diff --git a/trie/branchNode.go b/trie/branchNode.go index 8e3584d0589..d90c2235b98 100644 --- a/trie/branchNode.go +++ b/trie/branchNode.go @@ -252,8 +252,7 @@ func (bn *branchNode) hashNode() ([]byte, error) { return encodeNodeAndGetHash(bn) } -func (bn *branchNode) commitDirty(level byte, maxTrieLevelInMemory uint, originDb common.TrieStorageInteractor, targetDb common.BaseStorer) error { - level++ +func (bn *branchNode) commitDirty(originDb common.TrieStorageInteractor, targetDb common.BaseStorer, tmc MetricsCollector) error { err := bn.isEmptyOrNil() if err != nil { return fmt.Errorf("commit error %w", err) @@ -268,7 +267,7 @@ func (bn *branchNode) commitDirty(level byte, maxTrieLevelInMemory uint, originD continue } - err = bn.children[i].commitDirty(level, maxTrieLevelInMemory, originDb, targetDb) + err = bn.children[i].commitDirty(originDb, targetDb, tmc) if err != nil { return err } @@ -278,17 +277,15 @@ func (bn *branchNode) commitDirty(level byte, maxTrieLevelInMemory uint, originD if err != nil { return err } - if uint(level) == maxTrieLevelInMemory { - log.Trace("collapse branch node on commit") - var collapsedBn *branchNode - collapsedBn, err = bn.getCollapsedBn() - if err != nil { - return err + for i := range bn.children { + // TODO: do not collapse if maxSizeInMem is not reached + if isLeafNode(bn.children[i]) { + tmc.AddSizeLoadedInMem(-bn.children[i].sizeInBytes()) + bn.children[i] = nil } - - *bn = *collapsedBn } + return nil } @@ -302,12 +299,16 @@ func (bn *branchNode) commitSnapshot( stats common.TrieStatisticsHandler, idleProvider IdleNodeProvider, nodeBytes []byte, - depthLevel int, + tmc MetricsCollector, ) error { if shouldStopIfContextDoneBlockingIfBusy(ctx, idleProvider) { return core.ErrContextClosing } + depthLevel := tmc.GetCurrentDepth() + tmc.SetDepth(depthLevel + 1) + defer tmc.SetDepth(depthLevel) // Reset depth when returning from this level + for i := range bn.EncodedChildren { if len(bn.EncodedChildren[i]) == 0 { continue @@ -323,7 +324,7 @@ func (bn *branchNode) commitSnapshot( ctx, stats, idleProvider, - depthLevel, + tmc, bn.EncodedChildren[i], ) if err != nil { @@ -331,7 +332,7 @@ func (bn *branchNode) commitSnapshot( } } - stats.AddBranchNode(depthLevel, uint64(len(nodeBytes))) + stats.AddBranchNode(int(depthLevel), uint64(len(nodeBytes))) return nil } @@ -348,7 +349,7 @@ func (bn *branchNode) getEncodedNode() ([]byte, error) { return marshaledNode, nil } -func (bn *branchNode) resolveCollapsed(pos byte, db common.TrieStorageInteractor) error { +func (bn *branchNode) resolveCollapsed(pos byte, tmc MetricsCollector, db common.TrieStorageInteractor) error { err := bn.isEmptyOrNil() if err != nil { return fmt.Errorf("resolveCollapsed error %w", err) @@ -363,6 +364,7 @@ func (bn *branchNode) resolveCollapsed(pos byte, db common.TrieStorageInteractor return err } child.setGivenHash(bn.EncodedChildren[pos]) + tmc.AddSizeLoadedInMem(child.sizeInBytes()) bn.children[pos] = child } return nil @@ -381,31 +383,32 @@ func (bn *branchNode) isPosCollapsed(pos int) bool { return bn.children[pos] == nil && len(bn.EncodedChildren[pos]) != 0 } -func (bn *branchNode) tryGet(key []byte, currentDepth uint32, db common.TrieStorageInteractor) (value []byte, maxDepth uint32, err error) { +func (bn *branchNode) tryGet(key []byte, tmc MetricsCollector, db common.TrieStorageInteractor) (value []byte, err error) { err = bn.isEmptyOrNil() if err != nil { - return nil, currentDepth, fmt.Errorf("tryGet error %w", err) + return nil, fmt.Errorf("tryGet error %w", err) } if len(key) == 0 { - return nil, currentDepth, nil + return nil, nil } childPos := key[firstByte] if childPosOutOfRange(childPos) { - return nil, currentDepth, ErrChildPosOutOfRange + return nil, ErrChildPosOutOfRange } key = key[1:] - err = resolveIfCollapsed(bn, childPos, db) + err = resolveIfCollapsed(bn, childPos, tmc, db) if err != nil { - return nil, currentDepth, err + return nil, err } if bn.children[childPos] == nil { - return nil, currentDepth, nil + return nil, nil } - return bn.children[childPos].tryGet(key, currentDepth+1, db) + tmc.SetDepth(tmc.GetCurrentDepth() + 1) + return bn.children[childPos].tryGet(key, tmc, db) } -func (bn *branchNode) getNext(key []byte, db common.TrieStorageInteractor) (node, []byte, error) { +func (bn *branchNode) getNext(key []byte, tmc MetricsCollector, db common.TrieStorageInteractor) (node, []byte, error) { err := bn.isEmptyOrNil() if err != nil { return nil, nil, fmt.Errorf("getNext error %w", err) @@ -418,7 +421,7 @@ func (bn *branchNode) getNext(key []byte, db common.TrieStorageInteractor) (node return nil, nil, ErrChildPosOutOfRange } key = key[1:] - err = resolveIfCollapsed(bn, childPos, db) + err = resolveIfCollapsed(bn, childPos, tmc, db) if err != nil { return nil, nil, err } @@ -429,7 +432,7 @@ func (bn *branchNode) getNext(key []byte, db common.TrieStorageInteractor) (node return bn.children[childPos], key, nil } -func (bn *branchNode) insert(newData core.TrieData, db common.TrieStorageInteractor) (node, [][]byte, error) { +func (bn *branchNode) insert(newData core.TrieData, tmc MetricsCollector, db common.TrieStorageInteractor) (node, [][]byte, error) { emptyHashes := make([][]byte, 0) err := bn.isEmptyOrNil() if err != nil { @@ -445,19 +448,19 @@ func (bn *branchNode) insert(newData core.TrieData, db common.TrieStorageInterac } newData.Key = newData.Key[1:] - err = resolveIfCollapsed(bn, childPos, db) + err = resolveIfCollapsed(bn, childPos, tmc, db) if err != nil { return nil, emptyHashes, err } if bn.children[childPos] == nil { - return bn.insertOnNilChild(newData, childPos) + return bn.insertOnNilChild(newData, tmc, childPos) } - return bn.insertOnExistingChild(newData, childPos, db) + return bn.insertOnExistingChild(newData, tmc, childPos, db) } -func (bn *branchNode) insertOnNilChild(newData core.TrieData, childPos byte) (node, [][]byte, error) { +func (bn *branchNode) insertOnNilChild(newData core.TrieData, tmc MetricsCollector, childPos byte) (node, [][]byte, error) { newLn, err := newLeafNode(newData, bn.marsh, bn.hasher) if err != nil { return nil, [][]byte{}, err @@ -468,12 +471,13 @@ func (bn *branchNode) insertOnNilChild(newData core.TrieData, childPos byte) (no if err != nil { return nil, [][]byte{}, err } + tmc.AddSizeLoadedInMem(newLn.sizeInBytes()) return bn, modifiedHashes, nil } -func (bn *branchNode) insertOnExistingChild(newData core.TrieData, childPos byte, db common.TrieStorageInteractor) (node, [][]byte, error) { - newNode, modifiedHashes, err := bn.children[childPos].insert(newData, db) +func (bn *branchNode) insertOnExistingChild(newData core.TrieData, tmc MetricsCollector, childPos byte, db common.TrieStorageInteractor) (node, [][]byte, error) { + newNode, modifiedHashes, err := bn.children[childPos].insert(newData, tmc, db) if check.IfNil(newNode) || err != nil { return nil, [][]byte{}, err } @@ -504,7 +508,7 @@ func (bn *branchNode) modifyNodeAfterInsert(modifiedHashes [][]byte, childPos by return modifiedHashes, nil } -func (bn *branchNode) delete(key []byte, db common.TrieStorageInteractor) (bool, node, [][]byte, error) { +func (bn *branchNode) delete(key []byte, tmc MetricsCollector, db common.TrieStorageInteractor) (bool, node, [][]byte, error) { emptyHashes := make([][]byte, 0) err := bn.isEmptyOrNil() if err != nil { @@ -518,7 +522,7 @@ func (bn *branchNode) delete(key []byte, db common.TrieStorageInteractor) (bool, return false, nil, emptyHashes, ErrChildPosOutOfRange } key = key[1:] - err = resolveIfCollapsed(bn, childPos, db) + err = resolveIfCollapsed(bn, childPos, tmc, db) if err != nil { return false, nil, emptyHashes, err } @@ -527,7 +531,7 @@ func (bn *branchNode) delete(key []byte, db common.TrieStorageInteractor) (bool, return false, bn, emptyHashes, nil } - dirty, newNode, oldHashes, err := bn.children[childPos].delete(key, db) + dirty, newNode, oldHashes, err := bn.children[childPos].delete(key, tmc, db) if !dirty || err != nil { return false, bn, emptyHashes, err } @@ -536,7 +540,7 @@ func (bn *branchNode) delete(key []byte, db common.TrieStorageInteractor) (bool, oldHashes = append(oldHashes, bn.hash) } - err = bn.setNewChild(childPos, newNode) + err = bn.setNewChild(childPos, newNode, tmc) if err != nil { return false, nil, emptyHashes, err } @@ -544,18 +548,18 @@ func (bn *branchNode) delete(key []byte, db common.TrieStorageInteractor) (bool, numChildren, pos := getChildPosition(bn) if numChildren == 1 { - err = resolveIfCollapsed(bn, byte(pos), db) + err = resolveIfCollapsed(bn, byte(pos), tmc, db) if err != nil { return false, nil, emptyHashes, err } - err = resolveIfCollapsed(bn.children[pos], byte(pos), db) + err = resolveIfCollapsed(bn.children[pos], byte(pos), tmc, db) if err != nil { return false, nil, emptyHashes, err } var newChildHash bool - newNode, newChildHash, err = bn.children[pos].reduceNode(pos) + newNode, newChildHash, err = bn.children[pos].reduceNode(pos, tmc) if err != nil { return false, nil, emptyHashes, err } @@ -564,6 +568,7 @@ func (bn *branchNode) delete(key []byte, db common.TrieStorageInteractor) (bool, oldHashes = append(oldHashes, bn.children[pos].getHash()) } + tmc.AddSizeLoadedInMem(-bn.sizeInBytes()) return true, newNode, oldHashes, nil } @@ -572,12 +577,13 @@ func (bn *branchNode) delete(key []byte, db common.TrieStorageInteractor) (bool, return true, bn, oldHashes, nil } -func (bn *branchNode) setNewChild(childPos byte, newNode node) error { +func (bn *branchNode) setNewChild(childPos byte, newNode node, tmc MetricsCollector) error { bn.hash = nil bn.children[childPos] = newNode if check.IfNil(newNode) { bn.setVersionForChild(core.NotSpecified, childPos) bn.EncodedChildren[childPos] = nil + tmc.AddSizeLoadedInMem(-hashSizeInBytes) return nil } @@ -602,11 +608,12 @@ func (bn *branchNode) revertChildrenVersionSliceIfNeeded() { bn.ChildrenVersion = []byte(nil) } -func (bn *branchNode) reduceNode(pos int) (node, bool, error) { +func (bn *branchNode) reduceNode(pos int, tmc MetricsCollector) (node, bool, error) { newEn, err := newExtensionNode([]byte{byte(pos)}, bn, bn.marsh, bn.hasher) if err != nil { return nil, false, err } + tmc.AddSizeLoadedInMem(newEn.sizeInBytes()) return newEn, false, nil } @@ -638,7 +645,7 @@ func (bn *branchNode) isEmptyOrNil() error { return ErrEmptyBranchNode } -func (bn *branchNode) print(writer io.Writer, index int, db common.TrieStorageInteractor) { +func (bn *branchNode) print(writer io.Writer, index int, tmc MetricsCollector, db common.TrieStorageInteractor) { if bn == nil { return } @@ -646,7 +653,7 @@ func (bn *branchNode) print(writer io.Writer, index int, db common.TrieStorageIn str := fmt.Sprintf("B: %v - %v", hex.EncodeToString(bn.hash), bn.dirty) _, _ = fmt.Fprintln(writer, str) for i := 0; i < len(bn.children); i++ { - err := resolveIfCollapsed(bn, byte(i), db) + err := resolveIfCollapsed(bn, byte(i), tmc, db) if err != nil { log.Debug("branch node: print trie err", "error", err, "hash", bn.EncodedChildren[i]) } @@ -662,7 +669,7 @@ func (bn *branchNode) print(writer io.Writer, index int, db common.TrieStorageIn str2 := fmt.Sprintf("+ %d: ", i) _, _ = fmt.Fprint(writer, str2) childIndex := index + len(str) - 1 + len(str2) - child.print(writer, childIndex, db) + child.print(writer, childIndex, tmc, db) } } @@ -691,7 +698,7 @@ func (bn *branchNode) getDirtyHashes(hashes common.ModifiedHashes) error { return nil } -func (bn *branchNode) getChildren(db common.TrieStorageInteractor) ([]node, error) { +func (bn *branchNode) getChildren(tmc MetricsCollector, db common.TrieStorageInteractor) ([]node, error) { err := bn.isEmptyOrNil() if err != nil { return nil, fmt.Errorf("getChildren error %w", err) @@ -700,7 +707,7 @@ func (bn *branchNode) getChildren(db common.TrieStorageInteractor) ([]node, erro nextNodes := make([]node, 0) for i := range bn.children { - err = resolveIfCollapsed(bn, byte(i), db) + err = resolveIfCollapsed(bn, byte(i), tmc, db) if err != nil { return nil, err } @@ -766,6 +773,7 @@ func (bn *branchNode) getAllLeavesOnChannel( marshalizer marshal.Marshalizer, chanClose chan struct{}, ctx context.Context, + tmc MetricsCollector, ) error { err := bn.isEmptyOrNil() if err != nil { @@ -781,7 +789,7 @@ func (bn *branchNode) getAllLeavesOnChannel( log.Trace("branchNode.getAllLeavesOnChannel context done") return nil default: - err = resolveIfCollapsed(bn, byte(i), db) + err = resolveIfCollapsed(bn, byte(i), tmc, db) if err != nil { return err } @@ -792,7 +800,7 @@ func (bn *branchNode) getAllLeavesOnChannel( clonedKeyBuilder := keyBuilder.ShallowClone() clonedKeyBuilder.BuildKey([]byte{byte(i)}) - err = bn.children[i].getAllLeavesOnChannel(leavesChannel, clonedKeyBuilder, trieLeafParser, db, marshalizer, chanClose, ctx) + err = bn.children[i].getAllLeavesOnChannel(leavesChannel, clonedKeyBuilder, trieLeafParser, db, marshalizer, chanClose, ctx, tmc) if err != nil { return err } @@ -804,7 +812,7 @@ func (bn *branchNode) getAllLeavesOnChannel( return nil } -func (bn *branchNode) getAllHashes(db common.TrieStorageInteractor) ([][]byte, error) { +func (bn *branchNode) getAllHashes(tmc MetricsCollector, db common.TrieStorageInteractor) ([][]byte, error) { err := bn.isEmptyOrNil() if err != nil { return nil, fmt.Errorf("getAllHashes error: %w", err) @@ -813,7 +821,7 @@ func (bn *branchNode) getAllHashes(db common.TrieStorageInteractor) ([][]byte, e var childrenHashes [][]byte hashes := make([][]byte, 0) for i := range bn.children { - err = resolveIfCollapsed(bn, byte(i), db) + err = resolveIfCollapsed(bn, byte(i), tmc, db) if err != nil { return nil, err } @@ -822,7 +830,7 @@ func (bn *branchNode) getAllHashes(db common.TrieStorageInteractor) ([][]byte, e continue } - childrenHashes, err = bn.children[i].getAllHashes(db) + childrenHashes, err = bn.children[i].getAllHashes(tmc, db) if err != nil { return nil, err } @@ -851,12 +859,15 @@ func (bn *branchNode) sizeInBytes() int { return 0 } - // hasher + marshalizer + dirty flag = numNodeInnerPointers * pointerSizeInBytes + 1 - nodeSize := len(bn.hash) + numNodeInnerPointers*pointerSizeInBytes + 1 - for _, collapsed := range bn.EncodedChildren { - nodeSize += len(collapsed) + nodeSize := baseNodeSizeInBytes + bnChildrenPointersSize + numChildren := 0 + for i := range bn.EncodedChildren { + if bn.children[i] != nil || len(bn.EncodedChildren[i]) != 0 { + numChildren++ + } } - nodeSize += len(bn.children) * pointerSizeInBytes + nodeSize += numChildren * hashSizeInBytes + nodeSize += len(bn.ChildrenVersion) return nodeSize } @@ -865,14 +876,18 @@ func (bn *branchNode) getValue() []byte { return []byte{} } -func (bn *branchNode) collectStats(ts common.TrieStatisticsHandler, depthLevel int, db common.TrieStorageInteractor) error { +func (bn *branchNode) collectStats(ts common.TrieStatisticsHandler, tmc MetricsCollector, db common.TrieStorageInteractor) error { err := bn.isEmptyOrNil() if err != nil { return fmt.Errorf("collectStats error %w", err) } + depthLevel := tmc.GetCurrentDepth() + tmc.SetDepth(depthLevel + 1) + defer tmc.SetDepth(depthLevel) // Reset depth when returning from this level + for i := range bn.children { - err = resolveIfCollapsed(bn, byte(i), db) + err = resolveIfCollapsed(bn, byte(i), tmc, db) if err != nil { return err } @@ -881,7 +896,7 @@ func (bn *branchNode) collectStats(ts common.TrieStatisticsHandler, depthLevel i continue } - err = bn.children[i].collectStats(ts, depthLevel+1, db) + err = bn.children[i].collectStats(ts, tmc, db) if err != nil { return err } @@ -892,7 +907,7 @@ func (bn *branchNode) collectStats(ts common.TrieStatisticsHandler, depthLevel i return err } - ts.AddBranchNode(depthLevel, uint64(len(val))) + ts.AddBranchNode(int(depthLevel), uint64(len(val))) return nil } @@ -936,6 +951,7 @@ func (bn *branchNode) getVersionForChild(childIndex byte) core.TrieNodeVersion { func (bn *branchNode) collectLeavesForMigration( migrationArgs vmcommon.ArgsMigrateDataTrieLeaves, + tmc MetricsCollector, db common.TrieStorageInteractor, keyBuilder common.KeyBuilder, ) (bool, error) { @@ -961,14 +977,14 @@ func (bn *branchNode) collectLeavesForMigration( continue } - err = resolveIfCollapsed(bn, byte(i), db) + err = resolveIfCollapsed(bn, byte(i), tmc, db) if err != nil { return false, err } clonedKeyBuilder := keyBuilder.ShallowClone() clonedKeyBuilder.BuildKey([]byte{byte(i)}) - shouldContinueMigrating, err := bn.children[i].collectLeavesForMigration(migrationArgs, db, clonedKeyBuilder) + shouldContinueMigrating, err := bn.children[i].collectLeavesForMigration(migrationArgs, tmc, db, clonedKeyBuilder) if err != nil { return false, err } diff --git a/trie/branchNode_test.go b/trie/branchNode_test.go index 4cea8910bad..0fd347ca79e 100644 --- a/trie/branchNode_test.go +++ b/trie/branchNode_test.go @@ -19,9 +19,14 @@ import ( "github.com/multiversx/mx-chain-go/trie/keyBuilder" "github.com/multiversx/mx-chain-go/trie/mock" "github.com/multiversx/mx-chain-go/trie/statistics" + "github.com/multiversx/mx-chain-go/trie/trieMetricsCollector" "github.com/stretchr/testify/assert" ) +const tenMBSize = uint64(10485760) + +var dtmc = trieMetricsCollector.NewDisabledTrieMetricsCollector() + func getTestMarshalizerAndHasher() (marshal.Marshalizer, hashing.Hasher) { marsh := &marshal.GogoProtoMarshalizer{} hash := &testscommon.KeccakMock{} @@ -55,6 +60,15 @@ func getBnAndCollapsedBn(marshalizer marshal.Marshalizer, hasher hashing.Hasher) return bn, collapsedBn } +func markNotDirtyBranchNode(bn *branchNode) { + bn.dirty = false + for i := range bn.children { + if bn.children[i] != nil { + bn.children[i].setDirty(false) + } + } +} + func emptyDirtyBranchNode() *branchNode { var children [nrOfChildren]node encChildren := make([][]byte, nrOfChildren) @@ -76,14 +90,14 @@ func newEmptyTrie() (*patriciaMerkleTrie, *trieStorageManager) { args := GetDefaultTrieStorageManagerParameters() trieStorage, _ := NewTrieStorageManager(args) tr := &patriciaMerkleTrie{ - trieStorage: trieStorage, - marshalizer: args.Marshalizer, - hasher: args.Hasher, - oldHashes: make([][]byte, 0), - oldRoot: make([]byte, 0), - maxTrieLevelInMemory: 5, - chanClose: make(chan struct{}), - enableEpochsHandler: &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, + trieStorage: trieStorage, + marshalizer: args.Marshalizer, + hasher: args.Hasher, + oldHashes: make([][]byte, 0), + oldRoot: make([]byte, 0), + chanClose: make(chan struct{}), + enableEpochsHandler: &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, + maxSizeInMem: tenMBSize, } return tr, trieStorage @@ -197,10 +211,9 @@ func TestBranchNode_setRootHash(t *testing.T) { trieStorage1, _ := NewTrieStorageManager(GetDefaultTrieStorageManagerParameters()) trieStorage2, _ := NewTrieStorageManager(GetDefaultTrieStorageManagerParameters()) - maxTrieLevelInMemory := uint(5) - tr1, _ := NewTrie(trieStorage1, marsh, hsh, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, maxTrieLevelInMemory) - tr2, _ := NewTrie(trieStorage2, marsh, hsh, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, maxTrieLevelInMemory) + tr1, _ := NewTrie(trieStorage1, marsh, hsh, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, tenMBSize) + tr2, _ := NewTrie(trieStorage2, marsh, hsh, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, tenMBSize) maxIterations := 10000 for i := 0; i < maxIterations; i++ { @@ -359,7 +372,7 @@ func TestBranchNode_commit(t *testing.T) { hash, _ := encodeNodeAndGetHash(collapsedBn) _ = bn.setHash() - err := bn.commitDirty(0, 5, db, db) + err := bn.commitDirty(db, db, dtmc) assert.Nil(t, err) encNode, _ := db.Get(hash) @@ -374,7 +387,7 @@ func TestBranchNode_commitEmptyNode(t *testing.T) { bn := emptyDirtyBranchNode() - err := bn.commitDirty(0, 5, nil, nil) + err := bn.commitDirty(nil, nil, nil) assert.True(t, errors.Is(err, ErrEmptyBranchNode)) } @@ -383,7 +396,7 @@ func TestBranchNode_commitNilNode(t *testing.T) { var bn *branchNode - err := bn.commitDirty(0, 5, nil, nil) + err := bn.commitDirty(nil, nil, nil) assert.True(t, errors.Is(err, ErrNilBranchNode)) } @@ -428,14 +441,16 @@ func TestBranchNode_resolveCollapsed(t *testing.T) { childPos := byte(2) _ = bn.setHash() - _ = bn.commitDirty(0, 5, db, db) + _ = bn.commitDirty(db, db, dtmc) resolved, _ := newLeafNode(getTrieDataWithDefaultVersion("dog", "dog"), bn.marsh, bn.hasher) resolved.dirty = false resolved.hash = bn.EncodedChildren[childPos] - err := collapsedBn.resolveCollapsed(childPos, db) + tmc := trieMetricsCollector.NewTrieMetricsCollector() + err := collapsedBn.resolveCollapsed(childPos, tmc, db) assert.Nil(t, err) assert.Equal(t, resolved, collapsedBn.children[childPos]) + assert.Equal(t, collapsedBn.children[childPos].sizeInBytes(), tmc.GetSizeLoadedInMem()) } func TestBranchNode_resolveCollapsedEmptyNode(t *testing.T) { @@ -443,7 +458,7 @@ func TestBranchNode_resolveCollapsedEmptyNode(t *testing.T) { bn := emptyDirtyBranchNode() - err := bn.resolveCollapsed(2, nil) + err := bn.resolveCollapsed(2, nil, nil) assert.True(t, errors.Is(err, ErrEmptyBranchNode)) } @@ -452,7 +467,7 @@ func TestBranchNode_resolveCollapsedENilNode(t *testing.T) { var bn *branchNode - err := bn.resolveCollapsed(2, nil) + err := bn.resolveCollapsed(2, nil, nil) assert.True(t, errors.Is(err, ErrNilBranchNode)) } @@ -461,7 +476,7 @@ func TestBranchNode_resolveCollapsedPosOutOfRange(t *testing.T) { bn, _ := getBnAndCollapsedBn(getTestMarshalizerAndHasher()) - err := bn.resolveCollapsed(17, nil) + err := bn.resolveCollapsed(17, nil, nil) assert.Equal(t, ErrChildPosOutOfRange, err) } @@ -484,10 +499,11 @@ func TestBranchNode_tryGet(t *testing.T) { childPos := byte(2) key := append([]byte{childPos}, []byte("dog")...) - val, maxDepth, err := bn.tryGet(key, 0, nil) + tmc := trieMetricsCollector.NewTrieMetricsCollector() + val, err := bn.tryGet(key, tmc, nil) assert.Equal(t, []byte("dog"), val) assert.Nil(t, err) - assert.Equal(t, uint32(1), maxDepth) + assert.Equal(t, uint32(1), tmc.GetMaxDepth()) } func TestBranchNode_tryGetEmptyKey(t *testing.T) { @@ -496,10 +512,11 @@ func TestBranchNode_tryGetEmptyKey(t *testing.T) { bn, _ := getBnAndCollapsedBn(getTestMarshalizerAndHasher()) var key []byte - val, maxDepth, err := bn.tryGet(key, 0, nil) + tmc := trieMetricsCollector.NewTrieMetricsCollector() + val, err := bn.tryGet(key, tmc, nil) assert.Nil(t, err) assert.Nil(t, val) - assert.Equal(t, uint32(0), maxDepth) + assert.Equal(t, uint32(0), tmc.GetMaxDepth()) } func TestBranchNode_tryGetChildPosOutOfRange(t *testing.T) { @@ -508,10 +525,11 @@ func TestBranchNode_tryGetChildPosOutOfRange(t *testing.T) { bn, _ := getBnAndCollapsedBn(getTestMarshalizerAndHasher()) key := []byte("dog") - val, maxDepth, err := bn.tryGet(key, 0, nil) + tmc := trieMetricsCollector.NewTrieMetricsCollector() + val, err := bn.tryGet(key, tmc, nil) assert.Equal(t, ErrChildPosOutOfRange, err) assert.Nil(t, val) - assert.Equal(t, uint32(0), maxDepth) + assert.Equal(t, uint32(0), tmc.GetMaxDepth()) } func TestBranchNode_tryGetNilChild(t *testing.T) { @@ -520,10 +538,11 @@ func TestBranchNode_tryGetNilChild(t *testing.T) { bn, _ := getBnAndCollapsedBn(getTestMarshalizerAndHasher()) nilChildKey := []byte{3} - val, maxDepth, err := bn.tryGet(nilChildKey, 0, nil) + tmc := trieMetricsCollector.NewTrieMetricsCollector() + val, err := bn.tryGet(nilChildKey, tmc, nil) assert.Nil(t, err) assert.Nil(t, val) - assert.Equal(t, uint32(0), maxDepth) + assert.Equal(t, uint32(0), tmc.GetMaxDepth()) } func TestBranchNode_tryGetCollapsedNode(t *testing.T) { @@ -533,15 +552,16 @@ func TestBranchNode_tryGetCollapsedNode(t *testing.T) { bn, collapsedBn := getBnAndCollapsedBn(getTestMarshalizerAndHasher()) _ = bn.setHash() - _ = bn.commitDirty(0, 5, db, db) + _ = bn.commitDirty(db, db, dtmc) childPos := byte(2) key := append([]byte{childPos}, []byte("dog")...) - val, maxDepth, err := collapsedBn.tryGet(key, 0, db) + tmc := trieMetricsCollector.NewTrieMetricsCollector() + val, err := collapsedBn.tryGet(key, tmc, db) assert.Equal(t, []byte("dog"), val) assert.Nil(t, err) - assert.Equal(t, uint32(1), maxDepth) + assert.Equal(t, uint32(1), tmc.GetMaxDepth()) } func TestBranchNode_tryGetEmptyNode(t *testing.T) { @@ -551,10 +571,11 @@ func TestBranchNode_tryGetEmptyNode(t *testing.T) { childPos := byte(2) key := append([]byte{childPos}, []byte("dog")...) - val, maxDepth, err := bn.tryGet(key, 0, nil) + tmc := trieMetricsCollector.NewTrieMetricsCollector() + val, err := bn.tryGet(key, tmc, nil) assert.True(t, errors.Is(err, ErrEmptyBranchNode)) assert.Nil(t, val) - assert.Equal(t, uint32(0), maxDepth) + assert.Equal(t, uint32(0), tmc.GetMaxDepth()) } func TestBranchNode_tryGetNilNode(t *testing.T) { @@ -564,10 +585,11 @@ func TestBranchNode_tryGetNilNode(t *testing.T) { childPos := byte(2) key := append([]byte{childPos}, []byte("dog")...) - val, maxDepth, err := bn.tryGet(key, 0, nil) + tmc := trieMetricsCollector.NewTrieMetricsCollector() + val, err := bn.tryGet(key, tmc, nil) assert.True(t, errors.Is(err, ErrNilBranchNode)) assert.Nil(t, val) - assert.Equal(t, uint32(0), maxDepth) + assert.Equal(t, uint32(0), tmc.GetMaxDepth()) } func TestBranchNode_getNext(t *testing.T) { @@ -578,7 +600,7 @@ func TestBranchNode_getNext(t *testing.T) { childPos := byte(2) key := append([]byte{childPos}, []byte("dog")...) - n, key, err := bn.getNext(key, nil) + n, key, err := bn.getNext(key, dtmc, nil) h1, _ := encodeNodeAndGetHash(nextNode) h2, _ := encodeNodeAndGetHash(n) @@ -593,7 +615,7 @@ func TestBranchNode_getNextWrongKey(t *testing.T) { bn, _ := getBnAndCollapsedBn(getTestMarshalizerAndHasher()) key := []byte("dog") - n, key, err := bn.getNext(key, nil) + n, key, err := bn.getNext(key, dtmc, nil) assert.Nil(t, n) assert.Nil(t, key) assert.Equal(t, ErrChildPosOutOfRange, err) @@ -606,7 +628,7 @@ func TestBranchNode_getNextNilChild(t *testing.T) { nilChildPos := byte(4) key := append([]byte{nilChildPos}, []byte("dog")...) - n, key, err := bn.getNext(key, nil) + n, key, err := bn.getNext(key, dtmc, nil) assert.Nil(t, n) assert.Nil(t, key) assert.Equal(t, ErrNodeNotFound, err) @@ -618,9 +640,11 @@ func TestBranchNode_insert(t *testing.T) { bn, _ := getBnAndCollapsedBn(getTestMarshalizerAndHasher()) nodeKey := []byte{0, 2, 3} - newBn, _, err := bn.insert(getTrieDataWithDefaultVersion(string(nodeKey), "dogs"), nil) + tmc := trieMetricsCollector.NewTrieMetricsCollector() + newBn, _, err := bn.insert(getTrieDataWithDefaultVersion(string(nodeKey), "dogs"), tmc, nil) assert.NotNil(t, newBn) assert.Nil(t, err) + assert.Equal(t, bn.children[0].sizeInBytes(), tmc.GetSizeLoadedInMem()) nodeKeyRemainder := nodeKey[1:] bn.children[0], _ = newLeafNode(getTrieDataWithDefaultVersion(string(nodeKeyRemainder), "dogs"), bn.marsh, bn.hasher) @@ -632,7 +656,7 @@ func TestBranchNode_insertEmptyKey(t *testing.T) { bn, _ := getBnAndCollapsedBn(getTestMarshalizerAndHasher()) - newBn, _, err := bn.insert(getTrieDataWithDefaultVersion("", "dogs"), nil) + newBn, _, err := bn.insert(getTrieDataWithDefaultVersion("", "dogs"), dtmc, nil) assert.Equal(t, ErrValueTooShort, err) assert.Nil(t, newBn) } @@ -642,7 +666,7 @@ func TestBranchNode_insertChildPosOutOfRange(t *testing.T) { bn, _ := getBnAndCollapsedBn(getTestMarshalizerAndHasher()) - newBn, _, err := bn.insert(getTrieDataWithDefaultVersion("dog", "dogs"), nil) + newBn, _, err := bn.insert(getTrieDataWithDefaultVersion("dog", "dogs"), dtmc, nil) assert.Equal(t, ErrChildPosOutOfRange, err) assert.Nil(t, newBn) } @@ -655,14 +679,20 @@ func TestBranchNode_insertCollapsedNode(t *testing.T) { childPos := byte(2) key := append([]byte{childPos}, []byte("dog")...) + originalSize := bn.children[childPos].sizeInBytes() _ = bn.setHash() - _ = bn.commitDirty(0, 5, db, db) + _ = bn.commitDirty(db, db, dtmc) - newBn, _, err := collapsedBn.insert(getTrieDataWithDefaultVersion(string(key), "dogs"), db) + tmc := trieMetricsCollector.NewTrieMetricsCollector() + newBn, _, err := collapsedBn.insert(getTrieDataWithDefaultVersion(string(key), "dogs"), tmc, db) assert.NotNil(t, newBn) assert.Nil(t, err) - val, _, _ := newBn.tryGet(key, 0, db) + newSize := collapsedBn.children[childPos].sizeInBytes() + sizeDiff := newSize - originalSize + assert.Equal(t, originalSize+sizeDiff, tmc.GetSizeLoadedInMem()) + + val, _ := newBn.tryGet(key, trieMetricsCollector.NewTrieMetricsCollector(), db) assert.Equal(t, []byte("dogs"), val) } @@ -674,16 +704,21 @@ func TestBranchNode_insertInStoredBnOnExistingPos(t *testing.T) { childPos := byte(2) key := append([]byte{childPos}, []byte("dog")...) - _ = bn.commitDirty(0, 5, db, db) + originalSize := bn.children[childPos].sizeInBytes() + _ = bn.setHash() + markNotDirtyBranchNode(bn) bnHash := bn.getHash() - ln, _, _ := bn.getNext(key, db) - lnHash := ln.getHash() + lnHash := bn.EncodedChildren[childPos] expectedHashes := [][]byte{lnHash, bnHash} - newNode, oldHashes, err := bn.insert(getTrieDataWithDefaultVersion(string(key), "dogs"), db) + tmc := trieMetricsCollector.NewTrieMetricsCollector() + newNode, oldHashes, err := bn.insert(getTrieDataWithDefaultVersion(string(key), "dogs"), tmc, db) assert.NotNil(t, newNode) assert.Nil(t, err) assert.Equal(t, expectedHashes, oldHashes) + + newSize := bn.children[childPos].sizeInBytes() + assert.Equal(t, newSize-originalSize, tmc.GetSizeLoadedInMem()) } func TestBranchNode_insertInStoredBnOnNilPos(t *testing.T) { @@ -694,14 +729,17 @@ func TestBranchNode_insertInStoredBnOnNilPos(t *testing.T) { nilChildPos := byte(11) key := append([]byte{nilChildPos}, []byte("dog")...) - _ = bn.commitDirty(0, 5, db, db) + _ = bn.setHash() + markNotDirtyBranchNode(bn) bnHash := bn.getHash() expectedHashes := [][]byte{bnHash} - newNode, oldHashes, err := bn.insert(getTrieDataWithDefaultVersion(string(key), "dogs"), db) + tmc := trieMetricsCollector.NewTrieMetricsCollector() + newNode, oldHashes, err := bn.insert(getTrieDataWithDefaultVersion(string(key), "dogs"), tmc, db) assert.NotNil(t, newNode) assert.Nil(t, err) assert.Equal(t, expectedHashes, oldHashes) + assert.Equal(t, bn.children[nilChildPos].sizeInBytes(), tmc.GetSizeLoadedInMem()) } func TestBranchNode_insertInDirtyBnOnNilPos(t *testing.T) { @@ -711,10 +749,12 @@ func TestBranchNode_insertInDirtyBnOnNilPos(t *testing.T) { nilChildPos := byte(11) key := append([]byte{nilChildPos}, []byte("dog")...) - newNode, oldHashes, err := bn.insert(getTrieDataWithDefaultVersion(string(key), "dogs"), nil) + tmc := trieMetricsCollector.NewTrieMetricsCollector() + newNode, oldHashes, err := bn.insert(getTrieDataWithDefaultVersion(string(key), "dogs"), tmc, nil) assert.NotNil(t, newNode) assert.Nil(t, err) assert.Equal(t, [][]byte{}, oldHashes) + assert.Equal(t, bn.children[nilChildPos].sizeInBytes(), tmc.GetSizeLoadedInMem()) } func TestBranchNode_insertInDirtyBnOnExistingPos(t *testing.T) { @@ -724,10 +764,14 @@ func TestBranchNode_insertInDirtyBnOnExistingPos(t *testing.T) { childPos := byte(2) key := append([]byte{childPos}, []byte("dog")...) - newNode, oldHashes, err := bn.insert(getTrieDataWithDefaultVersion(string(key), "dogs"), nil) + originalSize := bn.children[childPos].sizeInBytes() + tmc := trieMetricsCollector.NewTrieMetricsCollector() + newNode, oldHashes, err := bn.insert(getTrieDataWithDefaultVersion(string(key), "dogs"), tmc, nil) assert.NotNil(t, newNode) assert.Nil(t, err) assert.Equal(t, [][]byte{}, oldHashes) + newSize := bn.children[childPos].sizeInBytes() + assert.Equal(t, newSize-originalSize, tmc.GetSizeLoadedInMem()) } func TestBranchNode_insertInNilNode(t *testing.T) { @@ -735,7 +779,7 @@ func TestBranchNode_insertInNilNode(t *testing.T) { var bn *branchNode - newBn, _, err := bn.insert(getTrieDataWithDefaultVersion("key", "dogs"), nil) + newBn, _, err := bn.insert(getTrieDataWithDefaultVersion("key", "dogs"), dtmc, nil) assert.True(t, errors.Is(err, ErrNilBranchNode)) assert.Nil(t, newBn) } @@ -753,9 +797,12 @@ func TestBranchNode_delete(t *testing.T) { childPos := byte(2) key := append([]byte{childPos}, []byte("dog")...) - dirty, newBn, _, err := bn.delete(key, nil) + originalSize := bn.children[childPos].sizeInBytes() + tmc := trieMetricsCollector.NewTrieMetricsCollector() + dirty, newBn, _, err := bn.delete(key, tmc, nil) assert.True(t, dirty) assert.Nil(t, err) + assert.Equal(t, -(originalSize + hashSizeInBytes), tmc.GetSizeLoadedInMem()) _ = expectedBn.setHash() _ = newBn.setHash() @@ -770,16 +817,19 @@ func TestBranchNode_deleteFromStoredBn(t *testing.T) { childPos := byte(2) lnKey := append([]byte{childPos}, []byte("dog")...) - _ = bn.commitDirty(0, 5, db, db) + originalSize := bn.children[childPos].sizeInBytes() + _ = bn.setHash() + markNotDirtyBranchNode(bn) bnHash := bn.getHash() - ln, _, _ := bn.getNext(lnKey, db) - lnHash := ln.getHash() + lnHash := bn.EncodedChildren[childPos] expectedHashes := [][]byte{lnHash, bnHash} - dirty, _, oldHashes, err := bn.delete(lnKey, db) + tmc := trieMetricsCollector.NewTrieMetricsCollector() + dirty, _, oldHashes, err := bn.delete(lnKey, tmc, db) assert.True(t, dirty) assert.Nil(t, err) assert.Equal(t, expectedHashes, oldHashes) + assert.Equal(t, -(originalSize + hashSizeInBytes), tmc.GetSizeLoadedInMem()) } func TestBranchNode_deleteFromDirtyBn(t *testing.T) { @@ -789,10 +839,13 @@ func TestBranchNode_deleteFromDirtyBn(t *testing.T) { childPos := byte(2) lnKey := append([]byte{childPos}, []byte("dog")...) - dirty, _, oldHashes, err := bn.delete(lnKey, nil) + originalSize := bn.children[childPos].sizeInBytes() + tmc := trieMetricsCollector.NewTrieMetricsCollector() + dirty, _, oldHashes, err := bn.delete(lnKey, tmc, nil) assert.True(t, dirty) assert.Nil(t, err) assert.Equal(t, [][]byte{}, oldHashes) + assert.Equal(t, -(originalSize + hashSizeInBytes), tmc.GetSizeLoadedInMem()) } func TestBranchNode_deleteEmptyNode(t *testing.T) { @@ -802,10 +855,12 @@ func TestBranchNode_deleteEmptyNode(t *testing.T) { childPos := byte(2) key := append([]byte{childPos}, []byte("dog")...) - dirty, newBn, _, err := bn.delete(key, nil) + tmc := trieMetricsCollector.NewTrieMetricsCollector() + dirty, newBn, _, err := bn.delete(key, tmc, nil) assert.False(t, dirty) assert.True(t, errors.Is(err, ErrEmptyBranchNode)) assert.Nil(t, newBn) + assert.Equal(t, 0, tmc.GetSizeLoadedInMem()) } func TestBranchNode_deleteNilNode(t *testing.T) { @@ -815,10 +870,12 @@ func TestBranchNode_deleteNilNode(t *testing.T) { childPos := byte(2) key := append([]byte{childPos}, []byte("dog")...) - dirty, newBn, _, err := bn.delete(key, nil) + tmc := trieMetricsCollector.NewTrieMetricsCollector() + dirty, newBn, _, err := bn.delete(key, tmc, nil) assert.False(t, dirty) assert.True(t, errors.Is(err, ErrNilBranchNode)) assert.Nil(t, newBn) + assert.Equal(t, 0, tmc.GetSizeLoadedInMem()) } func TestBranchNode_deleteNonexistentNodeFromChild(t *testing.T) { @@ -829,10 +886,12 @@ func TestBranchNode_deleteNonexistentNodeFromChild(t *testing.T) { childPos := byte(2) key := append([]byte{childPos}, []byte("butterfly")...) - dirty, newBn, _, err := bn.delete(key, nil) + tmc := trieMetricsCollector.NewTrieMetricsCollector() + dirty, newBn, _, err := bn.delete(key, tmc, nil) assert.False(t, dirty) assert.Nil(t, err) assert.Equal(t, bn, newBn) + assert.Equal(t, 0, tmc.GetSizeLoadedInMem()) } func TestBranchNode_deleteEmptykey(t *testing.T) { @@ -840,10 +899,12 @@ func TestBranchNode_deleteEmptykey(t *testing.T) { bn, _ := getBnAndCollapsedBn(getTestMarshalizerAndHasher()) - dirty, newBn, _, err := bn.delete([]byte{}, nil) + tmc := trieMetricsCollector.NewTrieMetricsCollector() + dirty, newBn, _, err := bn.delete([]byte{}, tmc, nil) assert.False(t, dirty) assert.Equal(t, ErrValueTooShort, err) assert.Nil(t, newBn) + assert.Equal(t, 0, tmc.GetSizeLoadedInMem()) } func TestBranchNode_deleteCollapsedNode(t *testing.T) { @@ -852,16 +913,18 @@ func TestBranchNode_deleteCollapsedNode(t *testing.T) { db := testscommon.NewMemDbMock() bn, collapsedBn := getBnAndCollapsedBn(getTestMarshalizerAndHasher()) _ = bn.setHash() - _ = bn.commitDirty(0, 5, db, db) + _ = bn.commitDirty(db, db, dtmc) childPos := byte(2) key := append([]byte{childPos}, []byte("dog")...) - dirty, newBn, _, err := collapsedBn.delete(key, db) + tmc := trieMetricsCollector.NewTrieMetricsCollector() + dirty, newBn, _, err := collapsedBn.delete(key, tmc, db) assert.True(t, dirty) assert.Nil(t, err) + assert.Equal(t, -hashSizeInBytes, tmc.GetSizeLoadedInMem()) - val, _, err := newBn.tryGet(key, 0, db) + val, err := newBn.tryGet(key, trieMetricsCollector.NewTrieMetricsCollector(), db) assert.Nil(t, val) assert.Nil(t, err) } @@ -876,15 +939,21 @@ func TestBranchNode_deleteAndReduceBn(t *testing.T) { children[firstChildPos], _ = newLeafNode(getTrieDataWithDefaultVersion("dog", "dog"), bn.marsh, bn.hasher) children[secondChildPos], _ = newLeafNode(getTrieDataWithDefaultVersion("doe", "doe"), bn.marsh, bn.hasher) bn.children = children + bn.EncodedChildren[firstChildPos], _ = encodeNodeAndGetHash(children[firstChildPos]) + bn.EncodedChildren[secondChildPos], _ = encodeNodeAndGetHash(children[secondChildPos]) + extraLeafData := 1 + expectedSizeInMem := -bn.children[secondChildPos].sizeInBytes() - bn.sizeInBytes() + extraLeafData key := append([]byte{firstChildPos}, []byte("dog")...) ln, _ := newLeafNode(getTrieDataWithDefaultVersion(string(key), "dog"), bn.marsh, bn.hasher) + tmc := trieMetricsCollector.NewTrieMetricsCollector() key = append([]byte{secondChildPos}, []byte("doe")...) - dirty, newBn, _, err := bn.delete(key, nil) + dirty, newBn, _, err := bn.delete(key, tmc, nil) assert.True(t, dirty) assert.Nil(t, err) assert.Equal(t, ln, newBn) + assert.Equal(t, expectedSizeInMem, tmc.GetSizeLoadedInMem()) } func TestBranchNode_reduceNode(t *testing.T) { @@ -899,7 +968,7 @@ func TestBranchNode_reduceNode(t *testing.T) { key := append([]byte{childPos}, []byte("dog")...) ln, _ := newLeafNode(getTrieDataWithDefaultVersion(string(key), "dog"), bn.marsh, bn.hasher) - n, newChildHash, err := bn.children[childPos].reduceNode(int(childPos)) + n, newChildHash, err := bn.children[childPos].reduceNode(int(childPos), dtmc) assert.Equal(t, ln, n) assert.Nil(t, err) assert.True(t, newChildHash) @@ -1015,7 +1084,7 @@ func TestBranchNode_getChildren(t *testing.T) { bn, _ := getBnAndCollapsedBn(getTestMarshalizerAndHasher()) - children, err := bn.getChildren(nil) + children, err := bn.getChildren(dtmc, nil) assert.Nil(t, err) assert.Equal(t, 3, len(children)) } @@ -1025,9 +1094,9 @@ func TestBranchNode_getChildrenCollapsedBn(t *testing.T) { db := testscommon.NewMemDbMock() bn, collapsedBn := getBnAndCollapsedBn(getTestMarshalizerAndHasher()) - _ = bn.commitDirty(0, 5, db, db) + _ = bn.commitDirty(db, db, dtmc) - children, err := collapsedBn.getChildren(db) + children, err := collapsedBn.getChildren(dtmc, db) assert.Nil(t, err) assert.Equal(t, 3, len(children)) } @@ -1189,20 +1258,6 @@ func TestBranchNode_setRootHashCollapsedChildren(t *testing.T) { assert.Nil(t, err) } -func TestBranchNode_commitCollapsesTrieIfMaxTrieLevelInMemoryIsReached(t *testing.T) { - t.Parallel() - - bn, collapsedBn := getBnAndCollapsedBn(getTestMarshalizerAndHasher()) - _ = collapsedBn.setRootHash() - - err := bn.commitDirty(0, 1, testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) - assert.Nil(t, err) - - assert.Equal(t, collapsedBn.EncodedChildren, bn.EncodedChildren) - assert.Equal(t, collapsedBn.children, bn.children) - assert.Equal(t, collapsedBn.hash, bn.hash) -} - func TestBranchNode_reduceNodeBnChild(t *testing.T) { t.Parallel() @@ -1211,7 +1266,7 @@ func TestBranchNode_reduceNodeBnChild(t *testing.T) { pos := 5 expectedNode, _ := newExtensionNode([]byte{byte(pos)}, en.child, marsh, hasher) - newNode, newChildHash, err := en.child.reduceNode(pos) + newNode, newChildHash, err := en.child.reduceNode(pos, dtmc) assert.Nil(t, err) assert.Equal(t, expectedNode, newNode) assert.False(t, newChildHash) @@ -1225,11 +1280,11 @@ func TestBranchNode_printShouldNotPanicEvenIfNodeIsCollapsed(t *testing.T) { db := testscommon.NewMemDbMock() bn, collapsedBn := getBnAndCollapsedBn(getTestMarshalizerAndHasher()) - _ = bn.commitDirty(0, 5, db, db) - _ = collapsedBn.commitDirty(0, 5, db, db) + _ = bn.commitDirty(db, db, dtmc) + _ = collapsedBn.commitDirty(db, db, dtmc) - bn.print(bnWriter, 0, db) - collapsedBn.print(collapsedBnWriter, 0, db) + bn.print(bnWriter, 0, dtmc, db) + collapsedBn.print(collapsedBnWriter, 0, dtmc, db) assert.Equal(t, bnWriter.Bytes(), collapsedBnWriter.Bytes()) } @@ -1239,7 +1294,7 @@ func TestBranchNode_getDirtyHashesFromCleanNode(t *testing.T) { db := testscommon.NewMemDbMock() bn, _ := getBnAndCollapsedBn(getTestMarshalizerAndHasher()) - _ = bn.commitDirty(0, 5, db, db) + _ = bn.commitDirty(db, db, dtmc) dirtyHashes := make(common.ModifiedHashes) err := bn.getDirtyHashes(dirtyHashes) @@ -1253,7 +1308,7 @@ func TestBranchNode_getAllHashes(t *testing.T) { trieNodes := 4 bn, _ := getBnAndCollapsedBn(getTestMarshalizerAndHasher()) - hashes, err := bn.getAllHashes(testscommon.NewMemDbMock()) + hashes, err := bn.getAllHashes(dtmc, testscommon.NewMemDbMock()) assert.Nil(t, err) assert.Equal(t, trieNodes, len(hashes)) } @@ -1263,9 +1318,9 @@ func TestBranchNode_getAllHashesResolvesCollapsed(t *testing.T) { db := testscommon.NewMemDbMock() bn, collapsedBn := getBnAndCollapsedBn(getTestMarshalizerAndHasher()) - _ = bn.commitDirty(0, 5, db, db) + _ = bn.commitDirty(db, db, dtmc) - hashes, err := collapsedBn.getAllHashes(db) + hashes, err := collapsedBn.getAllHashes(dtmc, db) assert.Nil(t, err) assert.Equal(t, 4, len(hashes)) } @@ -1311,10 +1366,11 @@ func TestBranchNode_SizeInBytes(t *testing.T) { collapsed1 := []byte("collapsed1") collapsed2 := []byte("collapsed2") - hash := []byte("hash") + hash := bytes.Repeat([]byte{1}, 32) bn = &branchNode{ CollapsedBn: CollapsedBn{ EncodedChildren: [][]byte{collapsed1, collapsed2}, + ChildrenVersion: []byte("version"), }, children: [17]node{}, baseNode: &baseNode{ @@ -1324,7 +1380,8 @@ func TestBranchNode_SizeInBytes(t *testing.T) { hasher: nil, }, } - assert.Equal(t, len(collapsed1)+len(collapsed2)+len(hash)+1+19*pointerSizeInBytes, bn.sizeInBytes()) + numChildren := 2 + assert.Equal(t, numChildren*hashSizeInBytes+len(hash)+1+19*pointerSizeInBytes+len(bn.ChildrenVersion), bn.sizeInBytes()) } func TestBranchNode_commitContextDone(t *testing.T) { @@ -1335,7 +1392,7 @@ func TestBranchNode_commitContextDone(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() - err := bn.commitSnapshot(db, 0, nil, nil, ctx, statistics.NewTrieStatistics(), &testscommon.ProcessStatusHandlerStub{}, []byte{}, 0) + err := bn.commitSnapshot(db, 0, nil, nil, ctx, statistics.NewTrieStatistics(), &testscommon.ProcessStatusHandlerStub{}, []byte{}, dtmc) assert.Equal(t, core.ErrContextClosing, err) } @@ -1349,7 +1406,7 @@ func TestBranchNode_commitSnapshotDbIsClosing(t *testing.T) { _, collapsedBn := getBnAndCollapsedBn(getTestMarshalizerAndHasher()) missingNodesChan := make(chan []byte, 10) - err := collapsedBn.commitSnapshot(db, 0, nil, missingNodesChan, context.Background(), statistics.NewTrieStatistics(), &testscommon.ProcessStatusHandlerStub{}, []byte{}, 0) + err := collapsedBn.commitSnapshot(db, 0, nil, missingNodesChan, context.Background(), statistics.NewTrieStatistics(), &testscommon.ProcessStatusHandlerStub{}, []byte{}, dtmc) assert.True(t, core.IsClosingError(err)) assert.Equal(t, 0, len(missingNodesChan)) } @@ -1364,7 +1421,7 @@ func TestBranchNode_commitSnapshotChildIsMissingErr(t *testing.T) { _, collapsedBn := getBnAndCollapsedBn(getTestMarshalizerAndHasher()) missingNodesChan := make(chan []byte, 10) - err := collapsedBn.commitSnapshot(db, 0, nil, missingNodesChan, context.Background(), statistics.NewTrieStatistics(), &testscommon.ProcessStatusHandlerStub{}, []byte{}, 0) + err := collapsedBn.commitSnapshot(db, 0, nil, missingNodesChan, context.Background(), statistics.NewTrieStatistics(), &testscommon.ProcessStatusHandlerStub{}, []byte{}, dtmc) assert.Nil(t, err) assert.Equal(t, 3, len(missingNodesChan)) } @@ -1448,7 +1505,7 @@ func TestBranchNode_VerifyChildrenVersionIsSetCorrectlyAfterInsertAndDelete(t *t Value: []byte("value"), Version: 0, } - newBn, _, err := bn.insert(data, &testscommon.MemDbMock{}) + newBn, _, err := bn.insert(data, dtmc, &testscommon.MemDbMock{}) assert.Nil(t, err) assert.Nil(t, newBn.(*branchNode).ChildrenVersion) }) @@ -1461,7 +1518,7 @@ func TestBranchNode_VerifyChildrenVersionIsSetCorrectlyAfterInsertAndDelete(t *t bn.ChildrenVersion[2] = byte(core.AutoBalanceEnabled) childKey := []byte{2, 'd', 'o', 'g'} - _, newBn, _, err := bn.delete(childKey, &testscommon.MemDbMock{}) + _, newBn, _, err := bn.delete(childKey, dtmc, &testscommon.MemDbMock{}) assert.Nil(t, err) assert.Nil(t, newBn.(*branchNode).ChildrenVersion) }) @@ -1558,3 +1615,24 @@ func TestBranchNode_getNodeData(t *testing.T) { assert.False(t, thirdChildData.IsLeaf()) }) } + +func TestBranchNode_commitCollapsesLeaves(t *testing.T) { + t.Parallel() + + tr := initTrie() + en := tr.root.(*branchNode).children[7] + bn := en.(*extensionNode).child.(*branchNode) + + assert.NotNil(t, tr.root.(*branchNode).children[5]) + assert.NotNil(t, tr.root.(*branchNode).children[7]) + assert.NotNil(t, en.(*extensionNode).child) + assert.NotNil(t, bn.children[4]) + assert.NotNil(t, bn.children[16]) + _ = tr.Commit() + + assert.Nil(t, tr.root.(*branchNode).children[5]) // leaf node is collapsed + assert.NotNil(t, tr.root.(*branchNode).children[7]) + assert.NotNil(t, en.(*extensionNode).child) + assert.Nil(t, bn.children[4]) // leaf node is collapsed + assert.Nil(t, bn.children[16]) // leaf node is collapsed +} diff --git a/trie/doubleListSync_test.go b/trie/doubleListSync_test.go index 8e631237cc6..41ebc3cdd77 100644 --- a/trie/doubleListSync_test.go +++ b/trie/doubleListSync_test.go @@ -37,7 +37,7 @@ func createTrieStorageManager(store storage.Storer) (common.StorageManager, stor func createInMemoryTrie() (common.Trie, storage.Storer) { memUnit := testscommon.CreateMemUnit() tsm, _ := createTrieStorageManager(memUnit) - tr, _ := NewTrie(tsm, marshalizer, hasherMock, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, 6) + tr, _ := NewTrie(tsm, marshalizer, hasherMock, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, tenMBSize) return tr, memUnit } @@ -50,7 +50,7 @@ func createInMemoryTrieFromDB(db storage.Persister) (common.Trie, storage.Storer unit, _ := storageunit.NewStorageUnit(cache, db) tsm, _ := createTrieStorageManager(unit) - tr, _ := NewTrie(tsm, marshalizer, hasherMock, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, 6) + tr, _ := NewTrie(tsm, marshalizer, hasherMock, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, tenMBSize) return tr, unit } diff --git a/trie/errors.go b/trie/errors.go index a879fd6c94c..54ec2ec1d3e 100644 --- a/trie/errors.go +++ b/trie/errors.go @@ -129,3 +129,6 @@ var ErrEmptyInitialIteratorState = errors.New("empty initial iterator state") // ErrInvalidIteratorState signals that an invalid iterator state was provided var ErrInvalidIteratorState = errors.New("invalid iterator state") + +// ErrInvalidMaxSizeInMemory signals that the provided max trie size value is invalid +var ErrInvalidMaxSizeInMemory = errors.New("invalid max size in memory") diff --git a/trie/extensionNode.go b/trie/extensionNode.go index 0c1a657665b..1d43f6d1fe6 100644 --- a/trie/extensionNode.go +++ b/trie/extensionNode.go @@ -173,8 +173,7 @@ func (en *extensionNode) hashNode() ([]byte, error) { return encodeNodeAndGetHash(en) } -func (en *extensionNode) commitDirty(level byte, maxTrieLevelInMemory uint, originDb common.TrieStorageInteractor, targetDb common.BaseStorer) error { - level++ +func (en *extensionNode) commitDirty(originDb common.TrieStorageInteractor, targetDb common.BaseStorer, tmc MetricsCollector) error { err := en.isEmptyOrNil() if err != nil { return fmt.Errorf("commit error %w", err) @@ -185,7 +184,7 @@ func (en *extensionNode) commitDirty(level byte, maxTrieLevelInMemory uint, orig } if en.child != nil { - err = en.child.commitDirty(level, maxTrieLevelInMemory, originDb, targetDb) + err = en.child.commitDirty(originDb, targetDb, tmc) if err != nil { return err } @@ -196,17 +195,7 @@ func (en *extensionNode) commitDirty(level byte, maxTrieLevelInMemory uint, orig if err != nil { return err } - if uint(level) == maxTrieLevelInMemory { - log.Trace("collapse extension node on commit") - var collapsedEn *extensionNode - collapsedEn, err = en.getCollapsedEn() - if err != nil { - return err - } - - *en = *collapsedEn - } return nil } @@ -219,12 +208,16 @@ func (en *extensionNode) commitSnapshot( stats common.TrieStatisticsHandler, idleProvider IdleNodeProvider, nodeBytes []byte, - depthLevel int, + tmc MetricsCollector, ) error { if shouldStopIfContextDoneBlockingIfBusy(ctx, idleProvider) { return core.ErrContextClosing } + depthLevel := tmc.GetCurrentDepth() + tmc.SetDepth(depthLevel + 1) + defer tmc.SetDepth(depthLevel) // Reset depth when returning from this level + err := commitSnapshot( db, maxEpochToSearchFrom, @@ -235,14 +228,14 @@ func (en *extensionNode) commitSnapshot( ctx, stats, idleProvider, - depthLevel, + tmc, en.EncodedChild, ) if err != nil { return err } - stats.AddExtensionNode(depthLevel, uint64(len(nodeBytes))) + stats.AddExtensionNode(int(depthLevel), uint64(len(nodeBytes))) return nil } @@ -259,7 +252,7 @@ func (en *extensionNode) getEncodedNode() ([]byte, error) { return marshaledNode, nil } -func (en *extensionNode) resolveCollapsed(_ byte, db common.TrieStorageInteractor) error { +func (en *extensionNode) resolveCollapsed(_ byte, tmc MetricsCollector, db common.TrieStorageInteractor) error { err := en.isEmptyOrNil() if err != nil { return fmt.Errorf("resolveCollapsed error %w", err) @@ -269,6 +262,7 @@ func (en *extensionNode) resolveCollapsed(_ byte, db common.TrieStorageInteracto return err } child.setGivenHash(en.EncodedChild) + tmc.AddSizeLoadedInMem(child.sizeInBytes()) en.child = child return nil } @@ -281,29 +275,30 @@ func (en *extensionNode) isPosCollapsed(_ int) bool { return en.isCollapsed() } -func (en *extensionNode) tryGet(key []byte, currentDepth uint32, db common.TrieStorageInteractor) (value []byte, maxDepth uint32, err error) { +func (en *extensionNode) tryGet(key []byte, tmc MetricsCollector, db common.TrieStorageInteractor) (value []byte, err error) { err = en.isEmptyOrNil() if err != nil { - return nil, currentDepth, fmt.Errorf("tryGet error %w", err) + return nil, fmt.Errorf("tryGet error %w", err) } keyTooShort := len(key) < len(en.Key) if keyTooShort { - return nil, currentDepth, nil + return nil, nil } keysDontMatch := !bytes.Equal(en.Key, key[:len(en.Key)]) if keysDontMatch { - return nil, currentDepth, nil + return nil, nil } key = key[len(en.Key):] - err = resolveIfCollapsed(en, 0, db) + err = resolveIfCollapsed(en, 0, tmc, db) if err != nil { - return nil, currentDepth, err + return nil, err } - return en.child.tryGet(key, currentDepth+1, db) + tmc.SetDepth(tmc.GetCurrentDepth() + 1) + return en.child.tryGet(key, tmc, db) } -func (en *extensionNode) getNext(key []byte, db common.TrieStorageInteractor) (node, []byte, error) { +func (en *extensionNode) getNext(key []byte, tmc MetricsCollector, db common.TrieStorageInteractor) (node, []byte, error) { err := en.isEmptyOrNil() if err != nil { return nil, nil, fmt.Errorf("getNext error %w", err) @@ -316,7 +311,7 @@ func (en *extensionNode) getNext(key []byte, db common.TrieStorageInteractor) (n if keysDontMatch { return nil, nil, ErrNodeNotFound } - err = resolveIfCollapsed(en, 0, db) + err = resolveIfCollapsed(en, 0, tmc, db) if err != nil { return nil, nil, err } @@ -325,13 +320,13 @@ func (en *extensionNode) getNext(key []byte, db common.TrieStorageInteractor) (n return en.child, key, nil } -func (en *extensionNode) insert(newData core.TrieData, db common.TrieStorageInteractor) (node, [][]byte, error) { +func (en *extensionNode) insert(newData core.TrieData, tmc MetricsCollector, db common.TrieStorageInteractor) (node, [][]byte, error) { emptyHashes := make([][]byte, 0) err := en.isEmptyOrNil() if err != nil { return nil, emptyHashes, fmt.Errorf("insert error %w", err) } - err = resolveIfCollapsed(en, 0, db) + err = resolveIfCollapsed(en, 0, tmc, db) if err != nil { return nil, emptyHashes, err } @@ -341,16 +336,16 @@ func (en *extensionNode) insert(newData core.TrieData, db common.TrieStorageInte // If the whole key matches, keep this extension node as is // and only update the value. if keyMatchLen == len(en.Key) { - return en.insertInSameEn(newData, keyMatchLen, db) + return en.insertInSameEn(newData, keyMatchLen, tmc, db) } // Otherwise branch out at the index where they differ. - return en.insertInNewBn(newData, keyMatchLen) + return en.insertInNewBn(newData, keyMatchLen, tmc) } -func (en *extensionNode) insertInSameEn(newData core.TrieData, keyMatchLen int, db common.TrieStorageInteractor) (node, [][]byte, error) { +func (en *extensionNode) insertInSameEn(newData core.TrieData, keyMatchLen int, tmc MetricsCollector, db common.TrieStorageInteractor) (node, [][]byte, error) { newData.Key = newData.Key[keyMatchLen:] - newNode, oldHashes, err := en.child.insert(newData, db) + newNode, oldHashes, err := en.child.insert(newData, tmc, db) if check.IfNil(newNode) || err != nil { return nil, [][]byte{}, err } @@ -367,7 +362,7 @@ func (en *extensionNode) insertInSameEn(newData core.TrieData, keyMatchLen int, return newEn, oldHashes, nil } -func (en *extensionNode) insertInNewBn(newData core.TrieData, keyMatchLen int) (node, [][]byte, error) { +func (en *extensionNode) insertInNewBn(newData core.TrieData, keyMatchLen int, tmc MetricsCollector) (node, [][]byte, error) { oldHash := make([][]byte, 0) if !en.dirty { oldHash = append(oldHash, en.hash) @@ -384,16 +379,17 @@ func (en *extensionNode) insertInNewBn(newData core.TrieData, keyMatchLen int) ( return nil, [][]byte{}, ErrChildPosOutOfRange } - err = en.insertOldChildInBn(bn, oldChildPos, keyMatchLen) + err = en.insertOldChildInBn(bn, oldChildPos, keyMatchLen, tmc) if err != nil { return nil, [][]byte{}, err } - err = en.insertNewChildInBn(bn, newData, newChildPos, keyMatchLen) + err = en.insertNewChildInBn(bn, newData, newChildPos, keyMatchLen, tmc) if err != nil { return nil, [][]byte{}, err } + tmc.AddSizeLoadedInMem(bn.sizeInBytes()) if keyMatchLen == 0 { return bn, oldHash, nil } @@ -403,10 +399,12 @@ func (en *extensionNode) insertInNewBn(newData core.TrieData, keyMatchLen int) ( return nil, [][]byte{}, err } + tmc.AddSizeLoadedInMem(newEn.sizeInBytes()) + return newEn, oldHash, nil } -func (en *extensionNode) insertOldChildInBn(bn *branchNode, oldChildPos byte, keyMatchLen int) error { +func (en *extensionNode) insertOldChildInBn(bn *branchNode, oldChildPos byte, keyMatchLen int, tmc MetricsCollector) error { keyReminder := en.Key[keyMatchLen+1:] childVersion, err := en.child.getVersion() if err != nil { @@ -415,10 +413,12 @@ func (en *extensionNode) insertOldChildInBn(bn *branchNode, oldChildPos byte, ke bn.setVersionForChild(childVersion, oldChildPos) if len(keyReminder) < 1 { + tmc.AddSizeLoadedInMem(-en.sizeInBytes()) bn.children[oldChildPos] = en.child return nil } + tmc.AddSizeLoadedInMem(-keyMatchLen) followingExtensionNode, err := newExtensionNode(en.Key[keyMatchLen+1:], en.child, en.marsh, en.hasher) if err != nil { return err @@ -428,7 +428,7 @@ func (en *extensionNode) insertOldChildInBn(bn *branchNode, oldChildPos byte, ke return nil } -func (en *extensionNode) insertNewChildInBn(bn *branchNode, newData core.TrieData, newChildPos byte, keyMatchLen int) error { +func (en *extensionNode) insertNewChildInBn(bn *branchNode, newData core.TrieData, newChildPos byte, keyMatchLen int, tmc MetricsCollector) error { newData.Key = newData.Key[keyMatchLen+1:] newLeaf, err := newLeafNode(newData, en.marsh, en.hasher) @@ -436,12 +436,14 @@ func (en *extensionNode) insertNewChildInBn(bn *branchNode, newData core.TrieDat return err } + tmc.AddSizeLoadedInMem(newLeaf.sizeInBytes()) + bn.children[newChildPos] = newLeaf bn.setVersionForChild(newData.Version, newChildPos) return nil } -func (en *extensionNode) delete(key []byte, db common.TrieStorageInteractor) (bool, node, [][]byte, error) { +func (en *extensionNode) delete(key []byte, tmc MetricsCollector, db common.TrieStorageInteractor) (bool, node, [][]byte, error) { emptyHashes := make([][]byte, 0) err := en.isEmptyOrNil() if err != nil { @@ -454,12 +456,12 @@ func (en *extensionNode) delete(key []byte, db common.TrieStorageInteractor) (bo if keyMatchLen < len(en.Key) { return false, en, emptyHashes, nil } - err = resolveIfCollapsed(en, 0, db) + err = resolveIfCollapsed(en, 0, tmc, db) if err != nil { return false, nil, emptyHashes, err } - dirty, newNode, oldHashes, err := en.child.delete(key[len(en.Key):], db) + dirty, newNode, oldHashes, err := en.child.delete(key[len(en.Key):], tmc, db) if !dirty || err != nil { return false, en, emptyHashes, err } @@ -468,6 +470,7 @@ func (en *extensionNode) delete(key []byte, db common.TrieStorageInteractor) (bo oldHashes = append(oldHashes, en.hash) } + tmc.AddSizeLoadedInMem(-en.sizeInBytes()) switch newNode := newNode.(type) { case *leafNode: newLeafData := core.TrieData{ @@ -479,6 +482,7 @@ func (en *extensionNode) delete(key []byte, db common.TrieStorageInteractor) (bo if err != nil { return false, nil, emptyHashes, err } + tmc.AddSizeLoadedInMem(len(en.Key)) return true, n, oldHashes, nil case *extensionNode: @@ -486,6 +490,7 @@ func (en *extensionNode) delete(key []byte, db common.TrieStorageInteractor) (bo if err != nil { return false, nil, emptyHashes, err } + tmc.AddSizeLoadedInMem(len(en.Key)) return true, n, oldHashes, nil case *branchNode: @@ -493,6 +498,7 @@ func (en *extensionNode) delete(key []byte, db common.TrieStorageInteractor) (bo if err != nil { return false, nil, emptyHashes, err } + tmc.AddSizeLoadedInMem(n.sizeInBytes()) return true, n, oldHashes, nil case nil: @@ -503,13 +509,15 @@ func (en *extensionNode) delete(key []byte, db common.TrieStorageInteractor) (bo } } -func (en *extensionNode) reduceNode(pos int) (node, bool, error) { - k := append([]byte{byte(pos)}, en.Key...) +func (en *extensionNode) reduceNode(pos int, tmc MetricsCollector) (node, bool, error) { + extraKey := []byte{byte(pos)} + k := append(extraKey, en.Key...) newEn, err := newExtensionNode(k, en.child, en.marsh, en.hasher) if err != nil { return nil, false, err } + tmc.AddSizeLoadedInMem(len(extraKey)) return newEn, true, nil } @@ -529,12 +537,12 @@ func (en *extensionNode) isEmptyOrNil() error { return nil } -func (en *extensionNode) print(writer io.Writer, index int, db common.TrieStorageInteractor) { +func (en *extensionNode) print(writer io.Writer, index int, tmc MetricsCollector, db common.TrieStorageInteractor) { if en == nil { return } - err := resolveIfCollapsed(en, 0, db) + err := resolveIfCollapsed(en, 0, tmc, db) if err != nil { log.Debug("extension node: print trie err", "error", err, "hash", en.EncodedChild) } @@ -550,7 +558,7 @@ func (en *extensionNode) print(writer io.Writer, index int, db common.TrieStorag if en.child == nil { return } - en.child.print(writer, index+len(str), db) + en.child.print(writer, index+len(str), tmc, db) } func (en *extensionNode) getDirtyHashes(hashes common.ModifiedHashes) error { @@ -576,7 +584,7 @@ func (en *extensionNode) getDirtyHashes(hashes common.ModifiedHashes) error { return nil } -func (en *extensionNode) getChildren(db common.TrieStorageInteractor) ([]node, error) { +func (en *extensionNode) getChildren(tmc MetricsCollector, db common.TrieStorageInteractor) ([]node, error) { err := en.isEmptyOrNil() if err != nil { return nil, fmt.Errorf("getChildren error %w", err) @@ -584,7 +592,7 @@ func (en *extensionNode) getChildren(db common.TrieStorageInteractor) ([]node, e nextNodes := make([]node, 0) - err = resolveIfCollapsed(en, 0, db) + err = resolveIfCollapsed(en, 0, tmc, db) if err != nil { return nil, err } @@ -638,6 +646,7 @@ func (en *extensionNode) getAllLeavesOnChannel( marshalizer marshal.Marshalizer, chanClose chan struct{}, ctx context.Context, + tmc MetricsCollector, ) error { err := en.isEmptyOrNil() if err != nil { @@ -652,13 +661,13 @@ func (en *extensionNode) getAllLeavesOnChannel( log.Trace("extensionNode.getAllLeavesOnChannel: context done") return nil default: - err = resolveIfCollapsed(en, 0, db) + err = resolveIfCollapsed(en, 0, tmc, db) if err != nil { return err } keyBuilder.BuildKey(en.Key) - err = en.child.getAllLeavesOnChannel(leavesChannel, keyBuilder.ShallowClone(), trieLeafParser, db, marshalizer, chanClose, ctx) + err = en.child.getAllLeavesOnChannel(leavesChannel, keyBuilder.ShallowClone(), trieLeafParser, db, marshalizer, chanClose, ctx, tmc) if err != nil { return err } @@ -669,18 +678,18 @@ func (en *extensionNode) getAllLeavesOnChannel( return nil } -func (en *extensionNode) getAllHashes(db common.TrieStorageInteractor) ([][]byte, error) { +func (en *extensionNode) getAllHashes(tmc MetricsCollector, db common.TrieStorageInteractor) ([][]byte, error) { err := en.isEmptyOrNil() if err != nil { return nil, fmt.Errorf("getAllHashes error: %w", err) } - err = resolveIfCollapsed(en, 0, db) + err = resolveIfCollapsed(en, 0, tmc, db) if err != nil { return nil, err } - hashes, err := en.child.getAllHashes(db) + hashes, err := en.child.getAllHashes(tmc, db) if err != nil { return nil, err } @@ -706,9 +715,8 @@ func (en *extensionNode) sizeInBytes() int { return 0 } - // hasher + marshalizer + child + dirty flag = 3 * pointerSizeInBytes + 1 - nodeSize := len(en.hash) + len(en.Key) + (numNodeInnerPointers+1)*pointerSizeInBytes + 1 - nodeSize += len(en.EncodedChild) + nodeSize := baseNodeSizeInBytes + len(en.Key) + nodeVersionSizeInBytes + pointerSizeInBytes + nodeSize += hashSizeInBytes // child hash return nodeSize } @@ -717,18 +725,22 @@ func (en *extensionNode) getValue() []byte { return []byte{} } -func (en *extensionNode) collectStats(ts common.TrieStatisticsHandler, depthLevel int, db common.TrieStorageInteractor) error { +func (en *extensionNode) collectStats(ts common.TrieStatisticsHandler, tmc MetricsCollector, db common.TrieStorageInteractor) error { err := en.isEmptyOrNil() if err != nil { return fmt.Errorf("collectStats error %w", err) } - err = resolveIfCollapsed(en, 0, db) + err = resolveIfCollapsed(en, 0, tmc, db) if err != nil { return err } - err = en.child.collectStats(ts, depthLevel+1, db) + depthLevel := tmc.GetCurrentDepth() + tmc.SetDepth(depthLevel + 1) + defer tmc.SetDepth(depthLevel) // Reset depth when returning from this level + + err = en.child.collectStats(ts, tmc, db) if err != nil { return err } @@ -738,7 +750,7 @@ func (en *extensionNode) collectStats(ts common.TrieStatisticsHandler, depthLeve return err } - ts.AddExtensionNode(depthLevel, uint64(len(val))) + ts.AddExtensionNode(int(depthLevel), uint64(len(val))) return nil } @@ -753,6 +765,7 @@ func (en *extensionNode) getVersion() (core.TrieNodeVersion, error) { func (en *extensionNode) collectLeavesForMigration( migrationArgs vmcommon.ArgsMigrateDataTrieLeaves, + tmc MetricsCollector, db common.TrieStorageInteractor, keyBuilder common.KeyBuilder, ) (bool, error) { @@ -769,13 +782,13 @@ func (en *extensionNode) collectLeavesForMigration( return true, nil } - err = resolveIfCollapsed(en, 0, db) + err = resolveIfCollapsed(en, 0, tmc, db) if err != nil { return false, err } keyBuilder.BuildKey(en.Key) - return en.child.collectLeavesForMigration(migrationArgs, db, keyBuilder.ShallowClone()) + return en.child.collectLeavesForMigration(migrationArgs, tmc, db, keyBuilder.ShallowClone()) } func (en *extensionNode) getNodeData(keyBuilder common.KeyBuilder) ([]common.TrieNodeData, error) { diff --git a/trie/extensionNode_test.go b/trie/extensionNode_test.go index b68d9f14d79..2154577a4b0 100644 --- a/trie/extensionNode_test.go +++ b/trie/extensionNode_test.go @@ -17,6 +17,7 @@ import ( "github.com/multiversx/mx-chain-go/trie/keyBuilder" "github.com/multiversx/mx-chain-go/trie/mock" "github.com/multiversx/mx-chain-go/trie/statistics" + "github.com/multiversx/mx-chain-go/trie/trieMetricsCollector" "github.com/stretchr/testify/assert" ) @@ -31,6 +32,11 @@ func getEnAndCollapsedEn() (*extensionNode, *extensionNode) { return en, collapsedEn } +func markNotDirtyEn(en *extensionNode) { + en.dirty = false + markNotDirtyBranchNode(en.child.(*branchNode)) +} + func TestExtensionNode_newExtensionNode(t *testing.T) { t.Parallel() @@ -243,7 +249,7 @@ func TestExtensionNode_commit(t *testing.T) { hash, _ := encodeNodeAndGetHash(collapsedEn) _ = en.setHash() - err := en.commitDirty(0, 5, db, db) + err := en.commitDirty(db, db, dtmc) assert.Nil(t, err) encNode, _ := db.Get(hash) @@ -259,7 +265,7 @@ func TestExtensionNode_commitEmptyNode(t *testing.T) { en := &extensionNode{} - err := en.commitDirty(0, 5, nil, nil) + err := en.commitDirty(nil, nil, nil) assert.True(t, errors.Is(err, ErrEmptyExtensionNode)) } @@ -268,7 +274,7 @@ func TestExtensionNode_commitNilNode(t *testing.T) { var en *extensionNode - err := en.commitDirty(0, 5, nil, nil) + err := en.commitDirty(nil, nil, nil) assert.True(t, errors.Is(err, ErrNilExtensionNode)) } @@ -281,7 +287,7 @@ func TestExtensionNode_commitCollapsedNode(t *testing.T) { _ = collapsedEn.setHash() collapsedEn.dirty = true - err := collapsedEn.commitDirty(0, 5, db, db) + err := collapsedEn.commitDirty(db, db, dtmc) assert.Nil(t, err) encNode, _ := db.Get(hash) @@ -331,12 +337,15 @@ func TestExtensionNode_resolveCollapsed(t *testing.T) { db := testscommon.NewMemDbMock() en, collapsedEn := getEnAndCollapsedEn() _ = en.setHash() - _ = en.commitDirty(0, 5, db, db) + _ = en.commitDirty(db, db, dtmc) _, resolved := getBnAndCollapsedBn(en.marsh, en.hasher) + expectedSize := resolved.sizeInBytes() - err := collapsedEn.resolveCollapsed(0, db) + tmc := trieMetricsCollector.NewTrieMetricsCollector() + err := collapsedEn.resolveCollapsed(0, tmc, db) assert.Nil(t, err) assert.Equal(t, en.child.getHash(), collapsedEn.child.getHash()) + assert.Equal(t, expectedSize, tmc.GetSizeLoadedInMem()) h1, _ := encodeNodeAndGetHash(resolved) h2, _ := encodeNodeAndGetHash(collapsedEn.child) @@ -348,7 +357,7 @@ func TestExtensionNode_resolveCollapsedEmptyNode(t *testing.T) { en := &extensionNode{} - err := en.resolveCollapsed(0, nil) + err := en.resolveCollapsed(0, nil, nil) assert.True(t, errors.Is(err, ErrEmptyExtensionNode)) } @@ -357,7 +366,7 @@ func TestExtensionNode_resolveCollapsedNilNode(t *testing.T) { var en *extensionNode - err := en.resolveCollapsed(2, nil) + err := en.resolveCollapsed(2, nil, nil) assert.True(t, errors.Is(err, ErrNilExtensionNode)) } @@ -384,10 +393,11 @@ func TestExtensionNode_tryGet(t *testing.T) { key := append(enKey, bnKey...) key = append(key, lnKey...) - val, maxDepth, err := en.tryGet(key, 0, nil) + tmc := trieMetricsCollector.NewTrieMetricsCollector() + val, err := en.tryGet(key, tmc, nil) assert.Equal(t, dogBytes, val) assert.Nil(t, err) - assert.Equal(t, uint32(2), maxDepth) + assert.Equal(t, uint32(2), tmc.GetMaxDepth()) } func TestExtensionNode_tryGetEmptyKey(t *testing.T) { @@ -396,10 +406,11 @@ func TestExtensionNode_tryGetEmptyKey(t *testing.T) { en, _ := getEnAndCollapsedEn() var key []byte - val, maxDepth, err := en.tryGet(key, 0, nil) + tmc := trieMetricsCollector.NewTrieMetricsCollector() + val, err := en.tryGet(key, tmc, nil) assert.Nil(t, err) assert.Nil(t, val) - assert.Equal(t, uint32(0), maxDepth) + assert.Equal(t, uint32(0), tmc.GetMaxDepth()) } func TestExtensionNode_tryGetWrongKey(t *testing.T) { @@ -408,10 +419,11 @@ func TestExtensionNode_tryGetWrongKey(t *testing.T) { en, _ := getEnAndCollapsedEn() key := []byte("gdo") - val, maxDepth, err := en.tryGet(key, 0, nil) + tmc := trieMetricsCollector.NewTrieMetricsCollector() + val, err := en.tryGet(key, tmc, nil) assert.Nil(t, err) assert.Nil(t, val) - assert.Equal(t, uint32(0), maxDepth) + assert.Equal(t, uint32(0), tmc.GetMaxDepth()) } func TestExtensionNode_tryGetCollapsedNode(t *testing.T) { @@ -420,7 +432,7 @@ func TestExtensionNode_tryGetCollapsedNode(t *testing.T) { db := testscommon.NewMemDbMock() en, collapsedEn := getEnAndCollapsedEn() _ = en.setHash() - _ = en.commitDirty(0, 5, db, db) + _ = en.commitDirty(db, db, dtmc) enKey := []byte{100} bnKey := []byte{2} @@ -428,10 +440,14 @@ func TestExtensionNode_tryGetCollapsedNode(t *testing.T) { key := append(enKey, bnKey...) key = append(key, lnKey...) - val, maxDepth, err := collapsedEn.tryGet(key, 0, db) + tmc := trieMetricsCollector.NewTrieMetricsCollector() + val, err := collapsedEn.tryGet(key, tmc, db) assert.Equal(t, []byte("dog"), val) assert.Nil(t, err) - assert.Equal(t, uint32(2), maxDepth) + assert.Equal(t, uint32(2), tmc.GetMaxDepth()) + bn, collapsedBn := getBnAndCollapsedBn(getTestMarshalizerAndHasher()) + expectedSize := collapsedBn.sizeInBytes() + bn.children[2].sizeInBytes() + assert.Equal(t, expectedSize, tmc.GetSizeLoadedInMem()) } func TestExtensionNode_tryGetEmptyNode(t *testing.T) { @@ -440,10 +456,11 @@ func TestExtensionNode_tryGetEmptyNode(t *testing.T) { en := &extensionNode{} key := []byte("dog") - val, maxDepth, err := en.tryGet(key, 0, nil) + tmc := trieMetricsCollector.NewTrieMetricsCollector() + val, err := en.tryGet(key, tmc, nil) assert.True(t, errors.Is(err, ErrEmptyExtensionNode)) assert.Nil(t, val) - assert.Equal(t, uint32(0), maxDepth) + assert.Equal(t, uint32(0), tmc.GetMaxDepth()) } func TestExtensionNode_tryGetNilNode(t *testing.T) { @@ -452,10 +469,11 @@ func TestExtensionNode_tryGetNilNode(t *testing.T) { var en *extensionNode key := []byte("dog") - val, maxDepth, err := en.tryGet(key, 0, nil) + tmc := trieMetricsCollector.NewTrieMetricsCollector() + val, err := en.tryGet(key, tmc, nil) assert.True(t, errors.Is(err, ErrNilExtensionNode)) assert.Nil(t, val) - assert.Equal(t, uint32(0), maxDepth) + assert.Equal(t, uint32(0), tmc.GetMaxDepth()) } func TestExtensionNode_getNext(t *testing.T) { @@ -470,7 +488,7 @@ func TestExtensionNode_getNext(t *testing.T) { key := append(enKey, bnKey...) key = append(key, lnKey...) - n, newKey, err := en.getNext(key, nil) + n, newKey, err := en.getNext(key, dtmc, nil) assert.Equal(t, nextNode, n) assert.Equal(t, key[1:], newKey) assert.Nil(t, err) @@ -484,7 +502,7 @@ func TestExtensionNode_getNextWrongKey(t *testing.T) { lnKey := []byte("dog") key := append(bnKey, lnKey...) - n, key, err := en.getNext(key, nil) + n, key, err := en.getNext(key, nil, nil) assert.Nil(t, n) assert.Nil(t, key) assert.Equal(t, ErrNodeNotFound, err) @@ -495,12 +513,17 @@ func TestExtensionNode_insert(t *testing.T) { en, _ := getEnAndCollapsedEn() key := []byte{100, 15, 5, 6} + newData := getTrieDataWithDefaultVersion(string(key), "dogs") - newNode, _, err := en.insert(getTrieDataWithDefaultVersion(string(key), "dogs"), nil) + tmc := trieMetricsCollector.NewTrieMetricsCollector() + newNode, _, err := en.insert(newData, tmc, nil) assert.NotNil(t, newNode) assert.Nil(t, err) + newData.Key = newData.Key[2:] + newLn, _ := newLeafNode(newData, en.marsh, en.hasher) + assert.Equal(t, newLn.sizeInBytes(), tmc.GetSizeLoadedInMem()) - val, _, _ := newNode.tryGet(key, 0, nil) + val, _ := newNode.tryGet(key, trieMetricsCollector.NewTrieMetricsCollector(), nil) assert.Equal(t, []byte("dogs"), val) } @@ -510,15 +533,21 @@ func TestExtensionNode_insertCollapsedNode(t *testing.T) { db := testscommon.NewMemDbMock() en, collapsedEn := getEnAndCollapsedEn() key := []byte{100, 15, 5, 6} + newData := getTrieDataWithDefaultVersion(string(key), "dogs") _ = en.setHash() - _ = en.commitDirty(0, 5, db, db) + _ = en.commitDirty(db, db, dtmc) - newNode, _, err := collapsedEn.insert(getTrieDataWithDefaultVersion(string(key), "dogs"), db) + tmc := trieMetricsCollector.NewTrieMetricsCollector() + newNode, _, err := collapsedEn.insert(newData, tmc, db) assert.NotNil(t, newNode) assert.Nil(t, err) + newData.Key = newData.Key[2:] + newLn, _ := newLeafNode(newData, en.marsh, en.hasher) + expectedSize := newLn.sizeInBytes() + en.child.sizeInBytes() + assert.Equal(t, expectedSize, tmc.GetSizeLoadedInMem()) - val, _, _ := newNode.tryGet(key, 0, db) + val, _ := newNode.tryGet(key, trieMetricsCollector.NewTrieMetricsCollector(), db) assert.Equal(t, []byte("dogs"), val) } @@ -530,16 +559,22 @@ func TestExtensionNode_insertInStoredEnSameKey(t *testing.T) { enKey := []byte{100} key := append(enKey, []byte{11, 12}...) - _ = en.commitDirty(0, 5, db, db) + _ = en.setHash() + markNotDirtyEn(en) enHash := en.getHash() - bn, _, _ := en.getNext(enKey, db) + bn, _, _ := en.getNext(enKey, dtmc, db) bnHash := bn.getHash() expectedHashes := [][]byte{bnHash, enHash} + newData := getTrieDataWithDefaultVersion(string(key), "dogs") - newNode, oldHashes, err := en.insert(getTrieDataWithDefaultVersion(string(key), "dogs"), db) + tmc := trieMetricsCollector.NewTrieMetricsCollector() + newNode, oldHashes, err := en.insert(newData, tmc, db) assert.NotNil(t, newNode) assert.Nil(t, err) assert.Equal(t, expectedHashes, oldHashes) + newData.Key = newData.Key[2:] + newLn, _ := newLeafNode(newData, en.marsh, en.hasher) + assert.Equal(t, newLn.sizeInBytes(), tmc.GetSizeLoadedInMem()) } func TestExtensionNode_insertInStoredEnDifferentKey(t *testing.T) { @@ -551,13 +586,20 @@ func TestExtensionNode_insertInStoredEnDifferentKey(t *testing.T) { en, _ := newExtensionNode(enKey, bn, bn.marsh, bn.hasher) nodeKey := []byte{11, 12} - _ = en.commitDirty(0, 5, db, db) + _ = en.setHash() + markNotDirtyEn(en) expectedHashes := [][]byte{en.getHash()} + originalSize := en.sizeInBytes() - newNode, oldHashes, err := en.insert(getTrieDataWithDefaultVersion(string(nodeKey), "dogs"), db) + newData := getTrieDataWithDefaultVersion(string(nodeKey), "dogs") + tmc := trieMetricsCollector.NewTrieMetricsCollector() + newNode, oldHashes, err := en.insert(newData, tmc, db) assert.NotNil(t, newNode) assert.Nil(t, err) assert.Equal(t, expectedHashes, oldHashes) + + expectedSize := newNode.sizeInBytes() + newNode.(*branchNode).children[nodeKey[0]].sizeInBytes() - originalSize + assert.Equal(t, expectedSize, tmc.GetSizeLoadedInMem()) } func TestExtensionNode_insertInDirtyEnSameKey(t *testing.T) { @@ -566,10 +608,15 @@ func TestExtensionNode_insertInDirtyEnSameKey(t *testing.T) { en, _ := getEnAndCollapsedEn() nodeKey := []byte{100, 11, 12} - newNode, oldHashes, err := en.insert(getTrieDataWithDefaultVersion(string(nodeKey), "dogs"), nil) + newData := getTrieDataWithDefaultVersion(string(nodeKey), "dogs") + tmc := trieMetricsCollector.NewTrieMetricsCollector() + newNode, oldHashes, err := en.insert(newData, tmc, nil) assert.NotNil(t, newNode) assert.Nil(t, err) assert.Equal(t, [][]byte{}, oldHashes) + newLn, _ := newLeafNode(newData, en.marsh, en.hasher) + expectedSize := newLn.sizeInBytes() - 2 // 2 because of the parts of the key that is added to the parents + assert.Equal(t, expectedSize, tmc.GetSizeLoadedInMem()) } func TestExtensionNode_insertInDirtyEnDifferentKey(t *testing.T) { @@ -580,10 +627,15 @@ func TestExtensionNode_insertInDirtyEnDifferentKey(t *testing.T) { en, _ := newExtensionNode(enKey, bn, bn.marsh, bn.hasher) nodeKey := []byte{11, 12} - newNode, oldHashes, err := en.insert(getTrieDataWithDefaultVersion(string(nodeKey), "dogs"), nil) + newData := getTrieDataWithDefaultVersion(string(nodeKey), "dogs") + tmc := trieMetricsCollector.NewTrieMetricsCollector() + newNode, oldHashes, err := en.insert(newData, tmc, nil) assert.NotNil(t, newNode) assert.Nil(t, err) assert.Equal(t, [][]byte{}, oldHashes) + + expectedSize := newNode.sizeInBytes() + newNode.(*branchNode).children[nodeKey[0]].sizeInBytes() - en.sizeInBytes() + assert.Equal(t, expectedSize, tmc.GetSizeLoadedInMem()) } func TestExtensionNode_insertInNilNode(t *testing.T) { @@ -591,7 +643,7 @@ func TestExtensionNode_insertInNilNode(t *testing.T) { var en *extensionNode - newNode, _, err := en.insert(getTrieDataWithDefaultVersion("key", "val"), nil) + newNode, _, err := en.insert(getTrieDataWithDefaultVersion("key", "val"), nil, nil) assert.Nil(t, newNode) assert.True(t, errors.Is(err, ErrNilExtensionNode)) assert.Nil(t, newNode) @@ -609,13 +661,18 @@ func TestExtensionNode_delete(t *testing.T) { key := append(enKey, bnKey...) key = append(key, lnKey...) - val, _, _ := en.tryGet(key, 0, nil) + val, _ := en.tryGet(key, trieMetricsCollector.NewTrieMetricsCollector(), nil) assert.Equal(t, dogBytes, val) - dirty, _, _, err := en.delete(key, nil) + tmc := trieMetricsCollector.NewTrieMetricsCollector() + dirty, _, _, err := en.delete(key, tmc, nil) assert.True(t, dirty) assert.Nil(t, err) - val, _, _ = en.tryGet(key, 0, nil) + + deletedLeaf, _ := newLeafNode(getTrieDataWithDefaultVersion("dog", "dog"), en.marsh, en.hasher) + assert.Equal(t, -(deletedLeaf.sizeInBytes() + hashSizeInBytes), tmc.GetSizeLoadedInMem()) + + val, _ = en.tryGet(key, trieMetricsCollector.NewTrieMetricsCollector(), nil) assert.Nil(t, val) } @@ -631,12 +688,12 @@ func TestExtensionNode_deleteFromStoredEn(t *testing.T) { key = append(key, lnKey...) lnPathKey := key - _ = en.commitDirty(0, 5, db, db) - bn, key, _ := en.getNext(key, db) - ln, _, _ := bn.getNext(key, db) + _ = en.commitDirty(db, db, dtmc) + bn, key, _ := en.getNext(key, dtmc, db) + ln, _, _ := bn.getNext(key, dtmc, db) expectedHashes := [][]byte{ln.getHash(), bn.getHash(), en.getHash()} - dirty, _, oldHashes, err := en.delete(lnPathKey, db) + dirty, _, oldHashes, err := en.delete(lnPathKey, dtmc, db) assert.True(t, dirty) assert.Nil(t, err) assert.Equal(t, expectedHashes, oldHashes) @@ -648,7 +705,7 @@ func TestExtensionNode_deleteFromDirtyEn(t *testing.T) { en, _ := getEnAndCollapsedEn() lnKey := []byte{100, 2, 100, 111, 103} - dirty, _, oldHashes, err := en.delete(lnKey, nil) + dirty, _, oldHashes, err := en.delete(lnKey, dtmc, nil) assert.True(t, dirty) assert.Nil(t, err) assert.Equal(t, [][]byte{}, oldHashes) @@ -659,7 +716,7 @@ func TestExtendedNode_deleteEmptyNode(t *testing.T) { en := &extensionNode{} - dirty, newNode, _, err := en.delete([]byte("dog"), nil) + dirty, newNode, _, err := en.delete([]byte("dog"), nil, nil) assert.False(t, dirty) assert.True(t, errors.Is(err, ErrEmptyExtensionNode)) assert.Nil(t, newNode) @@ -670,7 +727,7 @@ func TestExtensionNode_deleteNilNode(t *testing.T) { var en *extensionNode - dirty, newNode, _, err := en.delete([]byte("dog"), nil) + dirty, newNode, _, err := en.delete([]byte("dog"), nil, nil) assert.False(t, dirty) assert.True(t, errors.Is(err, ErrNilExtensionNode)) assert.Nil(t, newNode) @@ -681,7 +738,7 @@ func TestExtensionNode_deleteEmptykey(t *testing.T) { en, _ := getEnAndCollapsedEn() - dirty, newNode, _, err := en.delete([]byte{}, nil) + dirty, newNode, _, err := en.delete([]byte{}, nil, nil) assert.False(t, dirty) assert.Equal(t, ErrValueTooShort, err) assert.Nil(t, newNode) @@ -693,7 +750,7 @@ func TestExtensionNode_deleteCollapsedNode(t *testing.T) { db := testscommon.NewMemDbMock() en, collapsedEn := getEnAndCollapsedEn() _ = en.setHash() - _ = en.commitDirty(0, 5, db, db) + _ = en.commitDirty(db, db, dtmc) enKey := []byte{100} bnKey := []byte{2} @@ -701,13 +758,16 @@ func TestExtensionNode_deleteCollapsedNode(t *testing.T) { key := append(enKey, bnKey...) key = append(key, lnKey...) - val, _, _ := en.tryGet(key, 0, db) + val, _ := en.tryGet(key, trieMetricsCollector.NewTrieMetricsCollector(), db) assert.Equal(t, []byte("dog"), val) - dirty, newNode, _, err := collapsedEn.delete(key, db) + tmc := trieMetricsCollector.NewTrieMetricsCollector() + dirty, newNode, _, err := collapsedEn.delete(key, tmc, db) assert.True(t, dirty) assert.Nil(t, err) - val, _, _ = newNode.tryGet(key, 0, db) + assert.Equal(t, collapsedEn.child.sizeInBytes(), tmc.GetSizeLoadedInMem()) + + val, _ = newNode.tryGet(key, trieMetricsCollector.NewTrieMetricsCollector(), db) assert.Nil(t, val) } @@ -723,7 +783,7 @@ func TestExtensionNode_reduceNode(t *testing.T) { expected.hasher = en.hasher expected.child = en.child - n, newChildPos, err := en.reduceNode(2) + n, newChildPos, err := en.reduceNode(2, dtmc) assert.Equal(t, expected, n) assert.Nil(t, err) assert.True(t, newChildPos) @@ -768,7 +828,7 @@ func TestExtensionNode_getChildren(t *testing.T) { en, _ := getEnAndCollapsedEn() - children, err := en.getChildren(nil) + children, err := en.getChildren(dtmc, nil) assert.Nil(t, err) assert.Equal(t, 1, len(children)) } @@ -778,9 +838,9 @@ func TestExtensionNode_getChildrenCollapsedEn(t *testing.T) { db := testscommon.NewMemDbMock() en, collapsedEn := getEnAndCollapsedEn() - _ = en.commitDirty(0, 5, db, db) + _ = en.commitDirty(db, db, dtmc) - children, err := collapsedEn.getChildren(db) + children, err := collapsedEn.getChildren(dtmc, db) assert.Nil(t, err) assert.Equal(t, 1, len(children)) } @@ -886,20 +946,6 @@ func TestExtensionNode_getMarshalizer(t *testing.T) { assert.Equal(t, marsh, en.getMarshalizer()) } -func TestExtensionNode_commitCollapsesTrieIfMaxTrieLevelInMemoryIsReached(t *testing.T) { - t.Parallel() - - en, collapsedEn := getEnAndCollapsedEn() - _ = collapsedEn.setRootHash() - - err := en.commitDirty(0, 1, testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) - assert.Nil(t, err) - - assert.Equal(t, collapsedEn.EncodedChild, en.EncodedChild) - assert.Equal(t, collapsedEn.child, en.child) - assert.Equal(t, collapsedEn.hash, en.hash) -} - func TestExtensionNode_printShouldNotPanicEvenIfNodeIsCollapsed(t *testing.T) { t.Parallel() @@ -908,11 +954,11 @@ func TestExtensionNode_printShouldNotPanicEvenIfNodeIsCollapsed(t *testing.T) { db := testscommon.NewMemDbMock() en, collapsedEn := getEnAndCollapsedEn() - _ = en.commitDirty(0, 5, db, db) + _ = en.commitDirty(db, db, dtmc) _ = collapsedEn.setHash() - en.print(enWriter, 0, db) - collapsedEn.print(collapsedEnWriter, 0, db) + en.print(enWriter, 0, dtmc, db) + collapsedEn.print(collapsedEnWriter, 0, dtmc, db) assert.Equal(t, enWriter.Bytes(), collapsedEnWriter.Bytes()) } @@ -922,7 +968,7 @@ func TestExtensionNode_getDirtyHashesFromCleanNode(t *testing.T) { db := testscommon.NewMemDbMock() en, _ := getEnAndCollapsedEn() - _ = en.commitDirty(0, 5, db, db) + _ = en.commitDirty(db, db, dtmc) dirtyHashes := make(common.ModifiedHashes) err := en.getDirtyHashes(dirtyHashes) @@ -936,7 +982,7 @@ func TestExtensionNode_getAllHashes(t *testing.T) { trieNodes := 5 en, _ := getEnAndCollapsedEn() - hashes, err := en.getAllHashes(testscommon.NewMemDbMock()) + hashes, err := en.getAllHashes(dtmc, testscommon.NewMemDbMock()) assert.Nil(t, err) assert.Equal(t, trieNodes, len(hashes)) } @@ -947,9 +993,9 @@ func TestExtensionNode_getAllHashesResolvesCollapsed(t *testing.T) { trieNodes := 5 db := testscommon.NewMemDbMock() en, collapsedEn := getEnAndCollapsedEn() - _ = en.commitDirty(0, 5, db, db) + _ = en.commitDirty(db, db, dtmc) - hashes, err := collapsedEn.getAllHashes(db) + hashes, err := collapsedEn.getAllHashes(dtmc, db) assert.Nil(t, err) assert.Equal(t, trieNodes, len(hashes)) } @@ -995,7 +1041,7 @@ func TestExtensionNode_SizeInBytes(t *testing.T) { collapsed := []byte("collapsed") key := []byte("key") - hash := []byte("hash") + hash := bytes.Repeat([]byte{1}, 32) en = &extensionNode{ CollapsedEn: CollapsedEn{ Key: key, @@ -1009,7 +1055,7 @@ func TestExtensionNode_SizeInBytes(t *testing.T) { hasher: nil, }, } - assert.Equal(t, len(collapsed)+len(key)+len(hash)+1+3*pointerSizeInBytes, en.sizeInBytes()) + assert.Equal(t, hashSizeInBytes+len(key)+nodeVersionSizeInBytes+len(hash)+1+3*pointerSizeInBytes, en.sizeInBytes()) } func TestExtensionNode_commitContextDone(t *testing.T) { @@ -1020,7 +1066,7 @@ func TestExtensionNode_commitContextDone(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() - err := en.commitSnapshot(db, 0, nil, nil, ctx, statistics.NewTrieStatistics(), &testscommon.ProcessStatusHandlerStub{}, []byte{}, 0) + err := en.commitSnapshot(db, 0, nil, nil, ctx, statistics.NewTrieStatistics(), &testscommon.ProcessStatusHandlerStub{}, []byte{}, dtmc) assert.Equal(t, core.ErrContextClosing, err) } @@ -1041,7 +1087,7 @@ func TestExtensionNode_commitSnapshotDbIsClosing(t *testing.T) { _, collapsedEn := getEnAndCollapsedEn() missingNodesChan := make(chan []byte, 10) - err := collapsedEn.commitSnapshot(db, 0, nil, missingNodesChan, context.Background(), statistics.NewTrieStatistics(), &testscommon.ProcessStatusHandlerStub{}, []byte{}, 0) + err := collapsedEn.commitSnapshot(db, 0, nil, missingNodesChan, context.Background(), statistics.NewTrieStatistics(), &testscommon.ProcessStatusHandlerStub{}, []byte{}, dtmc) assert.True(t, core.IsClosingError(err)) assert.Equal(t, 0, len(missingNodesChan)) } diff --git a/trie/factory/trieCreator.go b/trie/factory/trieCreator.go index 198b33a0455..621fbfb0e8b 100644 --- a/trie/factory/trieCreator.go +++ b/trie/factory/trieCreator.go @@ -7,7 +7,7 @@ import ( "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/config" "github.com/multiversx/mx-chain-go/dataRetriever" - "github.com/multiversx/mx-chain-go/state" + "github.com/multiversx/mx-chain-go/state/triesHolder" "github.com/multiversx/mx-chain-go/storage" "github.com/multiversx/mx-chain-go/trie" ) @@ -17,11 +17,11 @@ type TrieCreateArgs struct { MainStorer storage.Storer PruningEnabled bool SnapshotsEnabled bool - MaxTrieLevelInMem uint IdleProvider trie.IdleNodeProvider Identifier string EnableEpochsHandler common.EnableEpochsHandler StatsCollector common.StateStatisticsHandler + MaxSizeInMemory uint64 } type trieCreator struct { @@ -78,7 +78,7 @@ func (tc *trieCreator) Create(args TrieCreateArgs) (common.StorageManager, commo return nil, nil, err } - newTrie, err := trie.NewTrie(trieStorage, tc.marshalizer, tc.hasher, args.EnableEpochsHandler, args.MaxTrieLevelInMem) + newTrie, err := trie.NewTrie(trieStorage, tc.marshalizer, tc.hasher, args.EnableEpochsHandler, args.MaxSizeInMemory) if err != nil { return nil, nil, err } @@ -117,19 +117,19 @@ func CreateTriesComponentsForShardId( args := TrieCreateArgs{ MainStorer: mainStorer, PruningEnabled: generalConfig.StateTriesConfig.AccountsStatePruningEnabled, - MaxTrieLevelInMem: generalConfig.StateTriesConfig.MaxStateTrieLevelInMemory, SnapshotsEnabled: generalConfig.StateTriesConfig.SnapshotsEnabled, IdleProvider: coreComponentsHolder.ProcessStatusHandler(), Identifier: dataRetriever.UserAccountsUnit.String(), EnableEpochsHandler: coreComponentsHolder.EnableEpochsHandler(), StatsCollector: stateStatsHandler, + MaxSizeInMemory: generalConfig.StateTriesConfig.MaxUserTrieSizeInMemory, } userStorageManager, userAccountTrie, err := trFactory.Create(args) if err != nil { return nil, nil, err } - trieContainer := state.NewDataTriesHolder() + trieContainer := triesHolder.NewTriesHolder() trieStorageManagers := make(map[string]common.StorageManager) trieContainer.Put([]byte(dataRetriever.UserAccountsUnit.String()), userAccountTrie) @@ -143,12 +143,12 @@ func CreateTriesComponentsForShardId( args = TrieCreateArgs{ MainStorer: mainStorer, PruningEnabled: generalConfig.StateTriesConfig.PeerStatePruningEnabled, - MaxTrieLevelInMem: generalConfig.StateTriesConfig.MaxPeerTrieLevelInMemory, SnapshotsEnabled: generalConfig.StateTriesConfig.SnapshotsEnabled, IdleProvider: coreComponentsHolder.ProcessStatusHandler(), Identifier: dataRetriever.PeerAccountsUnit.String(), EnableEpochsHandler: coreComponentsHolder.EnableEpochsHandler(), StatsCollector: stateStatsHandler, + MaxSizeInMemory: generalConfig.StateTriesConfig.MaxPeerTrieSizeInMemory, } peerStorageManager, peerAccountsTrie, err := trFactory.Create(args) if err != nil { diff --git a/trie/factory/trieCreator_test.go b/trie/factory/trieCreator_test.go index c4a716e2cc4..e67b715be02 100644 --- a/trie/factory/trieCreator_test.go +++ b/trie/factory/trieCreator_test.go @@ -6,6 +6,7 @@ import ( "testing" "github.com/multiversx/mx-chain-core-go/core/check" + "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/common/statistics/disabled" "github.com/multiversx/mx-chain-go/config" "github.com/multiversx/mx-chain-go/dataRetriever" @@ -36,11 +37,11 @@ func getCreateArgs() factory.TrieCreateArgs { MainStorer: testscommon.CreateMemUnit(), PruningEnabled: false, SnapshotsEnabled: true, - MaxTrieLevelInMem: 5, IdleProvider: &testscommon.ProcessStatusHandlerStub{}, Identifier: dataRetriever.UserAccountsUnit.String(), EnableEpochsHandler: &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, StatsCollector: disabled.NewStateStatistics(), + MaxSizeInMemory: common.TenMbSize, } } @@ -140,20 +141,6 @@ func TestTrieCreator_CreateWithNilMainStorerShouldErr(t *testing.T) { require.True(t, strings.Contains(err.Error(), trie.ErrNilStorer.Error())) } -func TestTrieCreator_CreateWithInvalidMaxTrieLevelInMemShouldErr(t *testing.T) { - t.Parallel() - - args := getArgs() - tf, _ := factory.NewTrieFactory(args) - - createArgs := getCreateArgs() - createArgs.MaxTrieLevelInMem = 0 - _, tr, err := tf.Create(createArgs) - require.Nil(t, tr) - require.NotNil(t, err) - require.Contains(t, err.Error(), trie.ErrInvalidLevelValue.Error()) -} - func TestTrieCreator_CreateTriesComponentsForShardId(t *testing.T) { t.Parallel() diff --git a/trie/interface.go b/trie/interface.go index af1eb4c1eef..9358e3a5df7 100644 --- a/trie/interface.go +++ b/trie/interface.go @@ -24,38 +24,38 @@ type node interface { isPosCollapsed(pos int) bool isDirty() bool getEncodedNode() ([]byte, error) - resolveCollapsed(pos byte, db common.TrieStorageInteractor) error + resolveCollapsed(pos byte, tmc MetricsCollector, db common.TrieStorageInteractor) error hashNode() ([]byte, error) hashChildren() error - tryGet(key []byte, depth uint32, db common.TrieStorageInteractor) ([]byte, uint32, error) - getNext(key []byte, db common.TrieStorageInteractor) (node, []byte, error) - insert(newData core.TrieData, db common.TrieStorageInteractor) (node, [][]byte, error) - delete(key []byte, db common.TrieStorageInteractor) (bool, node, [][]byte, error) - reduceNode(pos int) (node, bool, error) + tryGet(key []byte, tmc MetricsCollector, db common.TrieStorageInteractor) ([]byte, error) + getNext(key []byte, tmc MetricsCollector, db common.TrieStorageInteractor) (node, []byte, error) + insert(newData core.TrieData, tmc MetricsCollector, db common.TrieStorageInteractor) (node, [][]byte, error) + delete(key []byte, tmc MetricsCollector, db common.TrieStorageInteractor) (bool, node, [][]byte, error) + reduceNode(pos int, tmc MetricsCollector) (node, bool, error) isEmptyOrNil() error - print(writer io.Writer, index int, db common.TrieStorageInteractor) + print(writer io.Writer, index int, tmc MetricsCollector, db common.TrieStorageInteractor) getDirtyHashes(common.ModifiedHashes) error - getChildren(db common.TrieStorageInteractor) ([]node, error) + getChildren(tmc MetricsCollector, db common.TrieStorageInteractor) ([]node, error) isValid() bool getNodeData(common.KeyBuilder) ([]common.TrieNodeData, error) setDirty(bool) loadChildren(func([]byte) (node, error)) ([][]byte, []node, error) - getAllLeavesOnChannel(chan core.KeyValueHolder, common.KeyBuilder, common.TrieLeafParser, common.TrieStorageInteractor, marshal.Marshalizer, chan struct{}, context.Context) error - getAllHashes(db common.TrieStorageInteractor) ([][]byte, error) + getAllLeavesOnChannel(chan core.KeyValueHolder, common.KeyBuilder, common.TrieLeafParser, common.TrieStorageInteractor, marshal.Marshalizer, chan struct{}, context.Context, MetricsCollector) error + getAllHashes(tmc MetricsCollector, db common.TrieStorageInteractor) ([][]byte, error) getNextHashAndKey([]byte) (bool, []byte, []byte) getValue() []byte getVersion() (core.TrieNodeVersion, error) - collectLeavesForMigration(migrationArgs vmcommon.ArgsMigrateDataTrieLeaves, db common.TrieStorageInteractor, keyBuilder common.KeyBuilder) (bool, error) + collectLeavesForMigration(migrationArgs vmcommon.ArgsMigrateDataTrieLeaves, tmc MetricsCollector, db common.TrieStorageInteractor, keyBuilder common.KeyBuilder) (bool, error) - commitDirty(level byte, maxTrieLevelInMemory uint, originDb common.TrieStorageInteractor, targetDb common.BaseStorer) error - commitSnapshot(originDb snapshotDb, maxEpochToSearchFrom uint32, leavesChan chan core.KeyValueHolder, missingNodesChan chan []byte, ctx context.Context, stats common.TrieStatisticsHandler, idleProvider IdleNodeProvider, encodedRoot []byte, depthLevel int) error + commitDirty(originDb common.TrieStorageInteractor, targetDb common.BaseStorer, tmc MetricsCollector) error + commitSnapshot(originDb snapshotDb, maxEpochToSearchFrom uint32, leavesChan chan core.KeyValueHolder, missingNodesChan chan []byte, ctx context.Context, stats common.TrieStatisticsHandler, idleProvider IdleNodeProvider, encodedRoot []byte, tmc MetricsCollector) error getMarshalizer() marshal.Marshalizer setMarshalizer(marshal.Marshalizer) getHasher() hashing.Hasher setHasher(hashing.Hasher) sizeInBytes() int - collectStats(handler common.TrieStatisticsHandler, depthLevel int, db common.TrieStorageInteractor) error + collectStats(handler common.TrieStatisticsHandler, tmc MetricsCollector, db common.TrieStorageInteractor) error IsInterfaceNil() bool } @@ -65,7 +65,7 @@ type dbWithGetFromEpoch interface { } type snapshotNode interface { - commitSnapshot(originDb snapshotDb, maxEpochToSearchFrom uint32, leavesChan chan core.KeyValueHolder, missingNodesChan chan []byte, ctx context.Context, stats common.TrieStatisticsHandler, idleProvider IdleNodeProvider, encodedRoot []byte, depthLevel int) error + commitSnapshot(originDb snapshotDb, maxEpochToSearchFrom uint32, leavesChan chan core.KeyValueHolder, missingNodesChan chan []byte, ctx context.Context, stats common.TrieStatisticsHandler, idleProvider IdleNodeProvider, encodedRoot []byte, tmc MetricsCollector) error } // RequestHandler defines the methods through which request to data can be made @@ -118,3 +118,12 @@ type snapshotDb interface { PutInEpochWithoutCache(key []byte, data []byte) error GetIdentifier() string } + +// MetricsCollector is used to collect metrics about the trie +type MetricsCollector interface { + SetDepth(depth uint32) + GetCurrentDepth() uint32 + GetMaxDepth() uint32 + AddSizeLoadedInMem(size int) + GetSizeLoadedInMem() int +} diff --git a/trie/leafNode.go b/trie/leafNode.go index 4f82059518d..07467745ce8 100644 --- a/trie/leafNode.go +++ b/trie/leafNode.go @@ -119,7 +119,7 @@ func (ln *leafNode) hashNode() ([]byte, error) { return encodeNodeAndGetHash(ln) } -func (ln *leafNode) commitDirty(_ byte, _ uint, _ common.TrieStorageInteractor, targetDb common.BaseStorer) error { +func (ln *leafNode) commitDirty(_ common.TrieStorageInteractor, targetDb common.BaseStorer, _ MetricsCollector) error { err := ln.isEmptyOrNil() if err != nil { return fmt.Errorf("commit error %w", err) @@ -144,7 +144,7 @@ func (ln *leafNode) commitSnapshot( stats common.TrieStatisticsHandler, idleProvider IdleNodeProvider, nodeBytes []byte, - depthLevel int, + tmc MetricsCollector, ) error { if shouldStopIfContextDoneBlockingIfBusy(ctx, idleProvider) { return core.ErrContextClosing @@ -160,7 +160,7 @@ func (ln *leafNode) commitSnapshot( return err } - stats.AddLeafNode(depthLevel, uint64(len(nodeBytes)), version) + stats.AddLeafNode(int(tmc.GetCurrentDepth()), uint64(len(nodeBytes)), version) return nil } @@ -194,7 +194,7 @@ func (ln *leafNode) getEncodedNode() ([]byte, error) { return marshaledNode, nil } -func (ln *leafNode) resolveCollapsed(_ byte, _ common.TrieStorageInteractor) error { +func (ln *leafNode) resolveCollapsed(_ byte, _ MetricsCollector, _ common.TrieStorageInteractor) error { return nil } @@ -206,19 +206,19 @@ func (ln *leafNode) isPosCollapsed(_ int) bool { return false } -func (ln *leafNode) tryGet(key []byte, currentDepth uint32, _ common.TrieStorageInteractor) (value []byte, maxDepth uint32, err error) { +func (ln *leafNode) tryGet(key []byte, _ MetricsCollector, _ common.TrieStorageInteractor) (value []byte, err error) { err = ln.isEmptyOrNil() if err != nil { - return nil, currentDepth, fmt.Errorf("tryGet error %w", err) + return nil, fmt.Errorf("tryGet error %w", err) } if bytes.Equal(key, ln.Key) { - return ln.Value, currentDepth, nil + return ln.Value, nil } - return nil, currentDepth, nil + return nil, nil } -func (ln *leafNode) getNext(key []byte, _ common.TrieStorageInteractor) (node, []byte, error) { +func (ln *leafNode) getNext(key []byte, _ MetricsCollector, _ common.TrieStorageInteractor) (node, []byte, error) { err := ln.isEmptyOrNil() if err != nil { return nil, nil, fmt.Errorf("getNext error %w", err) @@ -228,7 +228,7 @@ func (ln *leafNode) getNext(key []byte, _ common.TrieStorageInteractor) (node, [ } return nil, nil, ErrNodeNotFound } -func (ln *leafNode) insert(newData core.TrieData, _ common.TrieStorageInteractor) (node, [][]byte, error) { +func (ln *leafNode) insert(newData core.TrieData, tmc MetricsCollector, _ common.TrieStorageInteractor) (node, [][]byte, error) { err := ln.isEmptyOrNil() if err != nil { return nil, [][]byte{}, fmt.Errorf("insert error %w", err) @@ -242,14 +242,15 @@ func (ln *leafNode) insert(newData core.TrieData, _ common.TrieStorageInteractor nodeKey := ln.Key if bytes.Equal(newData.Key, nodeKey) { - return ln.insertInSameLn(newData, oldHash) + return ln.insertInSameLn(newData, oldHash, tmc) } keyMatchLen := prefixLen(newData.Key, nodeKey) - bn, err := ln.insertInNewBn(newData, keyMatchLen) + bn, err := ln.insertInNewBn(newData, keyMatchLen, tmc) if err != nil { return nil, [][]byte{}, err } + tmc.AddSizeLoadedInMem(bn.sizeInBytes()) if keyMatchLen == 0 { return bn, oldHash, nil @@ -259,15 +260,19 @@ func (ln *leafNode) insert(newData core.TrieData, _ common.TrieStorageInteractor if err != nil { return nil, [][]byte{}, err } + tmc.AddSizeLoadedInMem(newEn.sizeInBytes()) return newEn, oldHash, nil } -func (ln *leafNode) insertInSameLn(newData core.TrieData, oldHashes [][]byte) (node, [][]byte, error) { +func (ln *leafNode) insertInSameLn(newData core.TrieData, oldHashes [][]byte, tmc MetricsCollector) (node, [][]byte, error) { if bytes.Equal(ln.Value, newData.Value) { return nil, [][]byte{}, nil } + sizeDiff := len(newData.Value) - len(ln.Value) + tmc.AddSizeLoadedInMem(sizeDiff) + ln.Value = newData.Value ln.Version = uint32(newData.Version) ln.dirty = true @@ -275,12 +280,13 @@ func (ln *leafNode) insertInSameLn(newData core.TrieData, oldHashes [][]byte) (n return ln, oldHashes, nil } -func (ln *leafNode) insertInNewBn(newData core.TrieData, keyMatchLen int) (node, error) { +func (ln *leafNode) insertInNewBn(newData core.TrieData, keyMatchLen int, tmc MetricsCollector) (node, error) { bn, err := newBranchNode(ln.marsh, ln.hasher) if err != nil { return nil, err } + originalSize := ln.sizeInBytes() oldChildPos := ln.Key[keyMatchLen] newChildPos := newData.Key[keyMatchLen] if childPosOutOfRange(oldChildPos) || childPosOutOfRange(newChildPos) { @@ -303,6 +309,7 @@ func (ln *leafNode) insertInNewBn(newData core.TrieData, keyMatchLen int) (node, } bn.children[oldChildPos] = newLnOldChildPos bn.setVersionForChild(oldLnVersion, oldChildPos) + newSize := newLnOldChildPos.sizeInBytes() newData.Key = newData.Key[keyMatchLen+1:] newLnNewChildPos, err := newLeafNode(newData, ln.marsh, ln.hasher) @@ -311,24 +318,28 @@ func (ln *leafNode) insertInNewBn(newData core.TrieData, keyMatchLen int) (node, } bn.children[newChildPos] = newLnNewChildPos bn.setVersionForChild(newData.Version, newChildPos) + newSize += newLnNewChildPos.sizeInBytes() + tmc.AddSizeLoadedInMem(newSize - originalSize) return bn, nil } -func (ln *leafNode) delete(key []byte, _ common.TrieStorageInteractor) (bool, node, [][]byte, error) { +func (ln *leafNode) delete(key []byte, tmc MetricsCollector, _ common.TrieStorageInteractor) (bool, node, [][]byte, error) { if bytes.Equal(key, ln.Key) { oldHash := make([][]byte, 0) if !ln.dirty { oldHash = append(oldHash, ln.hash) } + tmc.AddSizeLoadedInMem(-ln.sizeInBytes()) return true, nil, oldHash, nil } return false, ln, [][]byte{}, nil } -func (ln *leafNode) reduceNode(pos int) (node, bool, error) { - k := append([]byte{byte(pos)}, ln.Key...) +func (ln *leafNode) reduceNode(pos int, tmc MetricsCollector) (node, bool, error) { + extraKey := []byte{byte(pos)} + k := append(extraKey, ln.Key...) oldLnVersion, err := ln.getVersion() if err != nil { @@ -345,6 +356,7 @@ func (ln *leafNode) reduceNode(pos int) (node, bool, error) { if err != nil { return nil, false, err } + tmc.AddSizeLoadedInMem(len(extraKey)) return newLn, true, nil } @@ -359,7 +371,7 @@ func (ln *leafNode) isEmptyOrNil() error { return nil } -func (ln *leafNode) print(writer io.Writer, _ int, _ common.TrieStorageInteractor) { +func (ln *leafNode) print(writer io.Writer, _ int, _ MetricsCollector, _ common.TrieStorageInteractor) { if ln == nil { return } @@ -391,7 +403,7 @@ func (ln *leafNode) getDirtyHashes(hashes common.ModifiedHashes) error { return nil } -func (ln *leafNode) getChildren(_ common.TrieStorageInteractor) ([]node, error) { +func (ln *leafNode) getChildren(_ MetricsCollector, _ common.TrieStorageInteractor) ([]node, error) { return nil, nil } @@ -415,6 +427,7 @@ func (ln *leafNode) getAllLeavesOnChannel( _ marshal.Marshalizer, chanClose chan struct{}, ctx context.Context, + _ MetricsCollector, ) error { err := ln.isEmptyOrNil() if err != nil { @@ -451,7 +464,7 @@ func (ln *leafNode) getAllLeavesOnChannel( } } -func (ln *leafNode) getAllHashes(_ common.TrieStorageInteractor) ([][]byte, error) { +func (ln *leafNode) getAllHashes(_ MetricsCollector, _ common.TrieStorageInteractor) ([][]byte, error) { err := ln.isEmptyOrNil() if err != nil { return nil, fmt.Errorf("getAllHashes error: %w", err) @@ -477,8 +490,7 @@ func (ln *leafNode) sizeInBytes() int { return 0 } - // hasher + marshalizer + dirty flag = numNodeInnerPointers * pointerSizeInBytes + 1 - nodeSize := len(ln.hash) + len(ln.Key) + len(ln.Value) + numNodeInnerPointers*pointerSizeInBytes + 1 + nodeSize := baseNodeSizeInBytes + len(ln.Key) + len(ln.Value) + nodeVersionSizeInBytes return nodeSize } @@ -487,7 +499,7 @@ func (ln *leafNode) getValue() []byte { return ln.Value } -func (ln *leafNode) collectStats(ts common.TrieStatisticsHandler, depthLevel int, _ common.TrieStorageInteractor) error { +func (ln *leafNode) collectStats(ts common.TrieStatisticsHandler, tmc MetricsCollector, _ common.TrieStorageInteractor) error { err := ln.isEmptyOrNil() if err != nil { return fmt.Errorf("collectStats error %w", err) @@ -503,7 +515,7 @@ func (ln *leafNode) collectStats(ts common.TrieStatisticsHandler, depthLevel int return err } - ts.AddLeafNode(depthLevel, uint64(len(val)), version) + ts.AddLeafNode(int(tmc.GetCurrentDepth()), uint64(len(val)), version) return nil } @@ -518,6 +530,7 @@ func (ln *leafNode) getVersion() (core.TrieNodeVersion, error) { func (ln *leafNode) collectLeavesForMigration( migrationArgs vmcommon.ArgsMigrateDataTrieLeaves, + _ MetricsCollector, _ common.TrieStorageInteractor, keyBuilder common.KeyBuilder, ) (bool, error) { diff --git a/trie/leafNode_test.go b/trie/leafNode_test.go index bbcdbbf9fce..a07422fa366 100644 --- a/trie/leafNode_test.go +++ b/trie/leafNode_test.go @@ -1,6 +1,7 @@ package trie import ( + "bytes" "context" "errors" "math" @@ -16,6 +17,7 @@ import ( "github.com/multiversx/mx-chain-go/trie/keyBuilder" "github.com/multiversx/mx-chain-go/trie/mock" "github.com/multiversx/mx-chain-go/trie/statistics" + "github.com/multiversx/mx-chain-go/trie/trieMetricsCollector" "github.com/stretchr/testify/assert" ) @@ -158,7 +160,7 @@ func TestLeafNode_commit(t *testing.T) { hash, _ := encodeNodeAndGetHash(ln) _ = ln.setHash() - err := ln.commitDirty(0, 5, db, db) + err := ln.commitDirty(db, db, dtmc) assert.Nil(t, err) encNode, _ := db.Get(hash) @@ -173,7 +175,7 @@ func TestLeafNode_commitEmptyNode(t *testing.T) { ln := &leafNode{} - err := ln.commitDirty(0, 5, nil, nil) + err := ln.commitDirty(nil, nil, nil) assert.True(t, errors.Is(err, ErrEmptyLeafNode)) } @@ -182,7 +184,7 @@ func TestLeafNode_commitNilNode(t *testing.T) { var ln *leafNode - err := ln.commitDirty(0, 5, nil, nil) + err := ln.commitDirty(nil, nil, nil) assert.True(t, errors.Is(err, ErrNilLeafNode)) } @@ -223,7 +225,7 @@ func TestLeafNode_resolveCollapsed(t *testing.T) { ln := getLn(getTestMarshalizerAndHasher()) - assert.Nil(t, ln.resolveCollapsed(0, nil)) + assert.Nil(t, ln.resolveCollapsed(0, nil, nil)) } func TestLeafNode_isCollapsed(t *testing.T) { @@ -239,10 +241,11 @@ func TestLeafNode_tryGet(t *testing.T) { ln := getLn(getTestMarshalizerAndHasher()) key := []byte("dog") - val, maxDepth, err := ln.tryGet(key, 0, nil) + tmc := trieMetricsCollector.NewTrieMetricsCollector() + val, err := ln.tryGet(key, tmc, nil) assert.Equal(t, []byte("dog"), val) assert.Nil(t, err) - assert.Equal(t, uint32(0), maxDepth) + assert.Equal(t, uint32(0), tmc.GetMaxDepth()) } func TestLeafNode_tryGetWrongKey(t *testing.T) { @@ -251,10 +254,11 @@ func TestLeafNode_tryGetWrongKey(t *testing.T) { ln := getLn(getTestMarshalizerAndHasher()) wrongKey := []byte{1, 2, 3} - val, maxDepth, err := ln.tryGet(wrongKey, 0, nil) + tmc := trieMetricsCollector.NewTrieMetricsCollector() + val, err := ln.tryGet(wrongKey, tmc, nil) assert.Nil(t, val) assert.Nil(t, err) - assert.Equal(t, uint32(0), maxDepth) + assert.Equal(t, uint32(0), tmc.GetMaxDepth()) } func TestLeafNode_tryGetEmptyNode(t *testing.T) { @@ -263,10 +267,11 @@ func TestLeafNode_tryGetEmptyNode(t *testing.T) { ln := &leafNode{} key := []byte("dog") - val, maxDepth, err := ln.tryGet(key, 0, nil) + tmc := trieMetricsCollector.NewTrieMetricsCollector() + val, err := ln.tryGet(key, tmc, nil) assert.True(t, errors.Is(err, ErrEmptyLeafNode)) assert.Nil(t, val) - assert.Equal(t, uint32(0), maxDepth) + assert.Equal(t, uint32(0), tmc.GetMaxDepth()) } func TestLeafNode_tryGetNilNode(t *testing.T) { @@ -275,10 +280,11 @@ func TestLeafNode_tryGetNilNode(t *testing.T) { var ln *leafNode key := []byte("dog") - val, maxDepth, err := ln.tryGet(key, 0, nil) + tmc := trieMetricsCollector.NewTrieMetricsCollector() + val, err := ln.tryGet(key, tmc, nil) assert.True(t, errors.Is(err, ErrNilLeafNode)) assert.Nil(t, val) - assert.Equal(t, uint32(0), maxDepth) + assert.Equal(t, uint32(0), tmc.GetMaxDepth()) } func TestLeafNode_getNext(t *testing.T) { @@ -287,7 +293,7 @@ func TestLeafNode_getNext(t *testing.T) { ln := getLn(getTestMarshalizerAndHasher()) key := []byte("dog") - n, key, err := ln.getNext(key, nil) + n, key, err := ln.getNext(key, nil, nil) assert.Nil(t, n) assert.Nil(t, key) assert.Nil(t, err) @@ -299,7 +305,7 @@ func TestLeafNode_getNextWrongKey(t *testing.T) { ln := getLn(getTestMarshalizerAndHasher()) wrongKey := append([]byte{2}, []byte("dog")...) - n, key, err := ln.getNext(wrongKey, nil) + n, key, err := ln.getNext(wrongKey, nil, nil) assert.Nil(t, n) assert.Nil(t, key) assert.Equal(t, ErrNodeNotFound, err) @@ -311,7 +317,7 @@ func TestLeafNode_getNextNilNode(t *testing.T) { var ln *leafNode key := []byte("dog") - n, key, err := ln.getNext(key, nil) + n, key, err := ln.getNext(key, nil, nil) assert.Nil(t, n) assert.Nil(t, key) assert.True(t, errors.Is(err, ErrNilLeafNode)) @@ -324,11 +330,13 @@ func TestLeafNode_insertAtSameKey(t *testing.T) { key := "dog" expectedVal := "dogs" - newNode, _, err := ln.insert(getTrieDataWithDefaultVersion(key, expectedVal), nil) + tmc := trieMetricsCollector.NewTrieMetricsCollector() + newNode, _, err := ln.insert(getTrieDataWithDefaultVersion(key, expectedVal), tmc, nil) assert.NotNil(t, newNode) assert.Nil(t, err) + assert.Equal(t, 1, tmc.GetSizeLoadedInMem()) - val, _, _ := newNode.tryGet([]byte(key), 0, nil) + val, _ := newNode.tryGet([]byte(key), dtmc, nil) assert.Equal(t, []byte(expectedVal), val) } @@ -343,11 +351,14 @@ func TestLeafNode_insertAtDifferentKey(t *testing.T) { nodeKey := []byte{3, 4, 5} nodeVal := []byte{3, 4, 5} - newNode, _, err := ln.insert(getTrieDataWithDefaultVersion(string(nodeKey), string(nodeVal)), nil) + tmc := trieMetricsCollector.NewTrieMetricsCollector() + newNode, _, err := ln.insert(getTrieDataWithDefaultVersion(string(nodeKey), string(nodeVal)), tmc, nil) assert.NotNil(t, newNode) assert.Nil(t, err) + expectedSize := newNode.sizeInBytes() + newNode.(*branchNode).children[3].sizeInBytes() - 1 + assert.Equal(t, expectedSize, tmc.GetSizeLoadedInMem()) - val, _, _ := newNode.tryGet(nodeKey, 0, nil) + val, _ := newNode.tryGet(nodeKey, trieMetricsCollector.NewTrieMetricsCollector(), nil) assert.Equal(t, nodeVal, val) assert.IsType(t, &branchNode{}, newNode) } @@ -357,13 +368,15 @@ func TestLeafNode_insertInStoredLnAtSameKey(t *testing.T) { db := testscommon.NewMemDbMock() ln := getLn(getTestMarshalizerAndHasher()) - _ = ln.commitDirty(0, 5, db, db) + _ = ln.commitDirty(db, db, dtmc) lnHash := ln.getHash() - newNode, oldHashes, err := ln.insert(getTrieDataWithDefaultVersion("dog", "dogs"), db) + tmc := trieMetricsCollector.NewTrieMetricsCollector() + newNode, oldHashes, err := ln.insert(getTrieDataWithDefaultVersion("dog", "dogs"), tmc, db) assert.NotNil(t, newNode) assert.Nil(t, err) assert.Equal(t, [][]byte{lnHash}, oldHashes) + assert.Equal(t, 1, tmc.GetSizeLoadedInMem()) } func TestLeafNode_insertInStoredLnAtDifferentKey(t *testing.T) { @@ -372,13 +385,16 @@ func TestLeafNode_insertInStoredLnAtDifferentKey(t *testing.T) { db := testscommon.NewMemDbMock() marsh, hasher := getTestMarshalizerAndHasher() ln, _ := newLeafNode(getTrieDataWithDefaultVersion(string([]byte{1, 2, 3}), "dog"), marsh, hasher) - _ = ln.commitDirty(0, 5, db, db) + _ = ln.commitDirty(db, db, dtmc) lnHash := ln.getHash() - newNode, oldHashes, err := ln.insert(getTrieDataWithDefaultVersion(string([]byte{4, 5, 6}), "dogs"), db) + tmc := trieMetricsCollector.NewTrieMetricsCollector() + newNode, oldHashes, err := ln.insert(getTrieDataWithDefaultVersion(string([]byte{4, 5, 6}), "dogs"), tmc, db) assert.NotNil(t, newNode) assert.Nil(t, err) assert.Equal(t, [][]byte{lnHash}, oldHashes) + expectedSize := newNode.sizeInBytes() + newNode.(*branchNode).children[4].sizeInBytes() - 1 + assert.Equal(t, expectedSize, tmc.GetSizeLoadedInMem()) } func TestLeafNode_insertInDirtyLnAtSameKey(t *testing.T) { @@ -386,7 +402,7 @@ func TestLeafNode_insertInDirtyLnAtSameKey(t *testing.T) { ln := getLn(getTestMarshalizerAndHasher()) - newNode, oldHashes, err := ln.insert(getTrieDataWithDefaultVersion("dog", "dogs"), nil) + newNode, oldHashes, err := ln.insert(getTrieDataWithDefaultVersion("dog", "dogs"), dtmc, nil) assert.NotNil(t, newNode) assert.Nil(t, err) assert.Equal(t, [][]byte{}, oldHashes) @@ -398,7 +414,7 @@ func TestLeafNode_insertInDirtyLnAtDifferentKey(t *testing.T) { marsh, hasher := getTestMarshalizerAndHasher() ln, _ := newLeafNode(getTrieDataWithDefaultVersion(string([]byte{1, 2, 3}), "dog"), marsh, hasher) - newNode, oldHashes, err := ln.insert(getTrieDataWithDefaultVersion(string([]byte{4, 5, 6}), "dogs"), nil) + newNode, oldHashes, err := ln.insert(getTrieDataWithDefaultVersion(string([]byte{4, 5, 6}), "dogs"), dtmc, nil) assert.NotNil(t, newNode) assert.Nil(t, err) assert.Equal(t, [][]byte{}, oldHashes) @@ -409,7 +425,7 @@ func TestLeafNode_insertInNilNode(t *testing.T) { var ln *leafNode - newNode, _, err := ln.insert(getTrieDataWithDefaultVersion("dog", "dogs"), nil) + newNode, _, err := ln.insert(getTrieDataWithDefaultVersion("dog", "dogs"), nil, nil) assert.Nil(t, newNode) assert.True(t, errors.Is(err, ErrNilLeafNode)) assert.Nil(t, newNode) @@ -420,10 +436,12 @@ func TestLeafNode_deletePresent(t *testing.T) { ln := getLn(getTestMarshalizerAndHasher()) - dirty, newNode, _, err := ln.delete([]byte("dog"), nil) + tmc := trieMetricsCollector.NewTrieMetricsCollector() + dirty, newNode, _, err := ln.delete([]byte("dog"), tmc, nil) assert.True(t, dirty) assert.Nil(t, err) assert.Nil(t, newNode) + assert.Equal(t, -ln.sizeInBytes(), tmc.GetSizeLoadedInMem()) } func TestLeafNode_deleteFromStoredLnAtSameKey(t *testing.T) { @@ -431,10 +449,10 @@ func TestLeafNode_deleteFromStoredLnAtSameKey(t *testing.T) { db := testscommon.NewMemDbMock() ln := getLn(getTestMarshalizerAndHasher()) - _ = ln.commitDirty(0, 5, db, db) + _ = ln.commitDirty(db, db, dtmc) lnHash := ln.getHash() - dirty, _, oldHashes, err := ln.delete([]byte("dog"), db) + dirty, _, oldHashes, err := ln.delete([]byte("dog"), dtmc, db) assert.True(t, dirty) assert.Nil(t, err) assert.Equal(t, [][]byte{lnHash}, oldHashes) @@ -445,13 +463,15 @@ func TestLeafNode_deleteFromLnAtDifferentKey(t *testing.T) { db := testscommon.NewMemDbMock() ln := getLn(getTestMarshalizerAndHasher()) - _ = ln.commitDirty(0, 5, db, db) + _ = ln.commitDirty(db, db, nil) wrongKey := []byte{1, 2, 3} - dirty, _, oldHashes, err := ln.delete(wrongKey, db) + tmc := trieMetricsCollector.NewTrieMetricsCollector() + dirty, _, oldHashes, err := ln.delete(wrongKey, tmc, db) assert.False(t, dirty) assert.Nil(t, err) assert.Equal(t, [][]byte{}, oldHashes) + assert.Equal(t, 0, tmc.GetSizeLoadedInMem()) } func TestLeafNode_deleteFromDirtyLnAtSameKey(t *testing.T) { @@ -459,7 +479,7 @@ func TestLeafNode_deleteFromDirtyLnAtSameKey(t *testing.T) { ln := getLn(getTestMarshalizerAndHasher()) - dirty, _, oldHashes, err := ln.delete([]byte("dog"), nil) + dirty, _, oldHashes, err := ln.delete([]byte("dog"), dtmc, nil) assert.True(t, dirty) assert.Nil(t, err) assert.Equal(t, [][]byte{}, oldHashes) @@ -471,7 +491,7 @@ func TestLeafNode_deleteNotPresent(t *testing.T) { ln := getLn(getTestMarshalizerAndHasher()) wrongKey := []byte{1, 2, 3} - dirty, newNode, _, err := ln.delete(wrongKey, nil) + dirty, newNode, _, err := ln.delete(wrongKey, dtmc, nil) assert.False(t, dirty) assert.Nil(t, err) assert.Equal(t, ln, newNode) @@ -485,7 +505,7 @@ func TestLeafNode_reduceNode(t *testing.T) { expected, _ := newLeafNode(getTrieDataWithDefaultVersion(string([]byte{2, 100, 111, 103}), ""), marsh, hasher) expected.dirty = true - n, newChildHash, err := ln.reduceNode(2) + n, newChildHash, err := ln.reduceNode(2, dtmc) assert.Equal(t, expected, n) assert.Nil(t, err) assert.True(t, newChildHash) @@ -506,7 +526,7 @@ func TestLeafNode_getChildren(t *testing.T) { ln := getLn(getTestMarshalizerAndHasher()) - children, err := ln.getChildren(nil) + children, err := ln.getChildren(nil, nil) assert.Nil(t, err) assert.Equal(t, 0, len(children)) } @@ -654,7 +674,7 @@ func TestLeafNode_getAllHashes(t *testing.T) { t.Parallel() ln := getLn(getTestMarshalizerAndHasher()) - hashes, err := ln.getAllHashes(testscommon.NewMemDbMock()) + hashes, err := ln.getAllHashes(nil, testscommon.NewMemDbMock()) assert.Nil(t, err) assert.Equal(t, 1, len(hashes)) assert.Equal(t, ln.hash, hashes[0]) @@ -690,7 +710,7 @@ func TestLeafNode_SizeInBytes(t *testing.T) { value := []byte("value") key := []byte("key") - hash := []byte("hash") + hash := bytes.Repeat([]byte{1}, 32) ln = &leafNode{ CollapsedLn: CollapsedLn{ Key: key, @@ -703,7 +723,7 @@ func TestLeafNode_SizeInBytes(t *testing.T) { hasher: nil, }, } - assert.Equal(t, len(key)+len(value)+len(hash)+1+2*pointerSizeInBytes, ln.sizeInBytes()) + assert.Equal(t, len(key)+len(value)+len(hash)+1+2*pointerSizeInBytes+nodeVersionSizeInBytes, ln.sizeInBytes()) } func TestLeafNode_writeNodeOnChannel(t *testing.T) { @@ -729,7 +749,7 @@ func TestLeafNode_commitContextDone(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() - err := ln.commitSnapshot(db, 0, nil, nil, ctx, statistics.NewTrieStatistics(), &testscommon.ProcessStatusHandlerStub{}, []byte{}, 0) + err := ln.commitSnapshot(db, 0, nil, nil, ctx, statistics.NewTrieStatistics(), &testscommon.ProcessStatusHandlerStub{}, []byte{}, dtmc) assert.Equal(t, core.ErrContextClosing, err) } diff --git a/trie/node.go b/trie/node.go index f15f182c1cb..ab2c0246fe3 100644 --- a/trie/node.go +++ b/trie/node.go @@ -16,13 +16,17 @@ import ( ) const ( - nrOfChildren = 17 - firstByte = 0 - hexTerminator = 16 - nibbleMask = 0x0f - pointerSizeInBytes = 8 - numNodeInnerPointers = 2 // each trie node contains a marshalizer and a hasher - pollingIdleNode = time.Millisecond + nrOfChildren = 17 + firstByte = 0 + hexTerminator = 16 + nibbleMask = 0x0f + pointerSizeInBytes = 8 + numNodeInnerPointers = 2 // each trie node contains a marshalizer and a hasher + pollingIdleNode = time.Millisecond + hashSizeInBytes = 32 // size of the hash in bytes + baseNodeSizeInBytes = hashSizeInBytes + numNodeInnerPointers*pointerSizeInBytes + 1 // 1 for the dirty flag + bnChildrenPointersSize = nrOfChildren * pointerSizeInBytes + nodeVersionSizeInBytes = 4 ) type baseNode struct { @@ -136,7 +140,7 @@ func treatLogError(logInstance logger.Logger, err error, key []byte) { logInstance.Trace(core.GetNodeFromDBErrorString, "error", err, "key", key, "stack trace", string(debug.Stack())) } -func resolveIfCollapsed(n node, pos byte, db common.TrieStorageInteractor) error { +func resolveIfCollapsed(n node, pos byte, tmc MetricsCollector, db common.TrieStorageInteractor) error { err := n.isEmptyOrNil() if err != nil { return err @@ -147,7 +151,7 @@ func resolveIfCollapsed(n node, pos byte, db common.TrieStorageInteractor) error return nil } - return n.resolveCollapsed(pos, db) + return n.resolveCollapsed(pos, tmc, db) } func handleStorageInteractorStats(db common.TrieStorageInteractor) { @@ -319,7 +323,7 @@ func commitSnapshot( ctx context.Context, stats common.TrieStatisticsHandler, idleProvider IdleNodeProvider, - depthLevel int, + tmc MetricsCollector, hash []byte, ) error { encChild, foundInEpoch, err := db.GetWithoutAddingToCache(hash, maxEpochToSearchFrom) @@ -340,10 +344,15 @@ func commitSnapshot( return err } - err = child.commitSnapshot(db, foundInEpoch, leavesChan, missingNodesChan, ctx, stats, idleProvider, encChild, depthLevel+1) + err = child.commitSnapshot(db, foundInEpoch, leavesChan, missingNodesChan, ctx, stats, idleProvider, encChild, tmc) if err != nil { return err } return db.PutInEpochWithoutCache(hash, encChild) } + +func isLeafNode(n node) bool { + _, ok := n.(*leafNode) + return ok +} diff --git a/trie/node_test.go b/trie/node_test.go index d5e8774a289..7200f27d17d 100644 --- a/trie/node_test.go +++ b/trie/node_test.go @@ -16,6 +16,7 @@ import ( "github.com/multiversx/mx-chain-go/state/parsers" "github.com/multiversx/mx-chain-go/testscommon" "github.com/multiversx/mx-chain-go/trie/keyBuilder" + "github.com/multiversx/mx-chain-go/trie/trieMetricsCollector" logger "github.com/multiversx/mx-chain-logger-go" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -165,7 +166,7 @@ func TestNode_getNodeFromDBAndDecodeBranchNode(t *testing.T) { db := testscommon.NewMemDbMock() bn, collapsedBn := getBnAndCollapsedBn(getTestMarshalizerAndHasher()) - _ = bn.commitDirty(0, 5, db, db) + _ = bn.commitDirty(db, db, dtmc) encNode, _ := bn.marsh.Marshal(collapsedBn) encNode = append(encNode, branch) @@ -184,7 +185,7 @@ func TestNode_getNodeFromDBAndDecodeExtensionNode(t *testing.T) { db := testscommon.NewMemDbMock() en, collapsedEn := getEnAndCollapsedEn() - _ = en.commitDirty(0, 5, db, db) + _ = en.commitDirty(db, db, dtmc) encNode, _ := en.marsh.Marshal(collapsedEn) encNode = append(encNode, extension) @@ -203,7 +204,7 @@ func TestNode_getNodeFromDBAndDecodeLeafNode(t *testing.T) { db := testscommon.NewMemDbMock() ln := getLn(getTestMarshalizerAndHasher()) - _ = ln.commitDirty(0, 5, db, db) + _ = ln.commitDirty(db, db, dtmc) encNode, _ := ln.marsh.Marshal(ln) encNode = append(encNode, leaf) @@ -223,11 +224,13 @@ func TestNode_resolveIfCollapsedBranchNode(t *testing.T) { db := testscommon.NewMemDbMock() bn, collapsedBn := getBnAndCollapsedBn(getTestMarshalizerAndHasher()) childPos := byte(2) - _ = bn.commitDirty(0, 5, db, db) + _ = bn.commitDirty(db, db, dtmc) - err := resolveIfCollapsed(collapsedBn, childPos, db) + tmc := trieMetricsCollector.NewTrieMetricsCollector() + err := resolveIfCollapsed(collapsedBn, childPos, tmc, db) assert.Nil(t, err) assert.False(t, collapsedBn.isCollapsed()) + assert.Equal(t, collapsedBn.children[childPos].sizeInBytes(), tmc.GetSizeLoadedInMem()) } func TestNode_resolveIfCollapsedExtensionNode(t *testing.T) { @@ -235,11 +238,13 @@ func TestNode_resolveIfCollapsedExtensionNode(t *testing.T) { db := testscommon.NewMemDbMock() en, collapsedEn := getEnAndCollapsedEn() - _ = en.commitDirty(0, 5, db, db) + _ = en.commitDirty(db, db, dtmc) - err := resolveIfCollapsed(collapsedEn, 0, db) + tmc := trieMetricsCollector.NewTrieMetricsCollector() + err := resolveIfCollapsed(collapsedEn, 0, tmc, db) assert.Nil(t, err) assert.False(t, collapsedEn.isCollapsed()) + assert.Equal(t, collapsedEn.child.sizeInBytes(), tmc.GetSizeLoadedInMem()) } func TestNode_resolveIfCollapsedLeafNode(t *testing.T) { @@ -247,11 +252,13 @@ func TestNode_resolveIfCollapsedLeafNode(t *testing.T) { db := testscommon.NewMemDbMock() ln := getLn(getTestMarshalizerAndHasher()) - _ = ln.commitDirty(0, 5, db, db) + _ = ln.commitDirty(db, db, dtmc) - err := resolveIfCollapsed(ln, 0, db) + tmc := trieMetricsCollector.NewTrieMetricsCollector() + err := resolveIfCollapsed(ln, 0, tmc, db) assert.Nil(t, err) assert.False(t, ln.isCollapsed()) + assert.Equal(t, 0, tmc.GetSizeLoadedInMem()) } func TestNode_resolveIfCollapsedNilNode(t *testing.T) { @@ -259,7 +266,7 @@ func TestNode_resolveIfCollapsedNilNode(t *testing.T) { var nodeInstance *extensionNode - err := resolveIfCollapsed(nodeInstance, 0, nil) + err := resolveIfCollapsed(nodeInstance, 0, nil, nil) assert.Equal(t, ErrNilExtensionNode, err) } @@ -1205,6 +1212,18 @@ func Test_treatCommitSnapshotErr(t *testing.T) { }) } +func TestIsLeafNode(t *testing.T) { + t.Parallel() + + bn, _ := getBnAndCollapsedBn(getTestMarshalizerAndHasher()) + assert.False(t, isLeafNode(bn)) + + ln, _ := newLeafNode(getTrieDataWithDefaultVersion("dog", "dog"), bn.marsh, bn.hasher) + assert.True(t, isLeafNode(ln)) + + assert.False(t, isLeafNode(nil)) +} + func Benchmark_ShouldStopIfContextDoneBlockingIfBusy(b *testing.B) { ctx := context.Background() b.ResetTimer() diff --git a/trie/patriciaMerkleTrie.go b/trie/patriciaMerkleTrie.go index ed92942eabe..8b744ee4dd0 100644 --- a/trie/patriciaMerkleTrie.go +++ b/trie/patriciaMerkleTrie.go @@ -19,6 +19,7 @@ import ( "github.com/multiversx/mx-chain-go/errors" "github.com/multiversx/mx-chain-go/trie/keyBuilder" "github.com/multiversx/mx-chain-go/trie/statistics" + "github.com/multiversx/mx-chain-go/trie/trieMetricsCollector" ) var log = logger.GetOrCreate("trie") @@ -29,10 +30,9 @@ const ( extension = iota leaf branch + minSizeInMemory = 1048576 // 1 MB ) -const rootDepthLevel = 0 - type patriciaMerkleTrie struct { root node @@ -43,10 +43,11 @@ type patriciaMerkleTrie struct { trieNodeVersionVerifier core.TrieNodeVersionVerifier mutOperation sync.RWMutex - oldHashes [][]byte - oldRoot []byte - maxTrieLevelInMemory uint - chanClose chan struct{} + oldHashes [][]byte + oldRoot []byte + chanClose chan struct{} + sizeInMemory int + maxSizeInMem uint64 } // NewTrie creates a new Patricia Merkle Trie @@ -55,7 +56,7 @@ func NewTrie( msh marshal.Marshalizer, hsh hashing.Hasher, enableEpochsHandler common.EnableEpochsHandler, - maxTrieLevelInMemory uint, + maxSizeInMemory uint64, ) (*patriciaMerkleTrie, error) { if check.IfNil(trieStorage) { return nil, ErrNilTrieStorage @@ -69,10 +70,9 @@ func NewTrie( if check.IfNil(enableEpochsHandler) { return nil, errors.ErrNilEnableEpochsHandler } - if maxTrieLevelInMemory == 0 { - return nil, ErrInvalidLevelValue + if maxSizeInMemory < minSizeInMemory { + return nil, fmt.Errorf("%w: provided %d, minimum %d", ErrInvalidMaxSizeInMemory, maxSizeInMemory, minSizeInMemory) } - log.Trace("created new trie", "max trie level in memory", maxTrieLevelInMemory) tnvv, err := core.NewTrieNodeVersionVerifier(enableEpochsHandler) if err != nil { @@ -85,10 +85,11 @@ func NewTrie( hasher: hsh, oldHashes: make([][]byte, 0), oldRoot: make([]byte, 0), - maxTrieLevelInMemory: maxTrieLevelInMemory, chanClose: make(chan struct{}), enableEpochsHandler: enableEpochsHandler, trieNodeVersionVerifier: tnvv, + // TODO collapse trie leaves if maxSizeInMemory is reached + maxSizeInMem: maxSizeInMemory, }, nil } @@ -103,13 +104,15 @@ func (tr *patriciaMerkleTrie) Get(key []byte) ([]byte, uint32, error) { } hexKey := keyBytesToHex(key) - val, depth, err := tr.root.tryGet(hexKey, rootDepthLevel, tr.trieStorage) + tmc := trieMetricsCollector.NewTrieMetricsCollector() + val, err := tr.root.tryGet(hexKey, tmc, tr.trieStorage) + tr.addSizeInMemory(tmc.GetSizeLoadedInMem()) if err != nil { err = fmt.Errorf("trie get error: %w, for key %v", err, hex.EncodeToString(key)) - return nil, depth, err + return nil, tmc.GetMaxDepth(), err } - return val, depth, nil + return val, tmc.GetMaxDepth(), nil } // Update updates the value at the given key. @@ -148,6 +151,7 @@ func (tr *patriciaMerkleTrie) update(key []byte, value []byte, version core.Trie if err != nil { return err } + tr.sizeInMemory += newRoot.sizeInBytes() tr.root = newRoot return nil @@ -157,7 +161,9 @@ func (tr *patriciaMerkleTrie) update(key []byte, value []byte, version core.Trie tr.oldRoot = tr.root.getHash() } - newRoot, oldHashes, err := tr.root.insert(newData, tr.trieStorage) + tmc := trieMetricsCollector.NewTrieMetricsCollector() + newRoot, oldHashes, err := tr.root.insert(newData, tmc, tr.trieStorage) + tr.addSizeInMemory(tmc.GetSizeLoadedInMem()) if err != nil { return err } @@ -177,6 +183,15 @@ func (tr *patriciaMerkleTrie) update(key []byte, value []byte, version core.Trie return nil } +func (tr *patriciaMerkleTrie) addSizeInMemory(size int) { + if tr.sizeInMemory+size < 0 { + log.Warn("trie size in memory is negative after adding size, resetting to 0", "size", size, "currentSize", tr.sizeInMemory) + tr.sizeInMemory = 0 + return + } + tr.sizeInMemory += size +} + // Delete removes the node that has the given key from the tree func (tr *patriciaMerkleTrie) Delete(key []byte) error { tr.mutOperation.Lock() @@ -195,7 +210,9 @@ func (tr *patriciaMerkleTrie) delete(hexKey []byte) error { tr.oldRoot = tr.root.getHash() } - _, newRoot, oldHashes, err := tr.root.delete(hexKey, tr.trieStorage) + tmc := trieMetricsCollector.NewTrieMetricsCollector() + _, newRoot, oldHashes, err := tr.root.delete(hexKey, tmc, tr.trieStorage) + tr.addSizeInMemory(tmc.GetSizeLoadedInMem()) if err != nil { return err } @@ -255,12 +272,10 @@ func (tr *patriciaMerkleTrie) Commit() error { log.Trace("started committing trie", "trie", tr.root.getHash()) } - err = tr.root.commitDirty(0, tr.maxTrieLevelInMemory, tr.trieStorage, tr.trieStorage) - if err != nil { - return err - } - - return nil + tmc := trieMetricsCollector.NewTrieMetricsCollector() + err = tr.root.commitDirty(tr.trieStorage, tr.trieStorage, tmc) + tr.addSizeInMemory(tmc.GetSizeLoadedInMem()) + return err } // Recreate returns a new trie, given the options @@ -288,7 +303,7 @@ func (tr *patriciaMerkleTrie) recreate(root []byte, tsm common.StorageManager) ( tr.marshalizer, tr.hasher, tr.enableEpochsHandler, - tr.maxTrieLevelInMemory, + tr.maxSizeInMem, ) } @@ -316,7 +331,7 @@ func (tr *patriciaMerkleTrie) String() string { if tr.root == nil { _, _ = fmt.Fprintln(writer, "*** EMPTY TRIE ***") } else { - tr.root.print(writer, 0, tr.trieStorage) + tr.root.print(writer, 0, trieMetricsCollector.NewDisabledTrieMetricsCollector(), tr.trieStorage) } return writer.String() @@ -369,7 +384,7 @@ func (tr *patriciaMerkleTrie) recreateFromDb(rootHash []byte, tsm common.Storage tr.marshalizer, tr.hasher, tr.enableEpochsHandler, - tr.maxTrieLevelInMemory, + tr.maxSizeInMem, ) if err != nil { return nil, nil, err @@ -382,6 +397,7 @@ func (tr *patriciaMerkleTrie) recreateFromDb(rootHash []byte, tsm common.Storage newRoot.setGivenHash(rootHash) newTr.root = newRoot + newTr.addSizeInMemory(newRoot.sizeInBytes()) return newTr, newRoot, nil } @@ -500,6 +516,7 @@ func (tr *patriciaMerkleTrie) GetAllLeavesOnChannel( tr.marshalizer, tr.chanClose, ctx, + trieMetricsCollector.NewDisabledTrieMetricsCollector(), ) if err != nil { leavesChannels.ErrChan.WriteInChanNonBlocking(err) @@ -530,7 +547,7 @@ func (tr *patriciaMerkleTrie) GetAllHashes() ([][]byte, error) { return nil, err } - hashes, err = tr.root.getAllHashes(tr.trieStorage) + hashes, err = tr.root.getAllHashes(trieMetricsCollector.NewDisabledTrieMetricsCollector(), tr.trieStorage) if err != nil { return nil, err } @@ -580,7 +597,7 @@ func (tr *patriciaMerkleTrie) GetProof(key []byte) ([][]byte, []byte, error) { proof = append(proof, encodedNode) value := currentNode.getValue() - currentNode, hexKey, errGet = currentNode.getNext(hexKey, tr.trieStorage) + currentNode, hexKey, errGet = currentNode.getNext(hexKey, trieMetricsCollector.NewDisabledTrieMetricsCollector(), tr.trieStorage) if errGet != nil { return nil, nil, errGet } @@ -656,7 +673,7 @@ func (tr *patriciaMerkleTrie) GetTrieStats(address string, rootHash []byte) (com } ts := statistics.NewTrieStatistics() - err = newTrie.root.collectStats(ts, rootDepthLevel, newTrie.trieStorage) + err = newTrie.root.collectStats(ts, trieMetricsCollector.NewTrieMetricsCollector(), newTrie.trieStorage) if err != nil { return nil, err } @@ -683,12 +700,10 @@ func (tr *patriciaMerkleTrie) CollectLeavesForMigration(args vmcommon.ArgsMigrat return err } - _, err = tr.root.collectLeavesForMigration(args, tr.trieStorage, keyBuilder.NewKeyBuilder()) - if err != nil { - return err - } - - return nil + tmc := trieMetricsCollector.NewTrieMetricsCollector() + _, err = tr.root.collectLeavesForMigration(args, tmc, tr.trieStorage, keyBuilder.NewKeyBuilder()) + tr.addSizeInMemory(tmc.GetSizeLoadedInMem()) + return err } func (tr *patriciaMerkleTrie) checkIfMigrationPossible(args vmcommon.ArgsMigrateDataTrieLeaves) error { @@ -735,6 +750,14 @@ func GetNodeDataFromHash(hash []byte, keyBuilder common.KeyBuilder, db common.Tr return n.getNodeData(keyBuilder) } +// SizeInMemory returns the size in memory of the trie +func (tr *patriciaMerkleTrie) SizeInMemory() int { + tr.mutOperation.RLock() + defer tr.mutOperation.RUnlock() + + return tr.sizeInMemory +} + // Close stops all the active goroutines started by the trie func (tr *patriciaMerkleTrie) Close() error { tr.mutOperation.Lock() diff --git a/trie/patriciaMerkleTrie_test.go b/trie/patriciaMerkleTrie_test.go index 76f4e34e230..7f9361638ae 100644 --- a/trie/patriciaMerkleTrie_test.go +++ b/trie/patriciaMerkleTrie_test.go @@ -37,6 +37,8 @@ import ( var emptyTrieHash = make([]byte, 32) +const tenMBSize = uint64(10485760) + func emptyTrie() common.Trie { tr, _ := trie.NewTrie(getDefaultTrieParameters()) @@ -44,18 +46,17 @@ func emptyTrie() common.Trie { } func emptyTrieWithCustomEnableEpochsHandler(handler common.EnableEpochsHandler) common.Trie { - storage, marshaller, hasher, _, maxTrieLevelInMem := getDefaultTrieParameters() + storage, marshaller, hasher, _, maxSizeInMem := getDefaultTrieParameters() - tr, _ := trie.NewTrie(storage, marshaller, hasher, handler, maxTrieLevelInMem) + tr, _ := trie.NewTrie(storage, marshaller, hasher, handler, maxSizeInMem) return tr } -func getDefaultTrieParameters() (common.StorageManager, marshal.Marshalizer, hashing.Hasher, common.EnableEpochsHandler, uint) { +func getDefaultTrieParameters() (common.StorageManager, marshal.Marshalizer, hashing.Hasher, common.EnableEpochsHandler, uint64) { args := trie.GetDefaultTrieStorageManagerParameters() trieStorageManager, _ := trie.NewTrieStorageManager(args) - maxTrieLevelInMemory := uint(1) - return trieStorageManager, args.Marshalizer, args.Hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, maxTrieLevelInMemory + return trieStorageManager, args.Marshalizer, args.Hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, tenMBSize } func initTrieMultipleValues(nr int) (common.Trie, [][]byte) { @@ -88,8 +89,8 @@ func addDefaultDataToTrie(tr common.Trie) { func TestNewTrieWithNilTrieStorage(t *testing.T) { t.Parallel() - _, marshalizer, hasher, enableEpochsHandler, maxTrieLevelInMemory := getDefaultTrieParameters() - tr, err := trie.NewTrie(nil, marshalizer, hasher, enableEpochsHandler, maxTrieLevelInMemory) + _, marshalizer, hasher, enableEpochsHandler, maxSizeInMem := getDefaultTrieParameters() + tr, err := trie.NewTrie(nil, marshalizer, hasher, enableEpochsHandler, maxSizeInMem) assert.Nil(t, tr) assert.Equal(t, trie.ErrNilTrieStorage, err) @@ -98,8 +99,8 @@ func TestNewTrieWithNilTrieStorage(t *testing.T) { func TestNewTrieWithNilMarshalizer(t *testing.T) { t.Parallel() - trieStorage, _, hasher, enableEpochsHandler, maxTrieLevelInMemory := getDefaultTrieParameters() - tr, err := trie.NewTrie(trieStorage, nil, hasher, enableEpochsHandler, maxTrieLevelInMemory) + trieStorage, _, hasher, enableEpochsHandler, maxSizeInMem := getDefaultTrieParameters() + tr, err := trie.NewTrie(trieStorage, nil, hasher, enableEpochsHandler, maxSizeInMem) assert.Nil(t, tr) assert.Equal(t, trie.ErrNilMarshalizer, err) @@ -108,8 +109,8 @@ func TestNewTrieWithNilMarshalizer(t *testing.T) { func TestNewTrieWithNilHasher(t *testing.T) { t.Parallel() - trieStorage, marshalizer, _, enableEpochsHandler, maxTrieLevelInMemory := getDefaultTrieParameters() - tr, err := trie.NewTrie(trieStorage, marshalizer, nil, enableEpochsHandler, maxTrieLevelInMemory) + trieStorage, marshalizer, _, enableEpochsHandler, maxSizeInMem := getDefaultTrieParameters() + tr, err := trie.NewTrie(trieStorage, marshalizer, nil, enableEpochsHandler, maxSizeInMem) assert.Nil(t, tr) assert.Equal(t, trie.ErrNilHasher, err) @@ -118,21 +119,21 @@ func TestNewTrieWithNilHasher(t *testing.T) { func TestNewTrieWithNilEnableEpochsHandler(t *testing.T) { t.Parallel() - trieStorage, marshalizer, hasher, _, maxTrieLevelInMemory := getDefaultTrieParameters() - tr, err := trie.NewTrie(trieStorage, marshalizer, hasher, nil, maxTrieLevelInMemory) + trieStorage, marshalizer, hasher, _, maxSizeInMem := getDefaultTrieParameters() + tr, err := trie.NewTrie(trieStorage, marshalizer, hasher, nil, maxSizeInMem) assert.Nil(t, tr) assert.Equal(t, errorsCommon.ErrNilEnableEpochsHandler, err) } -func TestNewTrieWithInvalidMaxTrieLevelInMemory(t *testing.T) { +func TestNewTrieWithInvalidMaxSizeInMemory(t *testing.T) { t.Parallel() trieStorage, marshalizer, hasher, enableEpochsHandler, _ := getDefaultTrieParameters() tr, err := trie.NewTrie(trieStorage, marshalizer, hasher, enableEpochsHandler, 0) assert.Nil(t, tr) - assert.Equal(t, trie.ErrInvalidLevelValue, err) + assert.True(t, errors.Is(err, trie.ErrInvalidMaxSizeInMemory)) } func TestPatriciaMerkleTree_Get(t *testing.T) { @@ -900,6 +901,7 @@ func TestPatriciaMerkleTrie_GetTrieStats(t *testing.T) { _ = tr.Update([]byte("dog"), []byte("reindeer")) _ = tr.Update([]byte("fog"), []byte("puppy")) _ = tr.Update([]byte("dogglesworth"), []byte("cat")) + _ = tr.Update([]byte("abch"), []byte("car")) _ = tr.Commit() rootHash, _ := tr.RootHash() @@ -911,10 +913,10 @@ func TestPatriciaMerkleTrie_GetTrieStats(t *testing.T) { stats, err := ts.GetTrieStats(address, rootHash) assert.Nil(t, err) - assert.Equal(t, uint64(2), stats.GetNumBranchNodes()) - assert.Equal(t, uint64(1), stats.GetNumExtensionNodes()) - assert.Equal(t, uint64(3), stats.GetNumLeafNodes()) - assert.Equal(t, uint64(6), stats.GetTotalNumNodes()) + assert.Equal(t, uint64(3), stats.GetNumBranchNodes()) + assert.Equal(t, uint64(2), stats.GetNumExtensionNodes()) + assert.Equal(t, uint64(4), stats.GetNumLeafNodes()) + assert.Equal(t, uint64(9), stats.GetTotalNumNodes()) assert.Equal(t, uint32(3), stats.GetMaxTrieDepth()) } @@ -1062,7 +1064,7 @@ func TestPatriciaMerkleTrie_GetSerializedNodesShouldSerializeTheCalls(t *testing }, } - tr, _ := trie.NewTrie(testTrieStorageManager, args.Marshalizer, args.Hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, 5) + tr, _ := trie.NewTrie(testTrieStorageManager, args.Marshalizer, args.Hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, tenMBSize) numGoRoutines := 100 wg := sync.WaitGroup{} wg.Add(numGoRoutines) @@ -1484,13 +1486,13 @@ func TestPatriciaMerkleTrie_IsMigrated(t *testing.T) { t.Run("not migrated", func(t *testing.T) { t.Parallel() - tsm, marshaller, hasher, _, maxTrieInMem := getDefaultTrieParameters() + tsm, marshaller, hasher, _, maxSizeInMem := getDefaultTrieParameters() enableEpochs := &enableEpochsHandlerMock.EnableEpochsHandlerStub{ IsFlagEnabledCalled: func(flag core.EnableEpochFlag) bool { return flag == common.AutoBalanceDataTriesFlag }, } - tr, _ := trie.NewTrie(tsm, marshaller, hasher, enableEpochs, maxTrieInMem) + tr, _ := trie.NewTrie(tsm, marshaller, hasher, enableEpochs, maxSizeInMem) _ = tr.Update([]byte("dog"), []byte("reindeer")) isMigrated, err := tr.IsMigratedToLatestVersion() @@ -1501,13 +1503,13 @@ func TestPatriciaMerkleTrie_IsMigrated(t *testing.T) { t.Run("migrated", func(t *testing.T) { t.Parallel() - tsm, marshaller, hasher, _, maxTrieInMem := getDefaultTrieParameters() + tsm, marshaller, hasher, _, maxSizeInMem := getDefaultTrieParameters() enableEpochs := &enableEpochsHandlerMock.EnableEpochsHandlerStub{ IsFlagEnabledCalled: func(flag core.EnableEpochFlag) bool { return flag == common.AutoBalanceDataTriesFlag }, } - tr, _ := trie.NewTrie(tsm, marshaller, hasher, enableEpochs, maxTrieInMem) + tr, _ := trie.NewTrie(tsm, marshaller, hasher, enableEpochs, maxSizeInMem) _ = tr.UpdateWithVersion([]byte("dog"), []byte("reindeer"), core.AutoBalanceEnabled) isMigrated, err := tr.IsMigratedToLatestVersion() @@ -1549,6 +1551,53 @@ func TestGetNodeDataFromHash(t *testing.T) { } +func TestPatriciaMerkleTree_SizeInMemory(t *testing.T) { + t.Parallel() + + tr := emptyTrie() + assert.Equal(t, 0, tr.SizeInMemory()) + addDefaultDataToTrie(tr) + + assert.Equal(t, 779, tr.SizeInMemory()) // 3 leaves + 2 branch nodes + 1 extension node + err := tr.Commit() + assert.Nil(t, err) + assert.Equal(t, 596, tr.SizeInMemory()) // leaves are collapsed + + err = tr.Delete([]byte("dog")) + assert.Nil(t, err) + assert.Equal(t, 313, tr.SizeInMemory()) // 1 branch node + 2 leaves + + err = tr.Commit() + assert.Nil(t, err) + assert.Equal(t, 249, tr.SizeInMemory()) // collapse leaf + + err = tr.Update([]byte("dog"), []byte("puppy")) + assert.Nil(t, err) + assert.Equal(t, 712, tr.SizeInMemory()) + + rootHash, err := tr.RootHash() + assert.Nil(t, err) + newTrie, err := tr.Recreate(holders.NewDefaultRootHashesHolder(rootHash)) + assert.Nil(t, err) + assert.Equal(t, 249, newTrie.SizeInMemory()) // only root node is in memory + + val, depth, err := newTrie.Get([]byte("dog")) + assert.Nil(t, err) + assert.Equal(t, []byte("puppy"), val) + assert.Equal(t, uint32(3), depth) + assert.Equal(t, 654, newTrie.SizeInMemory()) + + val, depth, err = newTrie.Get([]byte("dog")) + assert.Nil(t, err) + assert.Equal(t, []byte("puppy"), val) + assert.Equal(t, uint32(3), depth) + assert.Equal(t, 654, newTrie.SizeInMemory()) + + err = tr.Delete([]byte("doe")) // delete collapsed node + assert.Nil(t, err) + assert.Equal(t, 464, tr.SizeInMemory()) +} + func BenchmarkPatriciaMerkleTree_Insert(b *testing.B) { tr := emptyTrie() hsh := keccak.NewKeccak() diff --git a/trie/snapshotTrieStorageManager_test.go b/trie/snapshotTrieStorageManager_test.go index c8cc2df3ce2..9415edd6566 100644 --- a/trie/snapshotTrieStorageManager_test.go +++ b/trie/snapshotTrieStorageManager_test.go @@ -10,7 +10,7 @@ import ( "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/storage" "github.com/multiversx/mx-chain-go/testscommon" - "github.com/multiversx/mx-chain-go/testscommon/trie" + storageMock "github.com/multiversx/mx-chain-go/testscommon/storage" "github.com/stretchr/testify/assert" ) @@ -30,7 +30,7 @@ func TestNewSnapshotTrieStorageManager(t *testing.T) { t.Parallel() _, trieStorage := newEmptyTrie() - trieStorage.mainStorer = &trie.SnapshotPruningStorerStub{} + trieStorage.mainStorer = &storageMock.SnapshotPruningStorerStub{} stsm, err := newSnapshotTrieStorageManager(trieStorage, 0) assert.Nil(t, err) assert.False(t, check.IfNil(stsm)) @@ -43,7 +43,7 @@ func TestSnapshotTrieStorageManager_GetWithoutAddingToCache(t *testing.T) { t.Parallel() _, trieStorage := newEmptyTrie() - trieStorage.mainStorer = &trie.SnapshotPruningStorerStub{ + trieStorage.mainStorer = &storageMock.SnapshotPruningStorerStub{ GetWithoutAddingToCacheCalled: func(key []byte, maxEpochToSearchFrom uint32) ([]byte, core.OptionalUint32, error) { return nil, core.OptionalUint32{}, core.ErrContextClosing }, @@ -59,7 +59,7 @@ func TestSnapshotTrieStorageManager_GetWithoutAddingToCache(t *testing.T) { t.Parallel() _, trieStorage := newEmptyTrie() - trieStorage.mainStorer = &trie.SnapshotPruningStorerStub{ + trieStorage.mainStorer = &storageMock.SnapshotPruningStorerStub{ GetWithoutAddingToCacheCalled: func(_ []byte, _ uint32) ([]byte, core.OptionalUint32, error) { return nil, core.OptionalUint32{}, storage.ErrDBIsClosed }, @@ -75,7 +75,7 @@ func TestSnapshotTrieStorageManager_GetWithoutAddingToCache(t *testing.T) { _, trieStorage := newEmptyTrie() getFromOldEpochsWithoutCacheCalled := false - trieStorage.mainStorer = &trie.SnapshotPruningStorerStub{ + trieStorage.mainStorer = &storageMock.SnapshotPruningStorerStub{ GetWithoutAddingToCacheCalled: func(_ []byte, _ uint32) ([]byte, core.OptionalUint32, error) { getFromOldEpochsWithoutCacheCalled = true return nil, core.OptionalUint32{}, nil @@ -95,7 +95,7 @@ func TestSnapshotTrieStorageManager_PutInEpochWithoutCache(t *testing.T) { t.Parallel() _, trieStorage := newEmptyTrie() - trieStorage.mainStorer = &trie.SnapshotPruningStorerStub{ + trieStorage.mainStorer = &storageMock.SnapshotPruningStorerStub{ PutInEpochWithoutCacheCalled: func(_ []byte, _ []byte, _ uint32) error { return core.ErrContextClosing }, @@ -111,7 +111,7 @@ func TestSnapshotTrieStorageManager_PutInEpochWithoutCache(t *testing.T) { _, trieStorage := newEmptyTrie() putWithoutCacheCalled := false - trieStorage.mainStorer = &trie.SnapshotPruningStorerStub{ + trieStorage.mainStorer = &storageMock.SnapshotPruningStorerStub{ PutInEpochWithoutCacheCalled: func(_ []byte, _ []byte, _ uint32) error { putWithoutCacheCalled = true return nil @@ -131,7 +131,7 @@ func TestSnapshotTrieStorageManager_GetFromLastEpoch(t *testing.T) { t.Parallel() _, trieStorage := newEmptyTrie() - trieStorage.mainStorer = &trie.SnapshotPruningStorerStub{ + trieStorage.mainStorer = &storageMock.SnapshotPruningStorerStub{ GetFromLastEpochCalled: func(_ []byte) ([]byte, error) { return nil, core.ErrContextClosing }, @@ -148,7 +148,7 @@ func TestSnapshotTrieStorageManager_GetFromLastEpoch(t *testing.T) { _, trieStorage := newEmptyTrie() getFromLastEpochCalled := false - trieStorage.mainStorer = &trie.SnapshotPruningStorerStub{ + trieStorage.mainStorer = &storageMock.SnapshotPruningStorerStub{ GetFromLastEpochCalled: func(_ []byte) ([]byte, error) { getFromLastEpochCalled = true return nil, nil @@ -167,7 +167,7 @@ func TestSnapshotTrieStorageManager_AlsoAddInPreviousEpoch(t *testing.T) { t.Run("HasValue is false", func(t *testing.T) { val := []byte("val") _, trieStorage := newEmptyTrie() - trieStorage.mainStorer = &trie.SnapshotPruningStorerStub{ + trieStorage.mainStorer = &storageMock.SnapshotPruningStorerStub{ GetWithoutAddingToCacheCalled: func(_ []byte, _ uint32) ([]byte, core.OptionalUint32, error) { return val, core.OptionalUint32{}, nil }, @@ -184,7 +184,7 @@ func TestSnapshotTrieStorageManager_AlsoAddInPreviousEpoch(t *testing.T) { t.Run("epoch is previous epoch", func(t *testing.T) { val := []byte("val") _, trieStorage := newEmptyTrie() - trieStorage.mainStorer = &trie.SnapshotPruningStorerStub{ + trieStorage.mainStorer = &storageMock.SnapshotPruningStorerStub{ GetWithoutAddingToCacheCalled: func(_ []byte, _ uint32) ([]byte, core.OptionalUint32, error) { epoch := core.OptionalUint32{ Value: 4, @@ -205,7 +205,7 @@ func TestSnapshotTrieStorageManager_AlsoAddInPreviousEpoch(t *testing.T) { t.Run("epoch is 0", func(t *testing.T) { val := []byte("val") _, trieStorage := newEmptyTrie() - trieStorage.mainStorer = &trie.SnapshotPruningStorerStub{ + trieStorage.mainStorer = &storageMock.SnapshotPruningStorerStub{ GetWithoutAddingToCacheCalled: func(_ []byte, _ uint32) ([]byte, core.OptionalUint32, error) { epoch := core.OptionalUint32{ Value: 4, @@ -226,7 +226,7 @@ func TestSnapshotTrieStorageManager_AlsoAddInPreviousEpoch(t *testing.T) { t.Run("key is ActiveDBKey", func(t *testing.T) { val := []byte("val") _, trieStorage := newEmptyTrie() - trieStorage.mainStorer = &trie.SnapshotPruningStorerStub{ + trieStorage.mainStorer = &storageMock.SnapshotPruningStorerStub{ GetWithoutAddingToCacheCalled: func(_ []byte, _ uint32) ([]byte, core.OptionalUint32, error) { epoch := core.OptionalUint32{ Value: 3, @@ -247,7 +247,7 @@ func TestSnapshotTrieStorageManager_AlsoAddInPreviousEpoch(t *testing.T) { t.Run("key is TrieSyncedKey", func(t *testing.T) { val := []byte("val") _, trieStorage := newEmptyTrie() - trieStorage.mainStorer = &trie.SnapshotPruningStorerStub{ + trieStorage.mainStorer = &storageMock.SnapshotPruningStorerStub{ GetWithoutAddingToCacheCalled: func(_ []byte, _ uint32) ([]byte, core.OptionalUint32, error) { epoch := core.OptionalUint32{ Value: 3, @@ -269,7 +269,7 @@ func TestSnapshotTrieStorageManager_AlsoAddInPreviousEpoch(t *testing.T) { val := []byte("val") putInEpochCalled := false _, trieStorage := newEmptyTrie() - trieStorage.mainStorer = &trie.SnapshotPruningStorerStub{ + trieStorage.mainStorer = &storageMock.SnapshotPruningStorerStub{ GetWithoutAddingToCacheCalled: func(_ []byte, _ uint32) ([]byte, core.OptionalUint32, error) { epoch := core.OptionalUint32{ Value: 3, diff --git a/trie/sync.go b/trie/sync.go index ce48f8c8e6b..9e356405e8c 100644 --- a/trie/sync.go +++ b/trie/sync.go @@ -15,6 +15,7 @@ import ( "github.com/multiversx/mx-chain-core-go/marshal" "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/storage" + "github.com/multiversx/mx-chain-go/trie/trieMetricsCollector" ) type trieNodeInfo struct { @@ -175,6 +176,7 @@ func (ts *trieSyncer) checkIfSynced() (bool, error) { missingNodes := make(map[string]struct{}) currentMissingNodes := make([][]byte, 0) checkedNodes := make(map[string]struct{}) + tmc := trieMetricsCollector.NewDisabledTrieMetricsCollector() newElement := true shouldRetryAfterRequest := false @@ -233,7 +235,7 @@ func (ts *trieSyncer) checkIfSynced() (bool, error) { continue } - nextNodes, err = currentNode.getChildren(ts.db) + nextNodes, err = currentNode.getChildren(tmc, ts.db) if err != nil { return false, err } diff --git a/trie/syncTrieStorageManager_test.go b/trie/syncTrieStorageManager_test.go index 0e7c7532433..5e7fe9ce3e5 100644 --- a/trie/syncTrieStorageManager_test.go +++ b/trie/syncTrieStorageManager_test.go @@ -6,7 +6,7 @@ import ( "testing" "github.com/multiversx/mx-chain-go/testscommon" - "github.com/multiversx/mx-chain-go/testscommon/trie" + "github.com/multiversx/mx-chain-go/testscommon/storage" "github.com/stretchr/testify/assert" ) @@ -34,7 +34,7 @@ func TestNewSyncTrieStorageManager(t *testing.T) { t.Parallel() _, trieStorage := newEmptyTrie() - trieStorage.mainStorer = &trie.SnapshotPruningStorerStub{} + trieStorage.mainStorer = &storage.SnapshotPruningStorerStub{} stsm, err := NewSyncTrieStorageManager(trieStorage) assert.Nil(t, err) assert.NotNil(t, stsm) @@ -45,7 +45,7 @@ func TestNewSyncTrieStorageManager_PutInFirstEpoch(t *testing.T) { _, trieStorage := newEmptyTrie() putInEpochCalled := 0 - trieStorage.mainStorer = &trie.SnapshotPruningStorerStub{ + trieStorage.mainStorer = &storage.SnapshotPruningStorerStub{ PutInEpochCalled: func(_ []byte, _ []byte, _ uint32) error { putInEpochCalled++ return nil @@ -66,7 +66,7 @@ func TestNewSyncTrieStorageManager_PutInEpochError(t *testing.T) { expectedErr := errors.New("expected error") _, trieStorage := newEmptyTrie() - trieStorage.mainStorer = &trie.SnapshotPruningStorerStub{ + trieStorage.mainStorer = &storage.SnapshotPruningStorerStub{ PutInEpochCalled: func(_ []byte, _ []byte, _ uint32) error { return expectedErr }, @@ -82,7 +82,7 @@ func TestNewSyncTrieStorageManager_PutInEpoch(t *testing.T) { _, trieStorage := newEmptyTrie() putInEpochCalled := 0 - trieStorage.mainStorer = &trie.SnapshotPruningStorerStub{ + trieStorage.mainStorer = &storage.SnapshotPruningStorerStub{ PutInEpochCalled: func(_ []byte, _ []byte, _ uint32) error { putInEpochCalled++ return nil diff --git a/trie/sync_test.go b/trie/sync_test.go index 77fe8a6c75b..28b957d2bb8 100644 --- a/trie/sync_test.go +++ b/trie/sync_test.go @@ -3,6 +3,7 @@ package trie import ( "context" "errors" + "sync" "testing" "time" @@ -10,6 +11,7 @@ import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-core-go/data" + trieMock "github.com/multiversx/mx-chain-go/testscommon/storage" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -18,8 +20,8 @@ import ( "github.com/multiversx/mx-chain-go/testscommon/cache" "github.com/multiversx/mx-chain-go/testscommon/hashingMocks" "github.com/multiversx/mx-chain-go/testscommon/marshallerMock" - trieMock "github.com/multiversx/mx-chain-go/testscommon/trie" "github.com/multiversx/mx-chain-go/trie/statistics" + "github.com/multiversx/mx-chain-go/trie/trieMetricsCollector" ) func createMockArgument(timeout time.Duration) ArgTrieSyncer { @@ -225,10 +227,10 @@ func TestTrieSync_FoundInStorageShouldNotRequest(t *testing.T) { }, } - err = bn.commitDirty(0, 5, db, db) + err = bn.commitDirty(db, db, dtmc) require.Nil(t, err) - leaves, err := bn.getChildren(db) + leaves, err := bn.getChildren(trieMetricsCollector.NewDisabledTrieMetricsCollector(), db) require.Nil(t, err) numLeaves := len(leaves) diff --git a/trie/trieMetricsCollector/disabledTrieMetricsCollector.go b/trie/trieMetricsCollector/disabledTrieMetricsCollector.go new file mode 100644 index 00000000000..edb051b948e --- /dev/null +++ b/trie/trieMetricsCollector/disabledTrieMetricsCollector.go @@ -0,0 +1,31 @@ +package trieMetricsCollector + +type disabledTrieMetricsCollector struct{} + +// NewDisabledTrieMetricsCollector returns a new instance of disabledTrieMetricsCollector +func NewDisabledTrieMetricsCollector() *disabledTrieMetricsCollector { + return &disabledTrieMetricsCollector{} +} + +// SetDepth is a no-op for the disabled metrics collector +func (d *disabledTrieMetricsCollector) SetDepth(_ uint32) { +} + +// GetCurrentDepth returns 0 for the disabled metrics collector +func (d *disabledTrieMetricsCollector) GetCurrentDepth() uint32 { + return 0 +} + +// GetMaxDepth returns 0 for the disabled metrics collector +func (d *disabledTrieMetricsCollector) GetMaxDepth() uint32 { + return 0 +} + +// AddSizeLoadedInMem is a no-op for the disabled metrics collector +func (d *disabledTrieMetricsCollector) AddSizeLoadedInMem(_ int) { +} + +// GetSizeLoadedInMem returns 0 for the disabled metrics collector +func (d *disabledTrieMetricsCollector) GetSizeLoadedInMem() int { + return 0 +} diff --git a/trie/trieMetricsCollector/disabledTrieMetricsCollector_test.go b/trie/trieMetricsCollector/disabledTrieMetricsCollector_test.go new file mode 100644 index 00000000000..3b3aa14d21b --- /dev/null +++ b/trie/trieMetricsCollector/disabledTrieMetricsCollector_test.go @@ -0,0 +1,49 @@ +package trieMetricsCollector + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNewDisabledTrieMetricsCollector(t *testing.T) { + t.Parallel() + + assert.NotNil(t, NewDisabledTrieMetricsCollector()) +} + +func TestDisabledTrieMetricsCollector_SetDepthDoesNotPanic(t *testing.T) { + t.Parallel() + + collector := NewDisabledTrieMetricsCollector() + collector.SetDepth(5) + + // No assertion needed, just checking that it doesn't panic +} + +func TestDisabledTrieMetricsCollector_GetMaxDepthReturnsZero(t *testing.T) { + t.Parallel() + + collector := NewDisabledTrieMetricsCollector() + maxDepth := collector.GetMaxDepth() + + assert.Equal(t, uint32(0), maxDepth) +} + +func TestDisabledTrieMetricsCollector_AddSizeLoadedInMemDoesNotPanic(t *testing.T) { + t.Parallel() + + collector := NewDisabledTrieMetricsCollector() + collector.AddSizeLoadedInMem(100) + + // No assertion needed, just checking that it doesn't panic +} + +func TestDisabledTrieMetricsCollector_GetSizeLoadedInMemReturnsZero(t *testing.T) { + t.Parallel() + + collector := NewDisabledTrieMetricsCollector() + size := collector.GetSizeLoadedInMem() + + assert.Equal(t, 0, size) +} diff --git a/trie/trieMetricsCollector/trieMetricsCollector.go b/trie/trieMetricsCollector/trieMetricsCollector.go new file mode 100644 index 00000000000..77af2411a21 --- /dev/null +++ b/trie/trieMetricsCollector/trieMetricsCollector.go @@ -0,0 +1,45 @@ +package trieMetricsCollector + +type trieMetricsCollector struct { + currentDepth int + maxDepth int + sizeLoadedInMem int +} + +// NewTrieMetricsCollector creates a new instance of trieMetricsCollector +func NewTrieMetricsCollector() *trieMetricsCollector { + return &trieMetricsCollector{ + maxDepth: 0, + sizeLoadedInMem: 0, + } +} + +// SetDepth sets the maxDepth to the provided value if it is greater than the current maxDepth +func (tmc *trieMetricsCollector) SetDepth(depth uint32) { + tmc.currentDepth = int(depth) + if depth <= uint32(tmc.maxDepth) { + return + } + + tmc.maxDepth = int(depth) +} + +// GetCurrentDepth returns the current depth stored in the collector +func (tmc *trieMetricsCollector) GetCurrentDepth() uint32 { + return uint32(tmc.currentDepth) +} + +// GetMaxDepth returns the collected maxDepth +func (tmc *trieMetricsCollector) GetMaxDepth() uint32 { + return uint32(tmc.maxDepth) +} + +// AddSizeLoadedInMem adds the size of the loaded data in memory to the collector +func (tmc *trieMetricsCollector) AddSizeLoadedInMem(size int) { + tmc.sizeLoadedInMem += size +} + +// GetSizeLoadedInMem returns the total size of data loaded in memory +func (tmc *trieMetricsCollector) GetSizeLoadedInMem() int { + return tmc.sizeLoadedInMem +} diff --git a/trie/trieMetricsCollector/trieMetricsCollector_test.go b/trie/trieMetricsCollector/trieMetricsCollector_test.go new file mode 100644 index 00000000000..87d21c91691 --- /dev/null +++ b/trie/trieMetricsCollector/trieMetricsCollector_test.go @@ -0,0 +1,80 @@ +package trieMetricsCollector + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNewTrieMetricsCollector(t *testing.T) { + t.Parallel() + + collector := NewTrieMetricsCollector() + assert.NotNil(t, collector) + assert.Equal(t, 0, collector.maxDepth) + assert.Equal(t, 0, collector.sizeLoadedInMem) +} + +func TestTrieMetricsCollector_SetDepth(t *testing.T) { + t.Parallel() + + collector := NewTrieMetricsCollector() + collector.SetDepth(5) + + assert.Equal(t, 5, collector.maxDepth) + collector.SetDepth(3) + assert.Equal(t, 5, collector.maxDepth) // Should not change since 3 < 5 + collector.SetDepth(7) + assert.Equal(t, 7, collector.maxDepth) // Should update to 7 since it's greater than 5 +} + +func TestTrieMetricsCollector_GetCurrentDepth(t *testing.T) { + t.Parallel() + + collector := NewTrieMetricsCollector() + assert.Equal(t, uint32(0), collector.GetCurrentDepth()) + + collector.SetDepth(8) + assert.Equal(t, uint32(8), collector.GetCurrentDepth()) + + collector.SetDepth(4) + assert.Equal(t, uint32(4), collector.GetCurrentDepth()) // Current depth should update to 4 + assert.Equal(t, uint32(8), collector.GetMaxDepth()) // Max depth should remain 8 +} + +func TestTrieMetricsCollector_GetMaxDepth(t *testing.T) { + t.Parallel() + + collector := NewTrieMetricsCollector() + assert.Equal(t, uint32(0), collector.GetMaxDepth()) + + collector.SetDepth(10) + assert.Equal(t, uint32(10), collector.GetMaxDepth()) + + collector.SetDepth(5) + assert.Equal(t, uint32(10), collector.GetMaxDepth()) // Should not change since 5 < 10 +} + +func TestTrieMetricsCollector_AddSizeLoadedInMem(t *testing.T) { + t.Parallel() + + collector := NewTrieMetricsCollector() + collector.AddSizeLoadedInMem(100) + assert.Equal(t, 100, collector.sizeLoadedInMem) + + collector.AddSizeLoadedInMem(50) + assert.Equal(t, 150, collector.sizeLoadedInMem) // Should accumulate size +} + +func TestTrieMetricsCollector_GetSizeLoadedInMem(t *testing.T) { + t.Parallel() + + collector := NewTrieMetricsCollector() + assert.Equal(t, 0, collector.GetSizeLoadedInMem()) + + collector.AddSizeLoadedInMem(200) + assert.Equal(t, 200, collector.GetSizeLoadedInMem()) + + collector.AddSizeLoadedInMem(300) + assert.Equal(t, 500, collector.GetSizeLoadedInMem()) // Should accumulate size +} diff --git a/trie/trieStorageManager.go b/trie/trieStorageManager.go index 796eace218a..0c3cd4b97c2 100644 --- a/trie/trieStorageManager.go +++ b/trie/trieStorageManager.go @@ -18,6 +18,7 @@ import ( "github.com/multiversx/mx-chain-go/config" "github.com/multiversx/mx-chain-go/storage" "github.com/multiversx/mx-chain-go/trie/statistics" + "github.com/multiversx/mx-chain-go/trie/trieMetricsCollector" ) // trieStorageManager manages all the storage operations of the trie (commit, snapshot, checkpoint, pruning) @@ -336,7 +337,7 @@ func (tsm *trieStorageManager) takeSnapshot(snapshotEntry *snapshotsQueueEntry, } stats := statistics.NewTrieStatistics() - err = newRoot.commitSnapshot(stsm, foundInEpoch, snapshotEntry.iteratorChannels.LeavesChan, snapshotEntry.missingNodesChan, ctx, stats, tsm.idleProvider, encodedRoot, rootDepthLevel) + err = newRoot.commitSnapshot(stsm, foundInEpoch, snapshotEntry.iteratorChannels.LeavesChan, snapshotEntry.missingNodesChan, ctx, stats, tsm.idleProvider, encodedRoot, trieMetricsCollector.NewTrieMetricsCollector()) if err != nil { snapshotEntry.iteratorChannels.ErrChan.WriteInChanNonBlocking(err) treatSnapshotError(err, diff --git a/trie/trieStorageManagerInEpoch_test.go b/trie/trieStorageManagerInEpoch_test.go index bedf3734529..ca0717242d2 100644 --- a/trie/trieStorageManagerInEpoch_test.go +++ b/trie/trieStorageManagerInEpoch_test.go @@ -7,8 +7,8 @@ import ( "github.com/multiversx/mx-chain-go/storage" "github.com/multiversx/mx-chain-go/testscommon" + testCommon "github.com/multiversx/mx-chain-go/testscommon/storage" "github.com/multiversx/mx-chain-go/testscommon/storageManager" - "github.com/multiversx/mx-chain-go/testscommon/trie" "github.com/stretchr/testify/assert" ) @@ -72,7 +72,7 @@ func TestTrieStorageManagerInEpoch_GetFromEpoch(t *testing.T) { _, trieStorage := newEmptyTrie() getFromEpochCalled := false - trieStorage.mainStorer = &trie.SnapshotPruningStorerStub{ + trieStorage.mainStorer = &testCommon.SnapshotPruningStorerStub{ GetFromEpochCalled: func(_ []byte, _ uint32) ([]byte, error) { getFromEpochCalled = true return nil, nil @@ -89,7 +89,7 @@ func TestTrieStorageManagerInEpoch_GetFromEpoch(t *testing.T) { _, trieStorage := newEmptyTrie() getFromEpochCalled := false - trieStorage.mainStorer = &trie.SnapshotPruningStorerStub{ + trieStorage.mainStorer = &testCommon.SnapshotPruningStorerStub{ GetFromEpochCalled: func(_ []byte, _ uint32) ([]byte, error) { getFromEpochCalled = true return nil, storage.ErrDBIsClosed @@ -107,7 +107,7 @@ func TestTrieStorageManagerInEpoch_GetFromEpoch(t *testing.T) { _, trieStorage := newEmptyTrie() getFromEpochCalled := false - trieStorage.mainStorer = &trie.SnapshotPruningStorerStub{ + trieStorage.mainStorer = &testCommon.SnapshotPruningStorerStub{ GetFromEpochCalled: func(_ []byte, _ uint32) ([]byte, error) { getFromEpochCalled = true return nil, errors.New("not closing error") @@ -128,7 +128,7 @@ func TestTrieStorageManagerInEpoch_GetFromEpoch(t *testing.T) { getFromPreviousEpochCalled := false currentEpoch := uint32(5) expectedKey := []byte("key") - trieStorage.mainStorer = &trie.SnapshotPruningStorerStub{ + trieStorage.mainStorer = &testCommon.SnapshotPruningStorerStub{ GetFromEpochCalled: func(key []byte, epoch uint32) ([]byte, error) { assert.Equal(t, expectedKey, key) if epoch == currentEpoch { diff --git a/trie/trieStorageManager_test.go b/trie/trieStorageManager_test.go index 104c6e2578c..88c9bb7888a 100644 --- a/trie/trieStorageManager_test.go +++ b/trie/trieStorageManager_test.go @@ -207,7 +207,7 @@ func TestTrieStorageManager_RemoveFromAllActiveEpochs(t *testing.T) { RemoveFromAllActiveEpochsCalled := false args := trie.GetDefaultTrieStorageManagerParameters() - args.MainStorer = &trieMock.SnapshotPruningStorerStub{ + args.MainStorer = &storage.SnapshotPruningStorerStub{ MemDbMock: testscommon.NewMemDbMock(), RemoveFromAllActiveEpochsCalled: func(key []byte) error { RemoveFromAllActiveEpochsCalled = true @@ -237,7 +237,7 @@ func TestTrieStorageManager_PutInEpoch(t *testing.T) { putInEpochCalled := false args := trie.GetDefaultTrieStorageManagerParameters() - args.MainStorer = &trieMock.SnapshotPruningStorerStub{ + args.MainStorer = &storage.SnapshotPruningStorerStub{ MemDbMock: testscommon.NewMemDbMock(), PutInEpochCalled: func(key []byte, data []byte, epoch uint32) error { putInEpochCalled = true @@ -268,7 +268,7 @@ func TestTrieStorageManager_GetLatestStorageEpoch(t *testing.T) { getLatestSorageCalled := false args := trie.GetDefaultTrieStorageManagerParameters() - args.MainStorer = &trieMock.SnapshotPruningStorerStub{ + args.MainStorer = &storage.SnapshotPruningStorerStub{ MemDbMock: testscommon.NewMemDbMock(), GetLatestStorageEpochCalled: func() (uint32, error) { getLatestSorageCalled = true @@ -384,7 +384,7 @@ func TestTrieStorageManager_ShouldTakeSnapshot(t *testing.T) { t.Parallel() args := trie.GetDefaultTrieStorageManagerParameters() - args.MainStorer = &trieMock.SnapshotPruningStorerStub{ + args.MainStorer = &storage.SnapshotPruningStorerStub{ GetFromCurrentEpochCalled: func(key []byte) ([]byte, error) { return []byte(common.TrieSyncedVal), nil }, @@ -398,7 +398,7 @@ func TestTrieStorageManager_ShouldTakeSnapshot(t *testing.T) { t.Parallel() args := trie.GetDefaultTrieStorageManagerParameters() - args.MainStorer = &trieMock.SnapshotPruningStorerStub{ + args.MainStorer = &storage.SnapshotPruningStorerStub{ GetFromCurrentEpochCalled: func(key []byte) ([]byte, error) { return []byte("invalid marker"), nil }, @@ -412,7 +412,7 @@ func TestTrieStorageManager_ShouldTakeSnapshot(t *testing.T) { t.Parallel() args := trie.GetDefaultTrieStorageManagerParameters() - args.MainStorer = &trieMock.SnapshotPruningStorerStub{ + args.MainStorer = &storage.SnapshotPruningStorerStub{ GetFromCurrentEpochCalled: func(key []byte) ([]byte, error) { return nil, expectedErr // isTrieSynced returns false }, @@ -477,7 +477,7 @@ func TestNewSnapshotTrieStorageManager_GetFromCurrentEpoch(t *testing.T) { getFromCurrentEpochCalled := false args := trie.GetDefaultTrieStorageManagerParameters() - args.MainStorer = &trieMock.SnapshotPruningStorerStub{ + args.MainStorer = &storage.SnapshotPruningStorerStub{ MemDbMock: testscommon.NewMemDbMock(), GetFromCurrentEpochCalled: func(_ []byte) ([]byte, error) { getFromCurrentEpochCalled = true diff --git a/update/factory/accountDBSyncerContainerFactory.go b/update/factory/accountDBSyncerContainerFactory.go index 58684996dca..aaa6150f673 100644 --- a/update/factory/accountDBSyncerContainerFactory.go +++ b/update/factory/accountDBSyncerContainerFactory.go @@ -29,7 +29,6 @@ type ArgsNewAccountsDBSyncersContainerFactory struct { Marshalizer marshal.Marshalizer TrieStorageManager common.StorageManager TimoutGettingTrieNode time.Duration - MaxTrieLevelInMemory uint NumConcurrentTrieSyncers int MaxHardCapForMissingNodes int TrieSyncerVersion int @@ -47,7 +46,6 @@ type accountDBSyncersContainerFactory struct { marshalizer marshal.Marshalizer timeoutGettingTrieNode time.Duration trieStorageManager common.StorageManager - maxTrieLevelinMemory uint numConcurrentTrieSyncers int maxHardCapForMissingNodes int trieSyncerVersion int @@ -101,7 +99,6 @@ func NewAccountsDBSContainerFactory(args ArgsNewAccountsDBSyncersContainerFactor marshalizer: args.Marshalizer, trieStorageManager: args.TrieStorageManager, timeoutGettingTrieNode: args.TimoutGettingTrieNode, - maxTrieLevelinMemory: args.MaxTrieLevelInMemory, numConcurrentTrieSyncers: args.NumConcurrentTrieSyncers, maxHardCapForMissingNodes: args.MaxHardCapForMissingNodes, trieSyncerVersion: args.TrieSyncerVersion, @@ -151,7 +148,6 @@ func (a *accountDBSyncersContainerFactory) createUserAccountsSyncer(shardId uint RequestHandler: a.requestHandler, Timeout: a.timeoutGettingTrieNode, Cacher: a.trieCacher, - MaxTrieLevelInMemory: a.maxTrieLevelinMemory, MaxHardCapForMissingNodes: a.maxHardCapForMissingNodes, TrieSyncerVersion: a.trieSyncerVersion, CheckNodesOnDisk: a.checkNodesOnDisk, @@ -181,7 +177,6 @@ func (a *accountDBSyncersContainerFactory) createValidatorAccountsSyncer(shardId RequestHandler: a.requestHandler, Timeout: a.timeoutGettingTrieNode, Cacher: a.trieCacher, - MaxTrieLevelInMemory: a.maxTrieLevelinMemory, MaxHardCapForMissingNodes: a.maxHardCapForMissingNodes, TrieSyncerVersion: a.trieSyncerVersion, CheckNodesOnDisk: a.checkNodesOnDisk, diff --git a/update/factory/dataTrieFactory.go b/update/factory/dataTrieFactory.go index 10483099780..78e66f5a651 100644 --- a/update/factory/dataTrieFactory.go +++ b/update/factory/dataTrieFactory.go @@ -13,7 +13,7 @@ import ( "github.com/multiversx/mx-chain-go/config" "github.com/multiversx/mx-chain-go/dataRetriever" "github.com/multiversx/mx-chain-go/sharding" - "github.com/multiversx/mx-chain-go/state" + "github.com/multiversx/mx-chain-go/state/triesHolder" storageFactory "github.com/multiversx/mx-chain-go/storage/factory" "github.com/multiversx/mx-chain-go/storage/storageunit" "github.com/multiversx/mx-chain-go/trie" @@ -21,25 +21,25 @@ import ( "github.com/multiversx/mx-chain-go/update/genesis" ) +const tenMBSize = uint64(10485760) + // ArgsNewDataTrieFactory is the argument structure for the new data trie factory type ArgsNewDataTrieFactory struct { - StorageConfig config.StorageConfig - SyncFolder string - Marshalizer marshal.Marshalizer - Hasher hashing.Hasher - ShardCoordinator sharding.Coordinator - EnableEpochsHandler common.EnableEpochsHandler - StateStatsCollector common.StateStatisticsHandler - MaxTrieLevelInMemory uint + StorageConfig config.StorageConfig + SyncFolder string + Marshalizer marshal.Marshalizer + Hasher hashing.Hasher + ShardCoordinator sharding.Coordinator + EnableEpochsHandler common.EnableEpochsHandler + StateStatsCollector common.StateStatisticsHandler } type dataTrieFactory struct { - shardCoordinator sharding.Coordinator - trieStorage common.StorageManager - marshalizer marshal.Marshalizer - hasher hashing.Hasher - enableEpochsHandler common.EnableEpochsHandler - maxTrieLevelInMemory uint + shardCoordinator sharding.Coordinator + trieStorage common.StorageManager + marshalizer marshal.Marshalizer + hasher hashing.Hasher + enableEpochsHandler common.EnableEpochsHandler } // NewDataTrieFactory creates a data trie factory @@ -100,12 +100,11 @@ func NewDataTrieFactory(args ArgsNewDataTrieFactory) (*dataTrieFactory, error) { } d := &dataTrieFactory{ - shardCoordinator: args.ShardCoordinator, - trieStorage: trieStorage, - marshalizer: args.Marshalizer, - hasher: args.Hasher, - maxTrieLevelInMemory: args.MaxTrieLevelInMemory, - enableEpochsHandler: args.EnableEpochsHandler, + shardCoordinator: args.ShardCoordinator, + trieStorage: trieStorage, + marshalizer: args.Marshalizer, + hasher: args.Hasher, + enableEpochsHandler: args.EnableEpochsHandler, } return d, nil @@ -118,7 +117,7 @@ func (d *dataTrieFactory) TrieStorageManager() common.StorageManager { // Create creates a TriesHolder container to hold all the states func (d *dataTrieFactory) Create() (common.TriesHolder, error) { - container := state.NewDataTriesHolder() + container := triesHolder.NewTriesHolder() for i := uint32(0); i < d.shardCoordinator.NumberOfShards(); i++ { err := d.createAndAddOneTrie(i, genesis.UserAccount, container) @@ -141,7 +140,7 @@ func (d *dataTrieFactory) Create() (common.TriesHolder, error) { } func (d *dataTrieFactory) createAndAddOneTrie(shId uint32, accType genesis.Type, container common.TriesHolder) error { - dataTrie, err := trie.NewTrie(d.trieStorage, d.marshalizer, d.hasher, d.enableEpochsHandler, d.maxTrieLevelInMemory) + dataTrie, err := trie.NewTrie(d.trieStorage, d.marshalizer, d.hasher, d.enableEpochsHandler, tenMBSize) if err != nil { return err } diff --git a/update/factory/exportHandlerFactory.go b/update/factory/exportHandlerFactory.go index 44bb0904c86..1bce5c5a6b6 100644 --- a/update/factory/exportHandlerFactory.go +++ b/update/factory/exportHandlerFactory.go @@ -54,7 +54,6 @@ type ArgsExporter struct { ExportTriesStorageConfig config.StorageConfig ExportStateStorageConfig config.StorageConfig ExportStateKeysConfig config.StorageConfig - MaxTrieLevelInMemory uint WhiteListHandler process.WhiteListHandler WhiteListerVerifiedTxs process.WhiteListHandler MainInterceptorsContainer process.InterceptorsContainer @@ -88,7 +87,6 @@ type exportHandlerFactory struct { exportTriesStorageConfig config.StorageConfig exportStateStorageConfig config.StorageConfig exportStateKeysConfig config.StorageConfig - maxTrieLevelInMemory uint whiteListHandler process.WhiteListHandler whiteListerVerifiedTxs process.WhiteListHandler mainInterceptorsContainer process.InterceptorsContainer @@ -260,7 +258,6 @@ func NewExportHandlerFactory(args ArgsExporter) (*exportHandlerFactory, error) { headerSigVerifier: args.HeaderSigVerifier, headerIntegrityVerifier: args.HeaderIntegrityVerifier, validityAttester: args.ValidityAttester, - maxTrieLevelInMemory: args.MaxTrieLevelInMemory, roundHandler: args.RoundHandler, maxHardCapForMissingNodes: args.MaxHardCapForMissingNodes, numConcurrentTrieSyncers: args.NumConcurrentTrieSyncers, @@ -321,14 +318,13 @@ func (e *exportHandlerFactory) Create() (update.ExportHandler, error) { } argsDataTrieFactory := ArgsNewDataTrieFactory{ - StorageConfig: e.exportTriesStorageConfig, - SyncFolder: e.exportFolder, - Marshalizer: e.coreComponents.InternalMarshalizer(), - Hasher: e.coreComponents.Hasher(), - ShardCoordinator: e.shardCoordinator, - MaxTrieLevelInMemory: e.maxTrieLevelInMemory, - EnableEpochsHandler: e.coreComponents.EnableEpochsHandler(), - StateStatsCollector: e.statusCoreComponents.StateStatsHandler(), + StorageConfig: e.exportTriesStorageConfig, + SyncFolder: e.exportFolder, + Marshalizer: e.coreComponents.InternalMarshalizer(), + Hasher: e.coreComponents.Hasher(), + ShardCoordinator: e.shardCoordinator, + EnableEpochsHandler: e.coreComponents.EnableEpochsHandler(), + StateStatsCollector: e.statusCoreComponents.StateStatsHandler(), } dataTriesContainerFactory, err := NewDataTrieFactory(argsDataTrieFactory) if err != nil { @@ -413,7 +409,6 @@ func (e *exportHandlerFactory) Create() (update.ExportHandler, error) { Marshalizer: e.coreComponents.InternalMarshalizer(), TrieStorageManager: trieStorageManager, TimoutGettingTrieNode: common.TimeoutGettingTrieNodesInHardfork, - MaxTrieLevelInMemory: e.maxTrieLevelInMemory, MaxHardCapForMissingNodes: e.maxHardCapForMissingNodes, NumConcurrentTrieSyncers: e.numConcurrentTrieSyncers, TrieSyncerVersion: e.trieSyncerVersion, diff --git a/update/genesis/import.go b/update/genesis/import.go index 8e59e45b7f4..dd1984081af 100644 --- a/update/genesis/import.go +++ b/update/genesis/import.go @@ -25,9 +25,9 @@ import ( "github.com/multiversx/mx-chain-go/update" ) -var _ update.ImportHandler = (*stateImport)(nil) +const tenMBSize = uint64(10485760) -const maxTrieLevelInMemory = uint(5) +var _ update.ImportHandler = (*stateImport)(nil) // ArgsNewStateImport is the arguments structure to create a new state importer type ArgsNewStateImport struct { @@ -315,7 +315,7 @@ func (si *stateImport) getTrie(shardID uint32, accType Type) (common.Trie, error trieStorageManager = si.trieStorageManagers[dataRetriever.PeerAccountsUnit.String()] } - trieForShard, err := trie.NewTrie(trieStorageManager, si.marshalizer, si.hasher, si.enableEpochsHandler, maxTrieLevelInMemory) + trieForShard, err := trie.NewTrie(trieStorageManager, si.marshalizer, si.hasher, si.enableEpochsHandler, tenMBSize) if err != nil { return nil, err } @@ -347,7 +347,7 @@ func (si *stateImport) importDataTrie(identifier string, shID uint32, keys [][]b return fmt.Errorf("%w wanted a roothash", update.ErrWrongTypeAssertion) } - dataTrie, err := trie.NewTrie(si.trieStorageManagers[dataRetriever.UserAccountsUnit.String()], si.marshalizer, si.hasher, si.enableEpochsHandler, maxTrieLevelInMemory) + dataTrie, err := trie.NewTrie(si.trieStorageManagers[dataRetriever.UserAccountsUnit.String()], si.marshalizer, si.hasher, si.enableEpochsHandler, tenMBSize) if err != nil { return err } @@ -414,14 +414,15 @@ func (si *stateImport) getAccountsDB(accType Type, shardID uint32, accountFactor if accType == ValidatorAccount { if check.IfNil(si.validatorDB) { argsAccountDB := state.ArgsAccountsDB{ - Trie: currentTrie, - Hasher: si.hasher, - Marshaller: si.marshalizer, - AccountFactory: accountFactory, - StoragePruningManager: disabled.NewDisabledStoragePruningManager(), - AddressConverter: si.addressConverter, - SnapshotsManager: disabledState.NewDisabledSnapshotsManager(), - StateAccessesCollector: disabledState.NewDisabledStateAccessesCollector(), + Trie: currentTrie, + Hasher: si.hasher, + Marshaller: si.marshalizer, + AccountFactory: accountFactory, + StoragePruningManager: disabled.NewDisabledStoragePruningManager(), + AddressConverter: si.addressConverter, + SnapshotsManager: disabledState.NewDisabledSnapshotsManager(), + StateAccessesCollector: disabledState.NewDisabledStateAccessesCollector(), + MaxDataTriesSizeInMemory: tenMBSize, } accountsDB, errCreate := state.NewAccountsDB(argsAccountDB) if errCreate != nil { @@ -438,14 +439,15 @@ func (si *stateImport) getAccountsDB(accType Type, shardID uint32, accountFactor } argsAccountDB := state.ArgsAccountsDB{ - Trie: currentTrie, - Hasher: si.hasher, - Marshaller: si.marshalizer, - AccountFactory: accountFactory, - StoragePruningManager: disabled.NewDisabledStoragePruningManager(), - AddressConverter: si.addressConverter, - SnapshotsManager: disabledState.NewDisabledSnapshotsManager(), - StateAccessesCollector: disabledState.NewDisabledStateAccessesCollector(), + Trie: currentTrie, + Hasher: si.hasher, + Marshaller: si.marshalizer, + AccountFactory: accountFactory, + StoragePruningManager: disabled.NewDisabledStoragePruningManager(), + AddressConverter: si.addressConverter, + SnapshotsManager: disabledState.NewDisabledSnapshotsManager(), + StateAccessesCollector: disabledState.NewDisabledStateAccessesCollector(), + MaxDataTriesSizeInMemory: tenMBSize, } accountsDB, err = state.NewAccountsDB(argsAccountDB) si.accountDBsMap[shardID] = accountsDB