From f361f90e1ba348085bd170911410d1e0363665fe Mon Sep 17 00:00:00 2001 From: frank Date: Wed, 23 Apr 2025 15:21:40 +0800 Subject: [PATCH 1/4] feat_: implement low memory mode with garbage collection logging --- mobile/status.go | 60 ++++++++++++++++++++++++++++++++++++ mobile/status_test.go | 71 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 131 insertions(+) diff --git a/mobile/status.go b/mobile/status.go index 75ab3807451..4cf5a69f224 100644 --- a/mobile/status.go +++ b/mobile/status.go @@ -8,6 +8,8 @@ import ( "os" "path" "runtime" + "runtime/debug" + "sync" "time" "unsafe" @@ -57,6 +59,15 @@ import ( "github.com/status-im/status-go/signal" ) +var ( + lastGCTime time.Time + lastGCTimeLock sync.Mutex +) + +const ( + gcInterval = 10 * time.Second +) + func call(fn any, params ...any) any { return callog.Call(logutils.ZapLogger(), requestlog.GetRequestLogger(), fn, params...) } @@ -2429,3 +2440,52 @@ func IntendedPanic(message string) string { panic(err) }) } + +func SwitchToLowMemoryMode() string { + return callWithResponse(switchToLowMemoryMode) +} + +func switchToLowMemoryMode() string { + lastGCTimeLock.Lock() + defer lastGCTimeLock.Unlock() + + var err error + // Only run GC if 10 seconds have passed since the last call + if time.Since(lastGCTime) >= gcInterval { + releaseMemory() + lastGCTime = time.Now() + } else { + err = errors.New("skipping GC because it was called too recently") + } + + return makeJSONResponse(err) +} + +func releaseMemory() { + logger := logutils.ZapLogger() + var before, after runtime.MemStats + + // Collect stats before GC + runtime.ReadMemStats(&before) + logger.Info("Before garbage collection", + zap.Uint64("heap_alloc_mb", before.HeapAlloc/1024/1024), + zap.Uint64("heap_objects", before.HeapObjects), + zap.Uint64("sys_mb", before.Sys/1024/1024), + ) + + runtime.GC() + debug.FreeOSMemory() + + // Collect stats after GC + runtime.ReadMemStats(&after) + + // Log results with differences + logger.Info("After garbage collection", + zap.Uint64("heap_alloc_mb", after.HeapAlloc/1024/1024), + zap.Uint64("heap_objects", after.HeapObjects), + zap.Uint64("sys_mb", after.Sys/1024/1024), + zap.Int64("freed_mb", int64(before.HeapAlloc-after.HeapAlloc)/1024/1024), + zap.Int64("objects_freed", int64(before.HeapObjects-after.HeapObjects)), + zap.Int64("released_mb", int64(after.HeapReleased-before.HeapReleased)/1024/1024), + ) +} diff --git a/mobile/status_test.go b/mobile/status_test.go index d18a24313b3..d580969f966 100644 --- a/mobile/status_test.go +++ b/mobile/status_test.go @@ -2,7 +2,9 @@ package statusgo import ( "encoding/json" + "sync" "testing" + "time" "github.com/stretchr/testify/require" @@ -47,3 +49,72 @@ func TestIntendedPanic(t *testing.T) { IntendedPanic(message) }) } + +func TestSwitchToLowMemoryMode(t *testing.T) { + // Reset the lastGCTime to ensure we can trigger GC + lastGCTimeLock.Lock() + lastGCTime = time.Time{} + lastGCTimeLock.Unlock() + + // First call should succeed + result := SwitchToLowMemoryMode() + + var response APIResponse + err := json.Unmarshal([]byte(result), &response) + if err != nil { + t.Fatalf("Failed to unmarshal response: %v", err) + } + + if response.Error != "" { + t.Errorf("Expected no error on first call, got: %s", response.Error) + } + + // Second immediate call should be skipped due to rate limiting + result = SwitchToLowMemoryMode() + + err = json.Unmarshal([]byte(result), &response) + if err != nil { + t.Fatalf("Failed to unmarshal response: %v", err) + } + + if response.Error == "" { + t.Error("Expected error on second immediate call, but got none") + } + + // The error should indicate that GC was skipped + if response.Error != "skipping GC because it was called too recently" { + t.Errorf("Unexpected error message: %s", response.Error) + } + + // Test concurrent access + var wg sync.WaitGroup + for i := 0; i < 5; i++ { + wg.Add(1) + go func() { + defer wg.Done() + SwitchToLowMemoryMode() + }() + } + wg.Wait() + + // After waiting, we should be able to call it again + lastGCTimeLock.Lock() + lastGCTime = time.Now().Add(-(gcInterval + time.Second)) + lastGCTimeLock.Unlock() + + result = SwitchToLowMemoryMode() + + err = json.Unmarshal([]byte(result), &response) + if err != nil { + t.Fatalf("Failed to unmarshal response: %v", err) + } + + if response.Error != "" { + t.Errorf("Expected no error after waiting, got: %s", response.Error) + } +} + +// TestReleaseMemory is a simple test to ensure the function doesn't panic +func TestReleaseMemory(t *testing.T) { + releaseMemory() +} From 5d212011fe46b581795e66c853447d8c9cf25961 Mon Sep 17 00:00:00 2001 From: frank Date: Mon, 19 May 2025 19:25:23 +0800 Subject: [PATCH 2/4] chore_: move global var into geth backend --- api/backend_test.go | 14 ++++++++ api/geth_backend.go | 52 ++++++++++++++++++++++++++++++ mobile/status.go | 59 +++------------------------------- mobile/status_test.go | 74 +++---------------------------------------- 4 files changed, 75 insertions(+), 124 deletions(-) diff --git a/api/backend_test.go b/api/backend_test.go index 3bcfbce75c6..ea1903b8daf 100644 --- a/api/backend_test.go +++ b/api/backend_test.go @@ -1926,3 +1926,17 @@ func TestRestoreKeycardAccountAndLogin(t *testing.T) { require.NoError(t, err) require.NotNil(t, acc) } + +func TestReleaseOSMemory(t *testing.T) { + b := NewGethStatusBackend(tt.MustCreateTestLogger()) + + // the first call should succeed + require.NoError(t, b.ReleaseOSMemory()) + + // the second immediate call should be skipped due to rate limiting + require.Error(t, b.ReleaseOSMemory()) + + // reset the lastGCTime to allow the next call + b.lastGCTime = time.Now().Add(-(gcInterval + time.Second)) + require.NoError(t, b.ReleaseOSMemory()) +} diff --git a/api/geth_backend.go b/api/geth_backend.go index 37ddfcfd545..d94e39a3d0a 100644 --- a/api/geth_backend.go +++ b/api/geth_backend.go @@ -9,6 +9,8 @@ import ( "math/big" "os" "path/filepath" + "runtime" + "runtime/debug" "strings" "sync" "time" @@ -66,6 +68,10 @@ import ( "github.com/status-im/status-go/walletdatabase" ) +const ( + gcInterval = 10 * time.Second +) + var ( // ErrWhisperClearIdentitiesFailure clearing whisper identities has failed. ErrWhisperClearIdentitiesFailure = errors.New("failed to clear whisper identities") @@ -110,6 +116,9 @@ type GethStatusBackend struct { logger *zap.Logger preLoginLogConfig *logutils.PreLoginLogConfig + + lastGCTime time.Time + lastGCTimeLock sync.Mutex } // NewGethStatusBackend create a new GethStatusBackend instance @@ -3036,3 +3045,46 @@ func (b *GethStatusBackend) SetPreLoginLogLevel(level string) error { } return logutils.OverrideRootLoggerWithConfig(b.preLoginLogConfig.ConvertToLogSettings()) } + +func (b *GethStatusBackend) ReleaseOSMemory() error { + b.lastGCTimeLock.Lock() + defer b.lastGCTimeLock.Unlock() + + var err error + // Only run GC if 10 seconds have passed since the last call + if time.Since(b.lastGCTime) >= gcInterval { + b.releaseOSMemory() + b.lastGCTime = time.Now() + } else { + err = errors.New("skipping GC because it was called too recently") + } + return err +} + +func (b *GethStatusBackend) releaseOSMemory() { + var before, after runtime.MemStats + + // Collect stats before GC + runtime.ReadMemStats(&before) + b.logger.Info("Before garbage collection", + zap.Uint64("heap_alloc_mb", before.HeapAlloc/1024/1024), + zap.Uint64("heap_objects", before.HeapObjects), + zap.Uint64("sys_mb", before.Sys/1024/1024), + ) + + runtime.GC() + debug.FreeOSMemory() + + // Collect stats after GC + runtime.ReadMemStats(&after) + + // Log results with differences + b.logger.Info("After garbage collection", + zap.Uint64("heap_alloc_mb", after.HeapAlloc/1024/1024), + zap.Uint64("heap_objects", after.HeapObjects), + zap.Uint64("sys_mb", after.Sys/1024/1024), + zap.Int64("freed_mb", int64(before.HeapAlloc-after.HeapAlloc)/1024/1024), + zap.Int64("objects_freed", int64(before.HeapObjects-after.HeapObjects)), + zap.Int64("released_mb", int64(after.HeapReleased-before.HeapReleased)/1024/1024), + ) +} diff --git a/mobile/status.go b/mobile/status.go index 4cf5a69f224..30512e974e3 100644 --- a/mobile/status.go +++ b/mobile/status.go @@ -8,8 +8,6 @@ import ( "os" "path" "runtime" - "runtime/debug" - "sync" "time" "unsafe" @@ -59,15 +57,6 @@ import ( "github.com/status-im/status-go/signal" ) -var ( - lastGCTime time.Time - lastGCTimeLock sync.Mutex -) - -const ( - gcInterval = 10 * time.Second -) - func call(fn any, params ...any) any { return callog.Call(logutils.ZapLogger(), requestlog.GetRequestLogger(), fn, params...) } @@ -2441,51 +2430,11 @@ func IntendedPanic(message string) string { }) } -func SwitchToLowMemoryMode() string { - return callWithResponse(switchToLowMemoryMode) +func ReleaseOSMemory() string { + return callWithResponse(releaseOSMemory) } -func switchToLowMemoryMode() string { - lastGCTimeLock.Lock() - defer lastGCTimeLock.Unlock() - - var err error - // Only run GC if 10 seconds have passed since the last call - if time.Since(lastGCTime) >= gcInterval { - releaseMemory() - lastGCTime = time.Now() - } else { - err = errors.New("skipping GC because it was called too recently") - } - +func releaseOSMemory() string { + err := statusBackend.ReleaseOSMemory() return makeJSONResponse(err) } - -func releaseMemory() { - logger := logutils.ZapLogger() - var before, after runtime.MemStats - - // Collect stats before GC - runtime.ReadMemStats(&before) - logger.Info("Before garbage collection", - zap.Uint64("heap_alloc_mb", before.HeapAlloc/1024/1024), - zap.Uint64("heap_objects", before.HeapObjects), - zap.Uint64("sys_mb", before.Sys/1024/1024), - ) - - runtime.GC() - debug.FreeOSMemory() - - // Collect stats after GC - runtime.ReadMemStats(&after) - - // Log results with differences - logger.Info("After garbage collection", - zap.Uint64("heap_alloc_mb", after.HeapAlloc/1024/1024), - zap.Uint64("heap_objects", after.HeapObjects), - zap.Uint64("sys_mb", after.Sys/1024/1024), - zap.Int64("freed_mb", int64(before.HeapAlloc-after.HeapAlloc)/1024/1024), - zap.Int64("objects_freed", int64(before.HeapObjects-after.HeapObjects)), - zap.Int64("released_mb", int64(after.HeapReleased-before.HeapReleased)/1024/1024), - ) -} diff --git a/mobile/status_test.go b/mobile/status_test.go index d580969f966..5da47368883 100644 --- a/mobile/status_test.go +++ b/mobile/status_test.go @@ -2,9 +2,7 @@ package statusgo import ( "encoding/json" - "sync" "testing" - "time" "github.com/stretchr/testify/require" @@ -50,71 +48,9 @@ func TestIntendedPanic(t *testing.T) { }) } -func TestSwitchToLowMemoryMode(t *testing.T) { - // Reset the lastGCTime to ensure we can trigger GC - lastGCTimeLock.Lock() - lastGCTime = time.Time{} - lastGCTimeLock.Unlock() - - // First call should succeed - result := SwitchToLowMemoryMode() - - var response APIResponse - err := json.Unmarshal([]byte(result), &response) - if err != nil { - t.Fatalf("Failed to unmarshal response: %v", err) - } - - if response.Error != "" { - t.Errorf("Expected no error on first call, got: %s", response.Error) - } - - // Second immediate call should be skipped due to rate limiting - result = SwitchToLowMemoryMode() - - err = json.Unmarshal([]byte(result), &response) - if err != nil { - t.Fatalf("Failed to unmarshal response: %v", err) - } - - if response.Error == "" { - t.Error("Expected error on second immediate call, but got none") - } - - // The error should indicate that GC was skipped - if response.Error != "skipping GC because it was called too recently" { - t.Errorf("Unexpected error message: %s", response.Error) - } - - // Test concurrent access - var wg sync.WaitGroup - for i := 0; i < 5; i++ { - wg.Add(1) - go func() { - defer wg.Done() - SwitchToLowMemoryMode() - }() - } - wg.Wait() - - // After waiting, we should be able to call it again - lastGCTimeLock.Lock() - lastGCTime = time.Now().Add(-(gcInterval + time.Second)) - lastGCTimeLock.Unlock() - - result = SwitchToLowMemoryMode() - - err = json.Unmarshal([]byte(result), &response) - if err != nil { - t.Fatalf("Failed to unmarshal response: %v", err) - } - - if response.Error != "" { - t.Errorf("Expected no error after waiting, got: %s", response.Error) - } -} - -// TestReleaseMemory is a simple test to ensure the function doesn't panic -func TestReleaseMemory(t *testing.T) { - releaseMemory() +// TestReleaseOSMemory is a simple test to ensure the function doesn't panic +func TestReleaseOSMemory(t *testing.T) { + require.NotPanics(t, func() { + ReleaseOSMemory() + }) } From 842ab42879f4b5a3089c386c659fb42dee1aa8b1 Mon Sep 17 00:00:00 2001 From: frank Date: Mon, 19 May 2025 19:33:02 +0800 Subject: [PATCH 3/4] chore_: change Info to Debug --- api/geth_backend.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/geth_backend.go b/api/geth_backend.go index d94e39a3d0a..d9d65c30a86 100644 --- a/api/geth_backend.go +++ b/api/geth_backend.go @@ -3066,7 +3066,7 @@ func (b *GethStatusBackend) releaseOSMemory() { // Collect stats before GC runtime.ReadMemStats(&before) - b.logger.Info("Before garbage collection", + b.logger.Debug("Before garbage collection", zap.Uint64("heap_alloc_mb", before.HeapAlloc/1024/1024), zap.Uint64("heap_objects", before.HeapObjects), zap.Uint64("sys_mb", before.Sys/1024/1024), @@ -3079,7 +3079,7 @@ func (b *GethStatusBackend) releaseOSMemory() { runtime.ReadMemStats(&after) // Log results with differences - b.logger.Info("After garbage collection", + b.logger.Debug("After garbage collection", zap.Uint64("heap_alloc_mb", after.HeapAlloc/1024/1024), zap.Uint64("heap_objects", after.HeapObjects), zap.Uint64("sys_mb", after.Sys/1024/1024), From e174de44cdda9edc5aa665321bfae0a2d0cd1b47 Mon Sep 17 00:00:00 2001 From: frank Date: Mon, 19 May 2025 19:47:05 +0800 Subject: [PATCH 4/4] chore_: remove code branching --- api/geth_backend.go | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/api/geth_backend.go b/api/geth_backend.go index d9d65c30a86..f321988e297 100644 --- a/api/geth_backend.go +++ b/api/geth_backend.go @@ -3050,15 +3050,13 @@ func (b *GethStatusBackend) ReleaseOSMemory() error { b.lastGCTimeLock.Lock() defer b.lastGCTimeLock.Unlock() - var err error // Only run GC if 10 seconds have passed since the last call - if time.Since(b.lastGCTime) >= gcInterval { - b.releaseOSMemory() - b.lastGCTime = time.Now() - } else { - err = errors.New("skipping GC because it was called too recently") + if time.Since(b.lastGCTime) < gcInterval { + return errors.New("skipping GC because it was called too recently") } - return err + b.releaseOSMemory() + b.lastGCTime = time.Now() + return nil } func (b *GethStatusBackend) releaseOSMemory() {