From 4498f7bfe676e81c2d0a90567daefb3ff932d603 Mon Sep 17 00:00:00 2001 From: John Floren Date: Wed, 18 Aug 2021 13:31:14 -0700 Subject: [PATCH] Implement a fix for #66, excessive memory use in siphon. The siphon will now stop writing to its internal buffer once the size of the buffer exceeds the maximum cache size. Because we write until we *exceed* the max cache size, we're safe to attempt the cache update even if the buffer only contains partial data, because it's still over the limit & will be rejected. --- diskv.go | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/diskv.go b/diskv.go index 9f07b85..7df8257 100644 --- a/diskv.go +++ b/diskv.go @@ -466,15 +466,22 @@ func (s *siphon) Read(p []byte) (int, error) { n, err := s.f.Read(p) if err == nil { - return s.buf.Write(p[0:n]) // Write must succeed for Read to succeed - } - - if err == io.EOF { - s.d.cacheWithoutLock(s.key, s.buf.Bytes()) // cache may fail + // Only write into the buffer if the buffer is not yet over the size of the cache. + // This logic guarantees that we'll write into the buffer until one of two things happens: + // 1. We write the entire contents of the source into the buffer. + // 2. We've read enough into the buffer to exceed the max cache size. + // If we read more than the max cache size into the buffer, our later attempt + // to update the cache will be rejected (because the value exceeds the max cache size); + // it does *not* cache a partial value. + if uint64(s.buf.Len()) < s.d.CacheSizeMax { + return s.buf.Write(p[0:n]) // Write must succeed for Read to succeed + } + } else if err == io.EOF { + // The cache will reject this if we've exceeded the maximum cache size + s.d.cacheWithoutLock(s.key, s.buf.Bytes()) if closeErr := s.f.Close(); closeErr != nil { return n, closeErr // close must succeed for Read to succeed } - return n, err } return n, err