From a2affd7d911244da3e6b68eb191fbac7096d8faf Mon Sep 17 00:00:00 2001 From: Fangxin Lou Date: Tue, 18 Feb 2025 14:42:46 +0800 Subject: [PATCH 1/2] add multi block option for random read ahead --- cmd/flags.go | 4 ++++ cmd/mount.go | 1 + pkg/chunk/cached_store.go | 1 + pkg/vfs/reader.go | 12 +++++++++++- 4 files changed, 17 insertions(+), 1 deletion(-) diff --git a/cmd/flags.go b/cmd/flags.go index 36068af6dd17..ee6e178083d3 100644 --- a/cmd/flags.go +++ b/cmd/flags.go @@ -122,6 +122,10 @@ func storageFlags() []cli.Flag { Value: 10, Usage: "number of retries after network failure", }, + &cli.IntFlag{ + Name: "multi-block", + Usage: "number of blocks for random read ahead", + }, &cli.IntFlag{ Name: "max-uploads", Value: 20, diff --git a/cmd/mount.go b/cmd/mount.go index 68bb5936e0d7..bd4952a05176 100644 --- a/cmd/mount.go +++ b/cmd/mount.go @@ -339,6 +339,7 @@ func getChunkConf(c *cli.Context, format *meta.Format) *chunk.Config { GetTimeout: utils.Duration(c.String("get-timeout")), PutTimeout: utils.Duration(c.String("put-timeout")), + MultiBlock: c.Int("multi-block"), MaxUpload: c.Int("max-uploads"), MaxStageWrite: c.Int("max-stage-write"), MaxRetries: c.Int("io-retries"), diff --git a/pkg/chunk/cached_store.go b/pkg/chunk/cached_store.go index 70edac800bb5..c4d7cfc01c21 100644 --- a/pkg/chunk/cached_store.go +++ b/pkg/chunk/cached_store.go @@ -561,6 +561,7 @@ type Config struct { FreeSpace float32 AutoCreate bool Compress string + MultiBlock int MaxUpload int MaxStageWrite int MaxRetries int diff --git a/pkg/vfs/reader.go b/pkg/vfs/reader.go index e8a858b50f1a..9de991c3da9b 100644 --- a/pkg/vfs/reader.go +++ b/pkg/vfs/reader.go @@ -365,11 +365,19 @@ func (f *fileReader) release() { } } +func (f *fileReader) readMargin(ahead, blklen uint64) uint64 { + if f.r.multiBlock == 0 || + blklen * f.r.multiBlock > ahead + f.r.blockSize { + return ahead + f.r.blockSize + } + return blklen * f.r.multiBlock +} + func (f *fileReader) guessSession(block *frange) int { idx := -1 var closestOff uint64 for i, ses := range f.sessions { - if ses.lastOffset > closestOff && ses.lastOffset <= block.off && block.off <= ses.lastOffset+ses.readahead+f.r.blockSize { + if ses.lastOffset > closestOff && ses.lastOffset <= block.off && block.off <= ses.lastOffset+f.readMargin(ses.readahead, block.len) { idx = i closestOff = ses.lastOffset } @@ -694,6 +702,7 @@ type dataReader struct { m meta.Meta store chunk.ChunkStore files map[Ino]*fileReader + multiBlock uint64 blockSize uint64 bufferSize int64 readAheadMax uint64 @@ -715,6 +724,7 @@ func NewDataReader(conf *Config, m meta.Meta, store chunk.ChunkStore) DataReader m: m, store: store, files: make(map[Ino]*fileReader), + multiBlock: uint64(conf.Chunk.MultiBlock), blockSize: uint64(conf.Chunk.BlockSize), bufferSize: int64(conf.Chunk.BufferSize), readAheadTotal: uint64(readAheadTotal), From 9685e12a62b455e5a55c23210632ce27083fab8f Mon Sep 17 00:00:00 2001 From: Fangxin Lou Date: Tue, 18 Feb 2025 15:25:22 +0800 Subject: [PATCH 2/2] move option to cache category --- cmd/flags.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/flags.go b/cmd/flags.go index ee6e178083d3..64c04d14cda9 100644 --- a/cmd/flags.go +++ b/cmd/flags.go @@ -122,10 +122,6 @@ func storageFlags() []cli.Flag { Value: 10, Usage: "number of retries after network failure", }, - &cli.IntFlag{ - Name: "multi-block", - Usage: "number of blocks for random read ahead", - }, &cli.IntFlag{ Name: "max-uploads", Value: 20, @@ -185,6 +181,10 @@ func dataCacheFlags() []cli.Flag { Name: "max-readahead", Usage: "max buffering for read ahead in MiB per read session", }, + &cli.IntFlag{ + Name: "multi-block", + Usage: "number of blocks for random read ahead", + }, &cli.IntFlag{ Name: "prefetch", Value: 1,