diff --git a/api/common/config.go b/api/common/config.go index e4f23ad5..17253dee 100644 --- a/api/common/config.go +++ b/api/common/config.go @@ -45,11 +45,12 @@ type FlagStorage struct { Backend interface{} // Tuning - Cheap bool - ExplicitDir bool - StatCacheTTL time.Duration - TypeCacheTTL time.Duration - HTTPTimeout time.Duration + Cheap bool + ExplicitDir bool + StatCacheTTL time.Duration + TypeCacheTTL time.Duration + HTTPTimeout time.Duration + ReadAheadChunk uint32 // Debugging DebugFuse bool diff --git a/internal/file.go b/internal/file.go index a2ccbaa7..9a020575 100644 --- a/internal/file.go +++ b/internal/file.go @@ -67,7 +67,6 @@ type FileHandle struct { } const MAX_READAHEAD = uint32(400 * 1024 * 1024) -const READAHEAD_CHUNK = uint32(20 * 1024 * 1024) // NewFileHandle returns a new file handle for the given `inode` triggered by fuse // operation with the given `opMetadata` @@ -421,24 +420,27 @@ func (fh *FileHandle) readFromReadAhead(offset uint64, buf []byte) (bytesRead in } func (fh *FileHandle) readAhead(offset uint64, needAtLeast int) (err error) { + fs := fh.inode.fs existingReadahead := uint32(0) for _, b := range fh.buffers { existingReadahead += b.size } readAheadAmount := MAX_READAHEAD + readAheadChunk := fs.flags.ReadAheadChunk * 1024 * 1024 + - for readAheadAmount-existingReadahead >= READAHEAD_CHUNK { + for readAheadAmount-existingReadahead >= readAheadChunk { off := offset + uint64(existingReadahead) remaining := fh.inode.Attributes.Size - off // only read up to readahead chunk each time - size := MinUInt32(readAheadAmount-existingReadahead, READAHEAD_CHUNK) + size := MinUInt32(readAheadAmount-existingReadahead, readAheadChunk) // but don't read past the file size = uint32(MinUInt64(uint64(size), remaining)) if size != 0 { - fh.inode.logFuse("readahead", off, size, existingReadahead) + fh.inode.logFuse("readahead", off, size, existingReadahead, readAheadChunk) readAheadBuf := S3ReadBuffer{}.Init(fh, off, size) if readAheadBuf != nil { @@ -456,7 +458,7 @@ func (fh *FileHandle) readAhead(offset uint64, needAtLeast int) (err error) { } } - if size != READAHEAD_CHUNK { + if size != readAheadChunk { // that was the last remaining chunk to readahead break } @@ -543,7 +545,9 @@ func (fh *FileHandle) readFile(offset int64, buf []byte) (bytesRead int, err err fh.buffers = nil } - if !fs.flags.Cheap && fh.seqReadAmount >= uint64(READAHEAD_CHUNK) && fh.numOOORead < 3 { + readAheadChunk := fs.flags.ReadAheadChunk * 1024 * 1024 + + if !fs.flags.Cheap && fh.seqReadAmount >= uint64(readAheadChunk) && fh.numOOORead < 3 { if fh.reader != nil { fh.inode.logFuse("cutover to the parallel algorithm") fh.reader.Close() diff --git a/internal/flags.go b/internal/flags.go index f18633a8..bb66dd23 100644 --- a/internal/flags.go +++ b/internal/flags.go @@ -86,7 +86,7 @@ func NewApp() (app *cli.App) { app = &cli.App{ Name: "goofys", - Version: "0.24.0-" + VersionHash, + Version: "0.24.1-rc1-" + VersionHash, Usage: "Mount an S3 bucket locally", HideHelp: true, Writer: os.Stderr, @@ -237,12 +237,19 @@ func NewApp() (app *cli.App) { Usage: "How long to cache name -> file/dir mappings in directory " + "inodes.", }, + cli.DurationFlag{ Name: "http-timeout", Value: 30 * time.Second, Usage: "Set the timeout on HTTP requests to S3", }, + cli.IntFlag{ + Name: "read-ahead-chunk", + Value: 20, + Usage: "Read ahead size in MiB for S3 range requests.", + }, + ///////////////////////// // Debugging ///////////////////////// @@ -261,6 +268,7 @@ func NewApp() (app *cli.App) { Name: "f", Usage: "Run goofys in foreground.", }, + }, } @@ -275,7 +283,7 @@ func NewApp() (app *cli.App) { flagCategories[f] = "aws" } - for _, f := range []string{"cheap", "no-implicit-dir", "stat-cache-ttl", "type-cache-ttl", "http-timeout"} { + for _, f := range []string{"cheap", "no-implicit-dir", "stat-cache-ttl", "type-cache-ttl", "http-timeout", "read-ahead-chunk"} { flagCategories[f] = "tuning" } @@ -327,11 +335,12 @@ func PopulateFlags(c *cli.Context) (ret *FlagStorage) { Gid: uint32(c.Int("gid")), // Tuning, - Cheap: c.Bool("cheap"), - ExplicitDir: c.Bool("no-implicit-dir"), - StatCacheTTL: c.Duration("stat-cache-ttl"), - TypeCacheTTL: c.Duration("type-cache-ttl"), - HTTPTimeout: c.Duration("http-timeout"), + Cheap: c.Bool("cheap"), + ExplicitDir: c.Bool("no-implicit-dir"), + StatCacheTTL: c.Duration("stat-cache-ttl"), + TypeCacheTTL: c.Duration("type-cache-ttl"), + HTTPTimeout: c.Duration("http-timeout"), + ReadAheadChunk: uint32(c.Int("read-ahead-chunk")), // Common Backend Config Endpoint: c.String("endpoint"),