Skip to content

Commit 66e26d2

Browse files
kart2bckart2bc
committed
fix: extend makeRoom() to evict based on real partition free space
ai-assisted=yes TNZ-111552
1 parent 0e2f705 commit 66e26d2

4 files changed

Lines changed: 95 additions & 26 deletions

File tree

‎src/code.cloudfoundry.org/cacheddownloader/cached_downloader_test.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ var _ = Describe("File cache", func() {
6969

7070
transformer = cacheddownloader.NoopTransform
7171

72-
cache = cacheddownloader.NewCache(cachedPath, maxSizeInBytes)
72+
cache = cacheddownloader.NewCache(cachedPath, maxSizeInBytes, 0)
7373
downloader = cacheddownloader.NewDownloader(1*time.Second, MAX_CONCURRENT_DOWNLOADS, nil)
7474
cachedDownloader, err = cacheddownloader.New(downloader, cache, transformer)
7575
Expect(err).NotTo(HaveOccurred())
@@ -89,7 +89,7 @@ var _ = Describe("File cache", func() {
8989
Describe("When a new CachedDownloader fails to create a temp directory for the cache", func() {
9090
It("returns an error", func() {
9191
cachedPath = ""
92-
cache = cacheddownloader.NewCache(cachedPath, maxSizeInBytes)
92+
cache = cacheddownloader.NewCache(cachedPath, maxSizeInBytes, 0)
9393
downloader = cacheddownloader.NewDownloader(1*time.Second, MAX_CONCURRENT_DOWNLOADS, nil)
9494
cachedDownloader, err = cacheddownloader.New(downloader, cache, transformer)
9595
Expect(err).To(HaveOccurred())
@@ -1048,7 +1048,7 @@ var _ = Describe("File cache", func() {
10481048
ghttp.RespondWith(http.StatusOK, string(downloadContent), returnedHeader),
10491049
))
10501050

1051-
cache = cacheddownloader.NewCache(cachedPath, maxSizeInBytes)
1051+
cache = cacheddownloader.NewCache(cachedPath, maxSizeInBytes, 0)
10521052
cachedDownloader, err = cacheddownloader.New(downloader, cache, cacheddownloader.TarTransform)
10531053
Expect(err).ToNot(HaveOccurred())
10541054

@@ -1321,7 +1321,7 @@ var _ = Describe("File cache", func() {
13211321
Expect(cachedDownloader.SaveState(logger)).To(Succeed())
13221322

13231323
maxSizeInBytes = 32
1324-
cache = cacheddownloader.NewCache(cachedPath, maxSizeInBytes)
1324+
cache = cacheddownloader.NewCache(cachedPath, maxSizeInBytes, 0)
13251325
})
13261326

13271327
It("should evict old entries from the cache", func() {

‎src/code.cloudfoundry.org/cacheddownloader/file_cache.go‎

Lines changed: 26 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,11 @@ import (
44
"errors"
55
"fmt"
66
"io"
7+
"math"
78
"os"
89
"path/filepath"
910
"sync"
11+
"syscall"
1012
"time"
1113

1214
"code.cloudfoundry.org/archiver/compressor"
@@ -23,12 +25,21 @@ var (
2325
)
2426

2527
type FileCache struct {
26-
CachedPath string
27-
maxSizeInBytes int64
28-
minFreeBytes int64
29-
Entries map[string]*FileCacheEntry
30-
OldEntries map[string]*FileCacheEntry
31-
Seq uint64
28+
CachedPath string
29+
maxSizeInBytes int64
30+
minimumPartitionFreeBytes int64
31+
FreeSpaceFunc func(path string) int64 `json:"-"`
32+
Entries map[string]*FileCacheEntry
33+
OldEntries map[string]*FileCacheEntry
34+
Seq uint64
35+
}
36+
37+
func defaultFreeSpaceOnPartition(path string) int64 {
38+
var stat syscall.Statfs_t
39+
if err := syscall.Statfs(path, &stat); err != nil {
40+
return math.MaxInt64
41+
}
42+
return int64(stat.Bavail) * int64(stat.Bsize)
3243
}
3344

3445
type FileCacheEntry struct {
@@ -41,18 +52,15 @@ type FileCacheEntry struct {
4152
fileInUseCount int
4253
}
4354

44-
func NewCache(dir string, maxSizeInBytes int64, minFreeBytes ...int64) *FileCache {
45-
var minFree int64
46-
if len(minFreeBytes) > 0 {
47-
minFree = minFreeBytes[0]
48-
}
55+
func NewCache(dir string, maxSizeInBytes, minimumPartitionFreeBytes int64) *FileCache {
4956
return &FileCache{
50-
CachedPath: dir,
51-
maxSizeInBytes: maxSizeInBytes,
52-
minFreeBytes: minFree,
53-
Entries: map[string]*FileCacheEntry{},
54-
OldEntries: map[string]*FileCacheEntry{},
55-
Seq: 0,
57+
CachedPath: dir,
58+
maxSizeInBytes: maxSizeInBytes,
59+
minimumPartitionFreeBytes: minimumPartitionFreeBytes,
60+
FreeSpaceFunc: defaultFreeSpaceOnPartition,
61+
Entries: map[string]*FileCacheEntry{},
62+
OldEntries: map[string]*FileCacheEntry{},
63+
Seq: 0,
5664
}
5765
}
5866

@@ -384,7 +392,7 @@ func (c *FileCache) updateOldEntries(logger lager.Logger, cacheKey string, entry
384392

385393
func (c *FileCache) makeRoom(logger lager.Logger, size int64, excludedCacheKey string) {
386394
usedSpace := c.usedSpace(logger)
387-
for c.maxSizeInBytes < usedSpace+size {
395+
for c.maxSizeInBytes < usedSpace+size || (c.minimumPartitionFreeBytes > 0 && c.FreeSpaceFunc(c.CachedPath) < c.minimumPartitionFreeBytes) {
388396
var oldestEntry *FileCacheEntry
389397
oldestAccessTime, oldestCacheKey := maxTime(), ""
390398
for ck, f := range c.Entries {

‎src/code.cloudfoundry.org/cacheddownloader/file_cache_test.go‎

Lines changed: 64 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ var _ = Describe("FileCache", func() {
3333

3434
maxSizeInBytes = 123424
3535

36-
cache = cacheddownloader.NewCache(cacheDir, maxSizeInBytes)
36+
cache = cacheddownloader.NewCache(cacheDir, maxSizeInBytes, 0)
3737
})
3838

3939
AfterEach(func() {
@@ -516,7 +516,7 @@ var _ = Describe("FileCache", func() {
516516
Context("when there is not enough space in the cache", func() {
517517
BeforeEach(func() {
518518
maxSizeInBytes = 150
519-
cache = cacheddownloader.NewCache(cacheDir, maxSizeInBytes)
519+
cache = cacheddownloader.NewCache(cacheDir, maxSizeInBytes, 0)
520520
})
521521

522522
JustBeforeEach(func() {
@@ -773,7 +773,7 @@ var _ = Describe("FileCache", func() {
773773
Context("when there isn't enough space", func() {
774774
BeforeEach(func() {
775775
maxSizeInBytes = 10
776-
cache = cacheddownloader.NewCache(cacheDir, maxSizeInBytes)
776+
cache = cacheddownloader.NewCache(cacheDir, maxSizeInBytes, 0)
777777
})
778778

779779
JustBeforeEach(func() {
@@ -836,6 +836,67 @@ var _ = Describe("FileCache", func() {
836836
})
837837
})
838838

839+
Describe("makeRoom partition free-space eviction", func() {
840+
var (
841+
cacheKey string
842+
cacheInfo cacheddownloader.CachingInfoType
843+
)
844+
845+
BeforeEach(func() {
846+
cacheKey = "key"
847+
cacheInfo = cacheddownloader.CachingInfoType{}
848+
// large in-memory ceiling so only partition pressure triggers eviction
849+
maxSizeInBytes = 10 * 1024 * 1024 * 1024
850+
cache = cacheddownloader.NewCache(cacheDir, maxSizeInBytes, 5*1024*1024*1024)
851+
})
852+
853+
Context("when partition free space is below the minimum", func() {
854+
BeforeEach(func() {
855+
cache.FreeSpaceFunc = func(string) int64 { return 0 }
856+
})
857+
858+
It("evicts non-in-use entries even though in-memory ceiling is not exceeded", func() {
859+
f1 := createFile("cache-file-1", "content-1")
860+
defer os.RemoveAll(f1.Name())
861+
rc, err := cache.Add(logger, cacheKey, f1.Name(), 100, cacheInfo)
862+
Expect(err).NotTo(HaveOccurred())
863+
rc.Close()
864+
865+
f2 := createFile("cache-file-2", "content-2")
866+
defer os.RemoveAll(f2.Name())
867+
rc2, err := cache.Add(logger, "key2", f2.Name(), 100, cacheInfo)
868+
Expect(err).NotTo(HaveOccurred())
869+
rc2.Close()
870+
871+
_, _, err = cache.Get(logger, cacheKey)
872+
Expect(err).To(Equal(cacheddownloader.EntryNotFound))
873+
})
874+
})
875+
876+
Context("when partition free space exceeds the minimum", func() {
877+
BeforeEach(func() {
878+
cache.FreeSpaceFunc = func(string) int64 { return 10 * 1024 * 1024 * 1024 }
879+
})
880+
881+
It("does not evict entries solely due to partition pressure", func() {
882+
f1 := createFile("cache-file-1", "content-1")
883+
defer os.RemoveAll(f1.Name())
884+
rc, err := cache.Add(logger, cacheKey, f1.Name(), 100, cacheInfo)
885+
Expect(err).NotTo(HaveOccurred())
886+
rc.Close()
887+
888+
f2 := createFile("cache-file-2", "content-2")
889+
defer os.RemoveAll(f2.Name())
890+
rc2, err := cache.Add(logger, "key2", f2.Name(), 100, cacheInfo)
891+
Expect(err).NotTo(HaveOccurred())
892+
rc2.Close()
893+
894+
_, _, err = cache.Get(logger, cacheKey)
895+
Expect(err).NotTo(HaveOccurred())
896+
})
897+
})
898+
})
899+
839900
Describe("Remove", func() {
840901
var (
841902
cacheKey string

‎src/code.cloudfoundry.org/cacheddownloader/integration_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ var _ = Describe("Integration", func() {
4747
url, err = url.Parse(server.URL + "/file")
4848
Expect(err).NotTo(HaveOccurred())
4949

50-
cache = cacheddownloader.NewCache(cachedPath, cacheMaxSizeInBytes)
50+
cache = cacheddownloader.NewCache(cachedPath, cacheMaxSizeInBytes, 0)
5151
downloader = cacheddownloader.NewDownloader(downloadTimeout, 10, nil)
5252
cachedDownloader, err = cacheddownloader.New(downloader, cache, cacheddownloader.NoopTransform)
5353
Expect(err).ToNot(HaveOccurred())

0 commit comments

Comments
 (0)