Skip to content

Memory fragmentation is very high in memory store #289

@chrisstaite-menlo

Description

@chrisstaite-menlo

With the following configuration, the storage server running only turbo-cache got OOM killed even though it has 16Gb of storage after running for ~4 days. Not sure if there's a memory leak somewhere in the recent code?

{
  "stores": {
    "CAS_MAIN_STORE": {
      "fast_slow": {
        "fast": {
          "memory": {
            "eviction_policy": {
              // 4gb
              "max_bytes": 4000000000
            }
          }
        },
        "slow": {
          "filesystem": {
            "content_path": "/root/.cache/turbo-cache/content_path-cas",
            "temp_path": "/root/.cache/turbo-cache/tmp_path-cas",
            "eviction_policy": {
              // 150gb.
              "max_bytes": 150000000000,
              // 2gb
              "evict_bytes": 2000000000
            }
          }
        }
      }
    },
    "AC_MAIN_STORE": {
      "fast_slow": {
        "fast": {
          "memory": {
            "eviction_policy": {
              // 500mb
              "max_bytes": 500000000
            }
          }
        },
        "slow": {
           "filesystem": {
             "content_path": "/root/.cache/turbo-cache/content_path-cas_ac",
             "temp_path": "/root/.cache/turbo-cache/tmp_path-cas_ac",
             "eviction_policy": {
                // 10gb.
                "max_bytes": 10000000000,
             }
          }
        }
      }
    }
  },
  "servers": [{
    "listen_address": "0.0.0.0:50052",
    "services": {
      "cas": {
        "main": {
          "cas_store": "CAS_MAIN_STORE"
        },
      },
      "ac": {
        "main": {
          "ac_store": "AC_MAIN_STORE"
        }
      },
      "capabilities": {},
      "bytestream": {
        "cas_stores": {
          "main": "CAS_MAIN_STORE",
        },
        // According to https://github.com/grpc/grpc.github.io/issues/371 16KiB - 64KiB is optimal.
        "max_bytes_per_stream": 64000, // 64kb.
      }
    }
  }]
}

Metadata

Metadata

Assignees

No one assigned

    Labels

    No labels
    No labels

    Type

    No type
    No fields configured for issues without a type.

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions