fix: Add file descriptor pool to storage (#123)

* fix: Add file descriptor pool to storage

* sanity check - why do tests pass in windows?

* Revert "sanity check - why do tests pass in windows?"

This reverts commit dec316d.

* add explanation of iterator in comment

* Update tests/core-manager.js

Co-authored-by: Andrew Chou <andrewchou@fastmail.com>

---------

Co-authored-by: Andrew Chou <andrewchou@fastmail.com>

Author: gmaclennan

lib/core-manager/index.js

@@ -63,7 +63,7 @@ export class CoreManager extends TypedEmitter {
     projectKey,
     projectSecretKey,
     encryptionKeys = {},
-    storage
+    storage,
   }) {
     super()
     assert(

lib/core-manager/random-access-file-pool.js

@@ -0,0 +1,30 @@
+/**
+ * File descriptor pool for random-access-storage to limit the number of file
+ * descriptors used. Important particularly for Android where the hard limit for
+ * the app is 1024.
+ */
+export class RandomAccessFilePool {
+  /** @param {number} maxSize max number of file descriptors to use */
+  constructor (maxSize) {
+    this.maxSize = maxSize
+    /** @type {Set<import('random-access-file')>} */
+    this.active = new Set()
+  }
+
+  /** @param {import('random-access-file')} file */
+  _onactive (file) {
+    if (this.active.size >= this.maxSize) {
+      // suspend least recently inserted this manually iterates in insertion
+      // order, but only iterates to the first one (least recently inserted)
+      const toSuspend = this.active[Symbol.iterator]().next().value
+      toSuspend.suspend()
+      this.active.delete(toSuspend)
+    }
+    this.active.add(file)
+  }
+
+  /** @param {import('random-access-file')} file */
+  _oninactive (file) {
+    this.active.delete(file)
+  }
+}

package-lock.json

@@ -54,8 +54,10 @@
     "eslint-config-prettier": "^8.8.0",
     "nanobench": "^3.0.0",
     "prettier": "^2.8.8",
+    "random-access-file": "^4.0.4",
     "random-access-memory": "^6.2.0",
     "rimraf": "^5.0.0",
+    "tempy": "^3.1.0",
     "ts-proto": "^1.147.1",
     "type-fest": "^3.10.0",
     "typedoc": "^0.24.6",
@@ -69,7 +71,7 @@
     "b4a": "^1.6.3",
     "base32.js": "^0.1.0",
     "better-sqlite3": "^8.3.0",
-    "corestore": "^6.5.2",
+    "corestore": "^6.8.4",
     "hypercore": "^10.9.0",
     "hypercore-crypto": "^3.3.1",
     "hyperdrive": "^11.0.0-alpha.10",