Use sqlite cache of pantry data

We had to vendor the deno sqlite3 library as it had no other direct way to allow us to configure the path to the sqlite library at runtime.

To accomplish this we had to modify the sources somewhat also :/

Author: mxcl

.github/deno-to-node.ts

@@ -16,6 +16,17 @@ const version = (() => {
   }
 })()
 
+const mappings: Record<string, string> = {
+  "https://deno.land/x/is_what@v4.1.15/src/index.ts": "is-what",
+  "https://deno.land/x/outdent@v0.8.0/mod.ts": "outdent",
+  "./src/utils/flock.deno.ts": "./src/utils/flock.node.ts",
+  "./src/hooks/useSyncCache.ts": "./src/hooks/useSyncCache.node.ts"
+}
+
+if (test) {
+  mappings["./src/hooks/useSyncCache.test.ts"] = "./src/hooks/useCache.test.ts"  // no other easy way to skip the test
+}
+
 await build({
   entryPoints: ["./mod.ts"],
   outDir: "./dist",
@@ -30,11 +41,7 @@ await build({
     }],
   },
   importMap: "deno.json",
-  mappings: {
-    "https://deno.land/x/is_what@v4.1.15/src/index.ts": "is-what",
-    "https://deno.land/x/outdent@v0.8.0/mod.ts": "outdent",
-    "./src/utils/flock.deno.ts": "./src/utils/flock.node.ts"
-  },
+  mappings,
   package: {
     name: "libpkgx",
     version,

.github/workflows/ci.yml

@@ -66,7 +66,7 @@ jobs:
         with:
           path: src
       - uses: denoland/setup-deno@v1
-      - run: deno run --no-config --unstable src/mod.ts
+      - run: deno run --no-config --unstable --allow-all src/mod.ts
 
   dnt:
     runs-on: ${{ matrix.os }}

deno.json

@@ -12,7 +12,7 @@
   },
   "pkgx": "deno~1.39 npm",
   "tasks": {
-    "test": "deno test --parallel --unstable --allow-env --allow-read --allow-net=dist.pkgx.dev,github.com,codeload.github.com --allow-write --allow-run=tar,uname,/bin/sh,foo,'C:\\Windows\\system32\\cmd.exe'",
+    "test": "deno test --parallel --unstable --allow-env --allow-read --allow-ffi --allow-net=dist.pkgx.dev,github.com,codeload.github.com --allow-write --allow-run=tar,uname,/bin/sh,foo,'C:\\Windows\\system32\\cmd.exe'",
     "typecheck": "deno check --unstable ./mod.ts",
     "dnt": ".github/deno-to-node.ts"
   },

src/hooks/usePantry.test.ts

@@ -85,3 +85,10 @@ Deno.test("validatePackageRequirement - number constraint", () => {
   const result = validatePackageRequirement("pkgx.sh/test", 1)
   assertEquals(result?.constraint.toString(), "^1")
 })
+
+Deno.test("find", async () => {
+  useTestConfig()
+  const foo = await usePantry().find("python@3.11")
+  assertEquals(foo.length, 1)
+  assertEquals(foo[0].project, "python.org")
+})

src/hooks/usePantry.ts

@@ -1,10 +1,12 @@
 import { is_what, PlainObject } from "../deps.ts"
 const { isNumber, isPlainObject, isString, isArray, isPrimitive, isBoolean } = is_what
 import { Package, Installation, PackageRequirement } from "../types.ts"
+import { provides as cache_provides, available as cache_available, runtime_env as cache_runtime_env, companions as cache_companions, dependencies as cache_dependencies } from "./useSyncCache.ts";
 import SemVer, * as semver from "../utils/semver.ts"
 import useMoustaches from "./useMoustaches.ts"
 import { PkgxError } from "../utils/error.ts"
 import { validate } from "../utils/misc.ts"
+import * as pkgutils from "../utils/pkg.ts"
 import useConfig from "./useConfig.ts"
 import host from "../utils/host.ts"
 import Path from "../utils/Path.ts"
@@ -45,6 +47,7 @@ export class PantryNotFoundError extends PantryError {
 
 export default function usePantry() {
   const prefix = useConfig().data.join("pantry/projects")
+  const is_cache_available = cache_available() && pantry_paths().length == 1
 
   async function* ls(): AsyncGenerator<LsEntry> {
     const seen = new Set()
@@ -78,11 +81,23 @@ export default function usePantry() {
       throw new PackageNotFoundError(project)
     })()
 
-    const companions = async () => parse_pkgs_node((await yaml())["companions"])
+    const companions = async () => {
+      if (is_cache_available) {
+        return await cache_companions(project) ?? parse_pkgs_node((await yaml())["companions"])
+      } else {
+        return parse_pkgs_node((await yaml())["companions"])
+      }
+    }
 
     const runtime_env = async (version: SemVer, deps: Installation[]) => {
-      const yml = await yaml()
-      const obj = validate.obj(yml["runtime"]?.["env"] ?? {})
+      const obj = await (async () => {
+        if (is_cache_available) {
+          const cached = await cache_runtime_env(project)
+          if (cached) return cached
+        }
+        const yml = await yaml()
+        return validate.obj(yml["runtime"]?.["env"] ?? {})
+      })()
       return expand_env_obj(obj, { project, version }, deps)
     }
 
@@ -94,7 +109,13 @@ export default function usePantry() {
       return platforms.includes(host().platform) ||platforms.includes(`${host().platform}/${host().arch}`)
     }
 
-    const drydeps = async () => parse_pkgs_node((await yaml()).dependencies)
+    const drydeps = async () => {
+      if (is_cache_available) {
+        return await cache_dependencies(project) ?? parse_pkgs_node((await yaml()).dependencies)
+      } else {
+        return parse_pkgs_node((await yaml()).dependencies)
+      }
+    }
 
     const provides = async () => {
       let node = (await yaml())["provides"]
@@ -164,6 +185,27 @@ export default function usePantry() {
   async function find(name: string) {
     type Foo = ReturnType<typeof project> & LsEntry
 
+    //lol FIXME
+    name = pkgutils.parse(name).project
+
+    if (prefix.join(name).isDirectory()) {
+      const foo = project(name)
+      return [{...foo, project: name }]
+    }
+
+    /// only use cache if PKGX_PANTRY_PATH is not set
+    if (is_cache_available) {
+      const cached = await cache_provides(name)
+      if (cached?.length) {
+        return cached.map(x => ({
+          ...project(x),
+          project: x
+        }))
+      }
+
+      // else we need to still check for display-names
+    }
+
     name = name.toLowerCase()
 
     //TODO not very performant due to serial awaits
@@ -234,7 +276,8 @@ export default function usePantry() {
     parse_pkgs_node,
     expand_env_obj,
     missing,
-    neglected
+    neglected,
+    pantry_paths
   }
 
   function pantry_paths(): Path[] {

src/hooks/useSync.test.ts

@@ -1,44 +1,53 @@
+import specimen, { _internals } from "./useSync.ts"
 import { useTestConfig } from "./useTestConfig.ts"
+import * as mock from "deno/testing/mock.ts"
 import { assert } from "deno/assert/mod.ts"
 import usePantry from "./usePantry.ts"
-import useSync from "./useSync.ts"
 
 // NOTE actually syncs from github
 // TODO unit tests should not do actual network calls, instead make an implementation suite
 
 Deno.test("useSync", async runner => {
-  await runner.step("w/o git", async () => {
-    const conf = useTestConfig({})
-    usePantry().prefix.rm({ recursive: true })  // we need to delete the fixtured pantry
-    assert(conf.git === undefined)
-    await test()
-  })
-
-  await runner.step({
-    name: "w/git",
-    ignore: Deno.build.os == 'windows' && !Deno.env.get("CI"),
-    async fn() {
-      const conf = useTestConfig({ PATH: "/usr/bin" })
+  const stub = mock.stub(_internals, "cache", async () => {})
+
+  try {
+    await runner.step("w/o git", async () => {
+      const conf = useTestConfig({})
       usePantry().prefix.rm({ recursive: true })  // we need to delete the fixtured pantry
-      assert(conf.git !== undefined)
+      assert(conf.git === undefined)
       await test()
+    })
 
-      // test the “already cloned” code-path
-      await useSync()
-    }
-  })
-
-  async function test() {
-    let errord = false
-    try {
-      await usePantry().project("gnu.org/gcc").available()
-    } catch {
-      errord = true
-    }
-    assert(errord, `should be no pantry but there is! ${usePantry().prefix}`)
+    await runner.step({
+      name: "w/git",
+      ignore: Deno.build.os == 'windows' && !Deno.env.get("CI"),
+      async fn() {
+        const conf = useTestConfig({ PATH: "/usr/bin" })
+        usePantry().prefix.rm({ recursive: true })  // we need to delete the fixtured pantry
+        assert(conf.git !== undefined)
+        await test()
+
+        // test the “already cloned” code-path
+        await specimen()
+      }
+    })
 
-    await useSync()
+    async function test() {
+      let errord = false
+      try {
+        await usePantry().project("gnu.org/gcc").available()
+      } catch {
+        errord = true
+      }
+      assert(errord, `should be no pantry but there is! ${usePantry().prefix}`)
 
-    assert(await usePantry().project("gnu.org/gcc").available())
+      await specimen()
+
+      assert(await usePantry().project("gnu.org/gcc").available())
+    }
+
+  } finally {
+    stub.restore()
   }
+
 })

src/hooks/useSync.ts

@@ -7,12 +7,14 @@ import useDownload from "./useDownload.ts"
 import usePantry from "./usePantry.ts"
 import useConfig from "./useConfig.ts"
 import Path from "../utils/Path.ts"
+import useSyncCache from "./useSyncCache.ts";
 
 //FIXME tar is fetched from PATH :/ we want control
 //FIXME run in general is not controllable since it delegates to the shell
 
 interface Logger {
   syncing(path: Path): void
+  caching(path: Path): void
   syncd(path: Path): void
 }
 
@@ -23,6 +25,27 @@ export default async function(logger?: Logger) {
 
   const unflock = await flock(pantry_dir.mkdir('p'))
 
+  try {
+    await _internals.sync(pantry_dir)
+    try {
+      logger?.caching(pantry_dir)
+      await _internals.cache()
+    } catch (err) {
+      console.warn("failed to cache pantry")
+      console.error(err)
+    }
+  } finally {
+    await unflock()
+  }
+
+  logger?.syncd(pantry_dir)
+}
+
+export const _internals = {
+  sync, cache: useSyncCache
+}
+
+async function sync(pantry_dir: Path) {
   try {
     //TODO if there was already a lock, just wait on it, don’t do the following stuff
 
@@ -55,11 +78,7 @@ export default async function(logger?: Logger) {
 
     proc.close()
 
-  } finally {
-    await unflock()
   }
-
-  logger?.syncd(pantry_dir)
 }
 
 //////////////////////// utils

src/hooks/useSyncCache.node.ts

@@ -0,0 +1,31 @@
+// the sqlite lib we use only works in deno
+
+import { PackageRequirement } from "../../mod.ts";
+
+export default async function()
+{}
+
+export function provides(_program: string): string[] {
+  throw new Error()
+}
+
+export function dependencies(_project: string): PackageRequirement[] {
+  throw new Error()
+}
+
+export function completion(_prefix: string): string[] {
+  throw new Error()
+}
+
+/// is the cache available?
+export function available(): boolean {
+  return false
+}
+
+export function companions(_project: string): PackageRequirement[] {
+  throw new Error()
+}
+
+export function runtime_env(_project: string): Record<string, string> {
+  throw new Error()
+}

src/hooks/useSyncCache.test.ts

@@ -0,0 +1,27 @@
+import specimen, { provides, dependencies, available, runtime_env, completion, companions  } from "./useSyncCache.ts"
+import { useTestConfig } from "./useTestConfig.ts"
+import { assert, assertEquals } from "deno/assert/mod.ts"
+import { _internals } from "./useSync.ts"
+import usePantry from "./usePantry.ts"
+
+// NOTE actually syncs from github
+// TODO unit tests should not do actual network calls, instead make an implementation suite
+
+Deno.test({
+  name: "useSyncCache",
+  ignore: Deno.build.os == 'windows',
+  sanitizeResources: false,
+  async fn() {
+    useTestConfig()
+    await _internals.sync(usePantry().prefix.parent())
+    await specimen()
+
+    //TODO test better
+    assert(available())
+    assertEquals((await provides('node'))?.[0], 'nodejs.org')
+    // assertEquals((await dependencies('nodejs.org'))?.length, 3)
+    assert(new Set(await completion('nod')).has("node"))
+    assertEquals((await companions("nodejs.org"))?.[0]?.project, "npmjs.com")
+    assert((await runtime_env("numpy.org"))?.["PYTHONPATH"])
+  }
+})