|
1 | 1 | import { existsSync } from "node:fs"; |
2 | | -import { readdir, readFile } from "node:fs/promises"; |
| 2 | +import { readdir, readFile, mkdir } from "node:fs/promises"; |
| 3 | +import { createWriteStream } from "node:fs"; |
3 | 4 | import path from "node:path"; |
4 | | -import { pathToFileURL } from "node:url"; |
| 5 | +import { spawn } from "node:child_process"; |
5 | 6 | import { dsvFormat, autoType } from "d3-dsv"; |
6 | | -import { LoaderResolver, } from "@observablehq/framework/dist/loader.js"; |
7 | | -import { getResolvers } from "@observablehq/framework/dist/resolvers.js"; |
8 | | -import { normalizeConfig } from "@observablehq/framework/dist/config.js"; |
9 | | -import { parseMarkdown } from "@observablehq/framework/dist/markdown.js"; |
10 | 7 |
|
11 | | -function getOptions({ path, ...config }) { |
12 | | - return { ...normalizeConfig(config), path }; |
| 8 | +const CACHE_DIR = ".observablehq/cache"; |
| 9 | + |
| 10 | +// Interpreter commands for data loader scripts (mirrors @observablehq/framework defaults) |
| 11 | +function getInterpreterCommand(ext: string): [string, string[]] | null { |
| 12 | + switch (ext) { |
| 13 | + case ".js": return ["node", ["--no-warnings=ExperimentalWarning"]]; |
| 14 | + case ".ts": return ["tsx", []]; |
| 15 | + case ".py": return ["python3", []]; |
| 16 | + case ".r": |
| 17 | + case ".R": return ["Rscript", []]; |
| 18 | + default: return null; |
| 19 | + } |
| 20 | +} |
| 21 | + |
| 22 | +function findLoader(partialPath: string, root: string): { loaderPath: string; loaderExt: string } | null { |
| 23 | + for (const ext of [".js", ".ts", ".py", ".r", ".R"]) { |
| 24 | + const loaderPath = path.resolve(root, `${partialPath}${ext}`); |
| 25 | + if (existsSync(loaderPath)) { |
| 26 | + return { loaderPath, loaderExt: ext }; |
| 27 | + } |
| 28 | + } |
| 29 | + return null; |
| 30 | +} |
| 31 | + |
| 32 | +async function runLoader(loaderPath: string, loaderExt: string, cachePath: string): Promise<void> { |
| 33 | + const cmd = getInterpreterCommand(loaderExt); |
| 34 | + if (!cmd) throw new Error(`No interpreter for loader extension: ${loaderExt}`); |
| 35 | + await mkdir(path.dirname(cachePath), { recursive: true }); |
| 36 | + const [command, args] = cmd; |
| 37 | + const child = spawn(command, [...args, loaderPath], { |
| 38 | + windowsHide: true, |
| 39 | + stdio: ["ignore", "pipe", "inherit"] |
| 40 | + }); |
| 41 | + child.stdout!.pipe(createWriteStream(cachePath)); |
| 42 | + await new Promise<void>((resolve, reject) => { |
| 43 | + child.on("error", reject); |
| 44 | + child.on("exit", (code) => |
| 45 | + code === 0 ? resolve() : reject(new Error(`Loader "${loaderPath}" exited with code ${code}`)) |
| 46 | + ); |
| 47 | + }); |
13 | 48 | } |
14 | 49 |
|
15 | 50 | export class DataFile { |
16 | 51 |
|
17 | 52 | private filePath: string; |
18 | | - private options: any; |
19 | | - private resolvers: any; |
20 | 53 | readonly ext: string; |
21 | 54 |
|
22 | | - protected constructor(filePath, options, resolvers) { |
| 55 | + protected constructor(filePath: string) { |
23 | 56 | this.filePath = filePath; |
24 | | - this.options = options; |
25 | | - this.resolvers = resolvers; |
26 | 57 | this.ext = path.extname(filePath).substring(1); |
27 | 58 | } |
28 | 59 |
|
29 | 60 | static async attach(partialPath: string, root: string = path.resolve(".")) { |
30 | | - const exists = existsSync(path.resolve(root, partialPath)); |
31 | | - const loaders = new LoaderResolver({ root, interpreters: {} }); |
32 | | - const loader = loaders.find(partialPath); |
33 | | - if (loader) { |
34 | | - await loader.load(); |
35 | | - const ext = path.extname(partialPath); |
36 | | - const options = getOptions({ root, path: "dummy.md" }); |
37 | | - const page = parseMarkdown(`\${FileAttachment('${partialPath}')${ext}()}`, options); |
38 | | - const resolvers = await getResolvers(page, options); |
39 | | - resolvers.resolveFile(partialPath); |
40 | | - return new DataFile(exists ? path.resolve(root, partialPath) : path.resolve(path.join(root, ".observablehq", "cache", partialPath)), options, resolvers); |
| 61 | + const absolutePath = path.resolve(root, partialPath); |
| 62 | + if (existsSync(absolutePath)) { |
| 63 | + return new DataFile(absolutePath); |
41 | 64 | } |
42 | | - } |
43 | | - |
44 | | - async myResolve(module: string) { |
45 | | - const partialPath = await this.resolvers.resolveImport(module); |
46 | | - return path.resolve(path.join(this.options.root, ".observablehq", "cache", partialPath)); |
47 | | - } |
48 | | - |
49 | | - async myImport(module: string) { |
50 | | - if (module === "npm:apache-arrow") { |
51 | | - return import("apache-arrow"); |
| 65 | + const loaderInfo = findLoader(partialPath, root); |
| 66 | + if (loaderInfo) { |
| 67 | + const cachePath = path.resolve(root, CACHE_DIR, partialPath); |
| 68 | + if (!existsSync(cachePath)) { |
| 69 | + await runLoader(loaderInfo.loaderPath, loaderInfo.loaderExt, cachePath); |
| 70 | + } |
| 71 | + return new DataFile(cachePath); |
52 | 72 | } |
53 | | - const fullPath = await this.myResolve(module); |
54 | | - const href = pathToFileURL(fullPath).href; |
55 | | - return import(href).catch((error) => { |
56 | | - console.error(error); |
57 | | - }); |
| 73 | + return undefined; |
58 | 74 | } |
59 | 75 |
|
60 | 76 | buffer(): Promise<Buffer> { |
@@ -92,24 +108,13 @@ export class DataFile { |
92 | 108 | } |
93 | 109 |
|
94 | 110 | arrow() { |
95 | | - return Promise.all([this.myImport("npm:apache-arrow"), this.arrayBuffer()]).then(([Arrow, response]) => { |
| 111 | + return Promise.all([import("apache-arrow"), this.arrayBuffer()]).then(([Arrow, response]) => { |
96 | 112 | return Arrow.tableFromIPC(response); |
97 | 113 | }); |
98 | 114 | } |
99 | 115 |
|
100 | 116 | parquet() { |
101 | | - return this.myResolve("npm:parquet-wasm/esm/parquet_wasm_bg.wasm").then(wasmBytes => { |
102 | | - const wasmFile = new DataFile(wasmBytes, this.options, this.resolvers); |
103 | | - return Promise.all([ |
104 | | - this.myImport("npm:apache-arrow"), |
105 | | - this.myImport("npm:parquet-wasm"), |
106 | | - wasmFile.arrayBuffer(), |
107 | | - this.arrayBuffer() |
108 | | - ]).then(([Arrow, Parquet, wasm, buffer]) => { |
109 | | - Parquet.initSync(wasm); |
110 | | - return Arrow.tableFromIPC(Parquet.readParquet(new Uint8Array(buffer)).intoIPCStream()); |
111 | | - }); |
112 | | - }); |
| 117 | + throw new Error("parquet() is not supported; install @observablehq/framework for npm: package resolution."); |
113 | 118 | } |
114 | 119 |
|
115 | 120 | fetch() { |
|
0 commit comments