Move parquet-squirreling integration to src/lib/parquet

Author: platypii

bin/tools/parquetSql.js

@@ -1,7 +1,7 @@
 import { asyncBufferFromFile, asyncBufferFromUrl, parquetMetadataAsync } from 'hyparquet'
 import { compressors } from 'hyparquet-compressors'
 import { collect, executeSql } from 'squirreling'
-import { parquetDataSource } from './parquetDataSource.js'
+import { parquetDataSource } from '../../src/lib/parquet/parquetDataSource.js'
 import { markdownTable } from './markdownTable.js'
 
 const maxRows = 100

package.json

@@ -59,7 +59,7 @@
     "hyparquet": "1.25.1",
     "hyparquet-compressors": "1.1.1",
     "icebird": "0.3.1",
-    "squirreling": "0.9.4"
+    "squirreling": "0.10.2"
   },
   "devDependencies": {
     "@storybook/react-vite": "10.2.13",

src/lib/index.ts

@@ -3,5 +3,6 @@ export * from './sources/index.js'
 export { parseCsv } from './csv.js'
 export { csvDataFrame, jsonLinesDataFrame, parquetDataFrame, tableProvider } from './tableProvider.js'
 export { asyncBufferFrom, cn, contentTypes, formatFileSize, getFileDate, getFileDateShort, imageTypes, parseFileSize } from './utils.js'
+export { parquetDataSource } from './parquet/parquetDataSource.js'
 export { parquetQueryWorker, parquetReadObjectsWorker, parquetReadWorker } from './workers/parquetWorkerClient.js'
 export type { AsyncBufferFrom } from './workers/types.js'

bin/tools/parquetDataSource.js src/lib/parquet/parquetDataSource.jsbin/tools/parquetDataSource.js renamed to src/lib/parquet/parquetDataSource.js

@@ -15,7 +15,9 @@ import { whereToParquetFilter } from './parquetFilter.js'
  * @returns {AsyncDataSource}
  */
 export function parquetDataSource(file, metadata, compressors) {
+  const schema = parquetSchema(metadata)
   return {
+    columns: schema.children.map(child => child.element.name),
     scan({ columns, where, limit, offset, signal }) {
       // Convert WHERE AST to hyparquet filter format
       const whereFilter = where && whereToParquetFilter(where)