Merge pull request #2 from boilingdata/add-support-for-sqlite

feat: Add support for sqlite engine

Author: BoilingData

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@boilingdata/node-boilingdata",
-  "version": "0.0.12",
+  "version": "0.0.13",
   "description": "BoilingData client",
   "main": "dist/cjs/index.js",
   "types": "dist/cjs/index.d.ts",

src/boilingdata/boilingdata.api.ts

@@ -2,9 +2,10 @@
 // https://www.boilingdata.com/apidoc.html
 
 export interface IBDDataQuery {
-  messageType: string; // "SQL_QUERY"
+  messageType: string; // SQL_QUERY
   requestId: string;
   sql: string;
+  engine?: string; // DUCKDB (default), SQLITE
   keys?: string[];
   crossRegionPolicy?: string; // DISALLOWED, ALLOWED, SELECTED
   allowedRegions?: string[];
@@ -37,6 +38,11 @@ export enum EMessageTypes {
   SQL_QUERY = "SQL_QUERY",
 }
 
+export enum EEngineTypes {
+  DUCKDB = "DUCKDB",
+  SQLITE = "SQLITE",
+}
+
 export enum EEvent {
   REQUEST = "REQUEST",
   DATA = "DATA",

src/boilingdata/boilingdata.ts

@@ -1,6 +1,6 @@
 import { createLogger } from "bunyan";
 import { BDCredentials, getBoilingDataCredentials } from "../common/identity";
-import { EEvent, EMessageTypes, IBDDataQuery, IBDDataResponse } from "./boilingdata.api";
+import { EEngineTypes, EEvent, EMessageTypes, IBDDataQuery, IBDDataResponse } from "./boilingdata.api";
 import { v4 as uuidv4 } from "uuid";
 import { WebSocket, MessageEvent } from "ws";
 import { inspect } from "util";
@@ -28,6 +28,7 @@ export interface IBoilingData {
 
 export interface IBDQuery {
   sql: string;
+  engine?: EEngineTypes.DUCKDB | EEngineTypes.SQLITE;
   keys?: string[];
   requestId?: string;
   callbacks?: IBDCallbacks;
@@ -133,6 +134,7 @@ export class BoilingData {
       messageType: EMessageTypes.SQL_QUERY,
       sql: params.sql,
       keys: params.keys || [],
+      engine: params.engine ?? EEngineTypes.DUCKDB,
       requestId,
     };
     this.socketInstance.queries.set(requestId, {

src/tests/__snapshots__/query.test.ts.snap

@@ -1,6 +1,6 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
-exports[`boilingdata run all meta queries 1`] = `
+exports[`boilingdata with DuckDB run all meta queries 1`] = `
 Array [
   Array [],
   Array [
@@ -60,7 +60,7 @@ Array [
 ]
 `;
 
-exports[`boilingdata run multi-key query 1`] = `
+exports[`boilingdata with DuckDB run multi-key query 1`] = `
 Array [
   Object {
     "count": 44459136,
@@ -73,7 +73,7 @@ Array [
 ]
 `;
 
-exports[`boilingdata run single query 1`] = `
+exports[`boilingdata with DuckDB run single query 1`] = `
 Array [
   Object {
     "DOLocationID": 145,
@@ -117,3 +117,38 @@ Array [
   },
 ]
 `;
+
+exports[`boilingdata with SQLite3 run single query 1`] = `
+Array [
+  Object {
+    "birthdate": "3/8/1971",
+    "cc": "6759521864920116",
+    "comments": "1E+02",
+    "country": "Indonesia",
+    "email": "ajordan0@com.com",
+    "first_name": "Amanda",
+    "gender": "Female",
+    "id": 1,
+    "ip_address": "1.197.201.2",
+    "last_name": "Jordan",
+    "registration_dttm": 1454486129000,
+    "salary": 49756.53,
+    "title": "Internal Auditor",
+  },
+  Object {
+    "birthdate": "1/16/1968",
+    "cc": "",
+    "comments": "",
+    "country": "Canada",
+    "email": "afreeman1@is.gd",
+    "first_name": "Albert",
+    "gender": "Male",
+    "id": 2,
+    "ip_address": "218.111.175.34",
+    "last_name": "Freeman",
+    "registration_dttm": 1454519043000,
+    "salary": 150280.17,
+    "title": "Accountant IV",
+  },
+]
+`;

src/tests/query.test.ts

@@ -1,4 +1,4 @@
-import { globalCallbacksList, IBDDataResponse } from "../boilingdata/boilingdata.api";
+import { EEngineTypes, globalCallbacksList, IBDDataResponse } from "../boilingdata/boilingdata.api";
 import { BoilingData, isDataResponse } from "../boilingdata/boilingdata";
 import { createLogger } from "bunyan";
 
@@ -23,7 +23,7 @@ globalCallbacks.onSocketClose = () => {
 };
 const bdInstance = new BoilingData({ username, password, globalCallbacks, logLevel });
 
-describe("boilingdata", () => {
+describe("boilingdata with DuckDB", () => {
   beforeAll(async () => {
     await bdInstance.connect();
     logger.info("connected.");
@@ -39,6 +39,7 @@ describe("boilingdata", () => {
       const r: any[] = [];
       bdInstance.execQuery({
         sql: `SELECT * FROM parquet_scan('s3://boilingdata-demo/demo2.parquet:m=0') LIMIT 2;`,
+        // engine: EEngineTypes.DUCKDB, // DuckDB is the default
         keys: [],
         callbacks: {
           onData: (data: IBDDataResponse | unknown) => {
@@ -57,6 +58,7 @@ describe("boilingdata", () => {
       const r: any[] = [];
       bdInstance.execQuery({
         sql: `SELECT 's3://KEY' AS key, COUNT(*) AS count FROM parquet_scan('s3://KEY');`,
+        engine: EEngineTypes.DUCKDB,
         keys: ["s3://boilingdata-demo/demo.parquet", "s3://boilingdata-demo/demo2.parquet"],
         callbacks: {
           onData: (data: IBDDataResponse | unknown) => {
@@ -85,6 +87,7 @@ describe("boilingdata", () => {
           bdInstance.execQuery({
             sql,
             keys: [],
+            engine: EEngineTypes.DUCKDB,
             callbacks: {
               onData: (data: IBDDataResponse | unknown) => {
                 if (isDataResponse(data)) data.data.map(row => r.push(row));
@@ -99,3 +102,34 @@ describe("boilingdata", () => {
     expect(rows.sort()).toMatchSnapshot();
   });
 });
+
+describe("boilingdata with SQLite3", () => {
+  beforeAll(async () => {
+    await bdInstance.connect();
+    logger.info("connected.");
+  });
+
+  afterAll(async () => {
+    await bdInstance.close();
+    logger.info("connection closed.");
+  });
+
+  it("run single query", async () => {
+    const rows = await new Promise<any[]>((resolve, reject) => {
+      const r: any[] = [];
+      bdInstance.execQuery({
+        sql: `SELECT * FROM sqlite('s3://boilingdata-demo/uploads/userdata1.sqlite3','userdata1') LIMIT 2;`,
+        engine: EEngineTypes.SQLITE,
+        keys: [],
+        callbacks: {
+          onData: (data: IBDDataResponse | unknown) => {
+            if (isDataResponse(data)) data.data.map(row => r.push(row));
+            resolve(r);
+          },
+          onLogError: (data: any) => reject(data),
+        },
+      });
+    });
+    expect(rows.sort()).toMatchSnapshot();
+  });
+});