Merge pull request #3 from boilingdata/partition-filtering-pushdown-tests

feat: Partition filtering pushdown tests

Author: dforsber

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@boilingdata/node-boilingdata",
-  "version": "0.0.13",
+  "version": "0.0.14",
   "description": "BoilingData client",
   "main": "dist/cjs/index.js",
   "types": "dist/cjs/index.d.ts",

src/tests/__snapshots__/query.test.ts.snap

@@ -118,6 +118,56 @@ Array [
 ]
 `;
 
+exports[`boilingdata with Glue Tables can do partition filter push down 1`] = `
+Array [
+  Object {
+    "count": 370164,
+    "s3key": "s3://isecurefi-serverless-analytics/NY-Pub/year=2009/month=8/type=yellow/part-r-00268-6e222bd6-47be-424a-a29a-606961a23de1.gz.parquet",
+  },
+  Object {
+    "count": 370164,
+    "s3key": "s3://isecurefi-serverless-analytics/NY-Pub/year=2009/month=8/type=yellow/part-r-00268-90b05037-c70b-4bc7-978c-b53b496c4751.gz.parquet",
+  },
+  Object {
+    "count": 370330,
+    "s3key": "s3://isecurefi-serverless-analytics/NY-Pub/year=2009/month=8/type=yellow/part-r-00269-6e222bd6-47be-424a-a29a-606961a23de1.gz.parquet",
+  },
+  Object {
+    "count": 370330,
+    "s3key": "s3://isecurefi-serverless-analytics/NY-Pub/year=2009/month=8/type=yellow/part-r-00269-90b05037-c70b-4bc7-978c-b53b496c4751.gz.parquet",
+  },
+  Object {
+    "count": 370214,
+    "s3key": "s3://isecurefi-serverless-analytics/NY-Pub/year=2009/month=8/type=yellow/part-r-00270-6e222bd6-47be-424a-a29a-606961a23de1.gz.parquet",
+  },
+]
+`;
+
+exports[`boilingdata with Glue Tables can read S3 Keys from Glue Table 1`] = `
+Array [
+  Object {
+    "count": 372459,
+    "s3key": "s3://isecurefi-serverless-analytics/NY-Pub/year=2009/month=1/type=yellow/part-r-00000-6e222bd6-47be-424a-a29a-606961a23de1.gz.parquet",
+  },
+  Object {
+    "count": 372459,
+    "s3key": "s3://isecurefi-serverless-analytics/NY-Pub/year=2009/month=1/type=yellow/part-r-00000-90b05037-c70b-4bc7-978c-b53b496c4751.gz.parquet",
+  },
+  Object {
+    "count": 372633,
+    "s3key": "s3://isecurefi-serverless-analytics/NY-Pub/year=2009/month=1/type=yellow/part-r-00001-6e222bd6-47be-424a-a29a-606961a23de1.gz.parquet",
+  },
+  Object {
+    "count": 372633,
+    "s3key": "s3://isecurefi-serverless-analytics/NY-Pub/year=2009/month=1/type=yellow/part-r-00001-90b05037-c70b-4bc7-978c-b53b496c4751.gz.parquet",
+  },
+  Object {
+    "count": 372654,
+    "s3key": "s3://isecurefi-serverless-analytics/NY-Pub/year=2009/month=1/type=yellow/part-r-00002-6e222bd6-47be-424a-a29a-606961a23de1.gz.parquet",
+  },
+]
+`;
+
 exports[`boilingdata with SQLite3 run single query 1`] = `
 Array [
   Object {

src/tests/query.test.ts

@@ -133,3 +133,53 @@ describe("boilingdata with SQLite3", () => {
     expect(rows.sort()).toMatchSnapshot();
   });
 });
+
+describe("boilingdata with Glue Tables", () => {
+  beforeAll(async () => {
+    await bdInstance.connect();
+    logger.info("connected.");
+  });
+
+  afterAll(async () => {
+    await bdInstance.close();
+    logger.info("connection closed.");
+  });
+
+  it("can read S3 Keys from Glue Table", async () => {
+    const rows = await new Promise<any[]>((resolve, _reject) => {
+      const r: any[] = [];
+      bdInstance.execQuery({
+        sql: `SELECT 's3://KEY' AS s3key, COUNT(*) AS count FROM parquet_scan('s3://KEY');`,
+        engine: EEngineTypes.DUCKDB,
+        keys: ["glue.default.nyctaxis"],
+        callbacks: {
+          onData: (data: IBDDataResponse | unknown) => {
+            if (isDataResponse(data)) data.data.map(row => r.push(row));
+          },
+          onQueryFinished: () => resolve(r),
+          // onLogError: (data: any) => reject(data),
+        },
+      });
+    });
+    expect(rows.sort((a, b) => a.s3key.localeCompare(b.s3key))).toMatchSnapshot();
+  });
+
+  it("can do partition filter push down", async () => {
+    const rows = await new Promise<any[]>((resolve, _reject) => {
+      const r: any[] = [];
+      bdInstance.execQuery({
+        sql: `SELECT 's3://KEY' AS s3key, COUNT(*) AS count FROM parquet_scan('s3://KEY') WHERE year=2009 AND month=8;`,
+        engine: EEngineTypes.DUCKDB,
+        keys: ["glue.default.nyctaxis"],
+        callbacks: {
+          onData: (data: IBDDataResponse | unknown) => {
+            if (isDataResponse(data)) data.data.map(row => r.push(row));
+          },
+          onQueryFinished: () => resolve(r),
+          // onLogError: (data: any) => reject(data),
+        },
+      });
+    });
+    expect(rows.sort((a, b) => a.s3key.localeCompare(b.s3key))).toMatchSnapshot();
+  });
+});