Skip to content

Commit 1c1b840

Browse files
dforsberdforsber
committed
update all regions tests
1 parent 5276713 commit 1c1b840

4 files changed

Lines changed: 296 additions & 33 deletions

File tree

‎src/boilingdata/boilingdata.api.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ export interface IBDDataQuery {
2525

2626
export interface IBDDataResponse {
2727
messageType: "DATA";
28-
data: Array<string>;
28+
data: Array<any>;
2929
requestId: string;
3030
numOfRecords?: number;
3131
batchSerial?: number;

‎src/common/identity.ts‎

Lines changed: 6 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,9 @@ import { CognitoIdToken, CognitoUserPool, CognitoUser, AuthenticationDetails } f
33
import { getSignedWssUrl } from "./signature";
44
import { BDAWSRegion } from "boilingdata/boilingdata";
55

6-
// FIXME: Hard coded
7-
const region = "eu-west-1";
6+
const IDP_REGION = "eu-west-1";
87
const UserPoolId = "eu-west-1_0GLV9KO1p";
9-
const Logins = `cognito-idp.${region}.amazonaws.com/${UserPoolId}`;
8+
const Logins = `cognito-idp.${IDP_REGION}.amazonaws.com/${UserPoolId}`;
109
const IdentityPoolId = "eu-west-1:bce21571-e3a6-47a4-8032-fd015213405f";
1110
const poolData = { UserPoolId, ClientId: "6timr8knllr4frovfvq8r2o6oo" };
1211
const Pool = new CognitoUserPool(poolData);
@@ -31,38 +30,14 @@ function getIdToken(Username: string, Password: string): Promise<CognitoIdToken>
3130

3231
async function refreshCredsWithToken(idToken: string): Promise<CognitoIdentityCredentials> {
3332
const idParams = { IdentityPoolId, Logins: { [Logins]: idToken } };
34-
const creds = new CognitoIdentityCredentials(idParams, { region });
33+
const creds = new CognitoIdentityCredentials(idParams, { region: IDP_REGION });
3534
await creds.getPromise();
3635
return creds;
3736
}
3837

3938
function getWsApiDomain(region: string): string {
40-
switch (region) {
41-
case "eu-north-1":
42-
return "ei0k349i7d.execute-api.eu-north-1.amazonaws.com";
43-
case "eu-west-2":
44-
return "ej8h1tsab5.execute-api.eu-west-2.amazonaws.com";
45-
case "eu-west-3":
46-
return "qbqvpmm3o4.execute-api.eu-west-3.amazonaws.com";
47-
case "eu-south-1":
48-
return "dy5annfn83.execute-api.eu-south-1.amazonaws.com";
49-
case "eu-central-1":
50-
return "qy8elgcxyi.execute-api.eu-central-1.amazonaws.com";
51-
case "us-east-1":
52-
return "2waxvxnpa3.execute-api.us-east-1.amazonaws.com";
53-
case "us-east-2":
54-
return "bilcgl1zpf.execute-api.us-east-2.amazonaws.com";
55-
case "us-west-1":
56-
return "lal0xyf3d2.execute-api.us-west-1.amazonaws.com";
57-
case "us-west-2":
58-
return "wkz85e6919.execute-api.us-west-2.amazonaws.com";
59-
case "ca-central-1":
60-
return "vmbg81io85.execute-api.ca-central-1.amazonaws.com";
61-
case "eu-west-1":
62-
default:
63-
// eu-west-1
64-
return "m9fhs4t5vh.execute-api.eu-west-1.amazonaws.com";
65-
}
39+
return `${region}.api.boilingdata.com`;
40+
// return `api.boilingdata.com`;
6641
}
6742

6843
export async function getBoilingDataCredentials(
@@ -78,7 +53,7 @@ export async function getBoilingDataCredentials(
7853
const sessionToken = creds.data?.Credentials?.SessionToken;
7954
if (!accessKeyId || !secretAccessKey) throw new Error("Missing credentials (after refresh)!");
8055
const credentials = { accessKeyId, secretAccessKey, sessionToken };
81-
const signedWebsocketUrl = await getSignedWssUrl(webSocketHost, credentials, region);
56+
const signedWebsocketUrl = await getSignedWssUrl(webSocketHost, credentials, region, "wss", "");
8257
const cognitoUsername = idToken.decodePayload()["cognito:username"];
8358
return { cognitoUsername, signedWebsocketUrl };
8459
}

‎src/tests/__snapshots__/query.test.ts.snap‎

Lines changed: 216 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,186 @@
11
// Jest Snapshot v1, https://goo.gl/fbAQLP
22

3+
exports[`BoilingData in all North-America and Europe AWS Regions can query cross-region 1`] = `
4+
Array [
5+
Object {
6+
"birthdate": "3/8/1971",
7+
"cc": "6759521864920116",
8+
"comments": "1E+02",
9+
"country": "Indonesia",
10+
"email": "ajordan0@com.com",
11+
"first_name": "Amanda",
12+
"gender": "Female",
13+
"id": 1,
14+
"ip_address": "1.197.201.2",
15+
"key": "s3://boilingdata-demo/test.parquet",
16+
"last_name": "Jordan",
17+
"registration_dttm": 1454486129000,
18+
"salary": 49756.53,
19+
"title": "Internal Auditor",
20+
},
21+
Object {
22+
"birthdate": "3/8/1971",
23+
"cc": "6759521864920116",
24+
"comments": "1E+02",
25+
"country": "Indonesia",
26+
"email": "ajordan0@com.com",
27+
"first_name": "Amanda",
28+
"gender": "Female",
29+
"id": 1,
30+
"ip_address": "1.197.201.2",
31+
"key": "s3://ca-central-1-boilingdata-demo/test.parquet",
32+
"last_name": "Jordan",
33+
"registration_dttm": 1454486129000,
34+
"salary": 49756.53,
35+
"title": "Internal Auditor",
36+
},
37+
Object {
38+
"birthdate": "3/8/1971",
39+
"cc": "6759521864920116",
40+
"comments": "1E+02",
41+
"country": "Indonesia",
42+
"email": "ajordan0@com.com",
43+
"first_name": "Amanda",
44+
"gender": "Female",
45+
"id": 1,
46+
"ip_address": "1.197.201.2",
47+
"key": "s3://eu-central-1-boilingdata-demo/test.parquet",
48+
"last_name": "Jordan",
49+
"registration_dttm": 1454486129000,
50+
"salary": 49756.53,
51+
"title": "Internal Auditor",
52+
},
53+
Object {
54+
"birthdate": "3/8/1971",
55+
"cc": "6759521864920116",
56+
"comments": "1E+02",
57+
"country": "Indonesia",
58+
"email": "ajordan0@com.com",
59+
"first_name": "Amanda",
60+
"gender": "Female",
61+
"id": 1,
62+
"ip_address": "1.197.201.2",
63+
"key": "s3://eu-north-1-boilingdata-demo/test.parquet",
64+
"last_name": "Jordan",
65+
"registration_dttm": 1454486129000,
66+
"salary": 49756.53,
67+
"title": "Internal Auditor",
68+
},
69+
Object {
70+
"birthdate": "3/8/1971",
71+
"cc": "6759521864920116",
72+
"comments": "1E+02",
73+
"country": "Indonesia",
74+
"email": "ajordan0@com.com",
75+
"first_name": "Amanda",
76+
"gender": "Female",
77+
"id": 1,
78+
"ip_address": "1.197.201.2",
79+
"key": "s3://eu-south-1-boilingdata-demo/test.parquet",
80+
"last_name": "Jordan",
81+
"registration_dttm": 1454486129000,
82+
"salary": 49756.53,
83+
"title": "Internal Auditor",
84+
},
85+
Object {
86+
"birthdate": "3/8/1971",
87+
"cc": "6759521864920116",
88+
"comments": "1E+02",
89+
"country": "Indonesia",
90+
"email": "ajordan0@com.com",
91+
"first_name": "Amanda",
92+
"gender": "Female",
93+
"id": 1,
94+
"ip_address": "1.197.201.2",
95+
"key": "s3://eu-west-2-boilingdata-demo/test.parquet",
96+
"last_name": "Jordan",
97+
"registration_dttm": 1454486129000,
98+
"salary": 49756.53,
99+
"title": "Internal Auditor",
100+
},
101+
Object {
102+
"birthdate": "3/8/1971",
103+
"cc": "6759521864920116",
104+
"comments": "1E+02",
105+
"country": "Indonesia",
106+
"email": "ajordan0@com.com",
107+
"first_name": "Amanda",
108+
"gender": "Female",
109+
"id": 1,
110+
"ip_address": "1.197.201.2",
111+
"key": "s3://eu-west-3-boilingdata-demo/test.parquet",
112+
"last_name": "Jordan",
113+
"registration_dttm": 1454486129000,
114+
"salary": 49756.53,
115+
"title": "Internal Auditor",
116+
},
117+
Object {
118+
"birthdate": "3/8/1971",
119+
"cc": "6759521864920116",
120+
"comments": "1E+02",
121+
"country": "Indonesia",
122+
"email": "ajordan0@com.com",
123+
"first_name": "Amanda",
124+
"gender": "Female",
125+
"id": 1,
126+
"ip_address": "1.197.201.2",
127+
"key": "s3://us-east-1-boilingdata-demo/test.parquet",
128+
"last_name": "Jordan",
129+
"registration_dttm": 1454486129000,
130+
"salary": 49756.53,
131+
"title": "Internal Auditor",
132+
},
133+
Object {
134+
"birthdate": "3/8/1971",
135+
"cc": "6759521864920116",
136+
"comments": "1E+02",
137+
"country": "Indonesia",
138+
"email": "ajordan0@com.com",
139+
"first_name": "Amanda",
140+
"gender": "Female",
141+
"id": 1,
142+
"ip_address": "1.197.201.2",
143+
"key": "s3://us-east-2-boilingdata-demo/test.parquet",
144+
"last_name": "Jordan",
145+
"registration_dttm": 1454486129000,
146+
"salary": 49756.53,
147+
"title": "Internal Auditor",
148+
},
149+
Object {
150+
"birthdate": "3/8/1971",
151+
"cc": "6759521864920116",
152+
"comments": "1E+02",
153+
"country": "Indonesia",
154+
"email": "ajordan0@com.com",
155+
"first_name": "Amanda",
156+
"gender": "Female",
157+
"id": 1,
158+
"ip_address": "1.197.201.2",
159+
"key": "s3://us-west-1-boilingdata-demo/test.parquet",
160+
"last_name": "Jordan",
161+
"registration_dttm": 1454486129000,
162+
"salary": 49756.53,
163+
"title": "Internal Auditor",
164+
},
165+
Object {
166+
"birthdate": "3/8/1971",
167+
"cc": "6759521864920116",
168+
"comments": "1E+02",
169+
"country": "Indonesia",
170+
"email": "ajordan0@com.com",
171+
"first_name": "Amanda",
172+
"gender": "Female",
173+
"id": 1,
174+
"ip_address": "1.197.201.2",
175+
"key": "s3://us-west-2-boilingdata-demo/test.parquet",
176+
"last_name": "Jordan",
177+
"registration_dttm": 1454486129000,
178+
"salary": 49756.53,
179+
"title": "Internal Auditor",
180+
},
181+
]
182+
`;
183+
3184
exports[`BoilingData in all North-America and Europe AWS Regions runs query succesfully in other regions too 1`] = `
4185
Array [
5186
Object {
@@ -429,3 +610,38 @@ Array [
429610
},
430611
]
431612
`;
613+
614+
exports[`boilingdata with SQLite3 run single query, same one, 2nd time 1`] = `
615+
Array [
616+
Object {
617+
"birthdate": "3/8/1971",
618+
"cc": "6759521864920116",
619+
"comments": "1E+02",
620+
"country": "Indonesia",
621+
"email": "ajordan0@com.com",
622+
"first_name": "Amanda",
623+
"gender": "Female",
624+
"id": 1,
625+
"ip_address": "1.197.201.2",
626+
"last_name": "Jordan",
627+
"registration_dttm": 1454486129000,
628+
"salary": 49756.53,
629+
"title": "Internal Auditor",
630+
},
631+
Object {
632+
"birthdate": "1/16/1968",
633+
"cc": "",
634+
"comments": "",
635+
"country": "Canada",
636+
"email": "afreeman1@is.gd",
637+
"first_name": "Albert",
638+
"gender": "Male",
639+
"id": 2,
640+
"ip_address": "218.111.175.34",
641+
"last_name": "Freeman",
642+
"registration_dttm": 1454519043000,
643+
"salary": 150280.17,
644+
"title": "Accountant IV",
645+
},
646+
]
647+
`;

‎src/tests/query.test.ts‎

Lines changed: 73 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import { createLogger } from "bunyan";
44

55
jest.setTimeout(30000);
66

7-
const logLevel = "error";
7+
const logLevel = "info";
88
const logger = createLogger({ name: "TEST", level: logLevel });
99
const username = process.env["BD_USERNAME"];
1010
const password = process.env["BD_PASSWORD"];
@@ -134,6 +134,25 @@ describe("boilingdata with SQLite3", () => {
134134
});
135135
expect(rows.sort()).toMatchSnapshot();
136136
});
137+
138+
it("run single query, same one, 2nd time", async () => {
139+
const rows = await new Promise<any[]>((resolve, reject) => {
140+
const r: any[] = [];
141+
bdInstance.execQuery({
142+
sql: `SELECT * FROM sqlite('s3://boilingdata-demo/uploads/userdata1.sqlite3','userdata1') LIMIT 2;`,
143+
engine: EEngineTypes.SQLITE,
144+
keys: [],
145+
callbacks: {
146+
onData: (data: IBDDataResponse | unknown) => {
147+
if (isDataResponse(data)) data.data.map(row => r.push(row));
148+
resolve(r);
149+
},
150+
onLogError: (data: any) => reject(data),
151+
},
152+
});
153+
});
154+
expect(rows.sort()).toMatchSnapshot();
155+
});
137156
});
138157

139158
describe("boilingdata with Glue Tables", () => {
@@ -201,6 +220,7 @@ describe("BoilingData in all North-America and Europe AWS Regions", () => {
201220
"us-west-2",
202221
"ca-central-1",
203222
];
223+
204224
it("runs query succesfully in other regions too", async () => {
205225
await Promise.all(
206226
regions.map(async region => {
@@ -232,4 +252,56 @@ describe("BoilingData in all North-America and Europe AWS Regions", () => {
232252
}),
233253
);
234254
});
255+
256+
it("can query cross-region", async () => {
257+
const sourceRegion = "eu-west-3";
258+
const bdInstance = new BoilingData({ username, password, globalCallbacks, logLevel, region: sourceRegion });
259+
await bdInstance.connect();
260+
let allKeys = [
261+
"s3://boilingdata-demo/test.parquet",
262+
"s3://eu-west-2-boilingdata-demo/test.parquet",
263+
"s3://eu-west-3-boilingdata-demo/test.parquet",
264+
"s3://eu-north-1-boilingdata-demo/test.parquet",
265+
"s3://eu-south-1-boilingdata-demo/test.parquet",
266+
"s3://eu-central-1-boilingdata-demo/test.parquet",
267+
"s3://us-east-1-boilingdata-demo/test.parquet",
268+
"s3://us-east-2-boilingdata-demo/test.parquet",
269+
"s3://us-west-1-boilingdata-demo/test.parquet",
270+
"s3://us-west-2-boilingdata-demo/test.parquet",
271+
"s3://ca-central-1-boilingdata-demo/test.parquet",
272+
];
273+
const totalCount = allKeys.length;
274+
let count = 0;
275+
logger.info(`connected to region ${sourceRegion}`);
276+
const rows = await new Promise<any[]>((resolve, reject) => {
277+
const r: any[] = [];
278+
while (true) {
279+
const keys = allKeys.splice(0, 5);
280+
if (keys.length <= 0) break;
281+
console.log(totalCount, keys);
282+
bdInstance.execQuery({
283+
sql: `SELECT 's3://KEY' AS key, * FROM parquet_scan('s3://KEY') LIMIT 1;`,
284+
engine: EEngineTypes.DUCKDB,
285+
keys,
286+
callbacks: {
287+
onData: (data: IBDDataResponse | unknown) => {
288+
if (isDataResponse(data)) {
289+
data.data.map(row => r.push(row));
290+
count = count + 1;
291+
console.log("---------------------- ", count, "/", totalCount, data?.data[0].key);
292+
if (count >= totalCount) resolve(r);
293+
}
294+
},
295+
onLogError: (data: any) => reject(data),
296+
},
297+
});
298+
}
299+
});
300+
const sorted = rows.sort((a, b) => a.key.localeCompare(b.key));
301+
console.log(sorted);
302+
expect(sorted).toMatchSnapshot();
303+
304+
await bdInstance.close();
305+
logger.info(`connection closed to region ${sourceRegion}`);
306+
});
235307
});

0 commit comments

Comments
 (0)