-
Notifications
You must be signed in to change notification settings - Fork 117
Expand file tree
/
Copy pathgoogleCloud.ts
More file actions
376 lines (334 loc) · 9.37 KB
/
googleCloud.ts
File metadata and controls
376 lines (334 loc) · 9.37 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
import {
Bucket,
DownloadOptions,
Storage,
type SaveOptions,
} from '@google-cloud/storage';
import type { FastifyBaseLogger } from 'fastify';
import { acceptedResumeExtensions, PropsParameters } from '../types';
import path from 'path';
import { BigQuery } from '@google-cloud/bigquery';
import { Query } from '@google-cloud/bigquery/build/src/bigquery';
import { subDays } from 'date-fns';
import { logger } from '../logger';
import {
EMPLOYMENT_AGREEMENT_BUCKET_NAME,
RESUME_BUCKET_NAME,
} from '../config';
import { isProd } from './utils';
export const gcsBucketMap = {
resume: {
prefixedBlob: (blob: string) => (isProd ? blob : `resume/${blob}`),
bucketName: RESUME_BUCKET_NAME,
},
employmentAgreement: {
prefixedBlob: (blob: string) =>
isProd ? blob : `employment-agreement/${blob}`,
bucketName: EMPLOYMENT_AGREEMENT_BUCKET_NAME,
},
};
export const downloadFile = async ({
url,
options,
}: {
url: string;
options?: DownloadOptions;
}): Promise<string> => {
const bucket = path.dirname(url);
const fileName = path.basename(url);
const storage = new Storage();
const [result] = await storage
.bucket(bucket)
.file(fileName)
.download(options);
return result.toString();
};
export const downloadJsonFile = async <T>({
url,
options,
}: PropsParameters<typeof downloadFile>): Promise<T> => {
const result = await downloadFile({ url, options });
return JSON.parse(result);
};
interface UploadFileFromStreamParams {
bucketName: string;
fileName: string;
file: Buffer;
options?: SaveOptions;
}
export const uploadFileFromBuffer = async ({
bucketName,
fileName,
file,
options,
}: UploadFileFromStreamParams): Promise<string> => {
const storage = new Storage();
await storage.bucket(bucketName).file(fileName).save(file, options);
return `https://storage.cloud.google.com/${bucketName}/${fileName}`;
};
export const uploadResumeFromBuffer = async (
fileName: string,
file: Buffer,
options?: SaveOptions,
): Promise<string> => {
return uploadFileFromBuffer({
bucketName: RESUME_BUCKET_NAME,
fileName,
file,
options,
});
};
export const uploadEmploymentAgreementFromBuffer = async (
fileName: string,
file: Buffer,
options?: SaveOptions,
): Promise<string> => {
const { bucketName, prefixedBlob } = gcsBucketMap.employmentAgreement;
return uploadFileFromBuffer({
bucketName,
fileName: prefixedBlob(fileName),
file,
options,
});
};
/**
* Generate a signed URL for a file in GCS bucket
* Signed URLs are valid for 1 hour by default
*/
export const generateSignedUrl = async ({
bucketName,
blobName,
expiresInMinutes = 60,
}: {
bucketName: string;
blobName: string;
expiresInMinutes?: number;
}): Promise<string | null> => {
try {
const storage = new Storage();
const file = storage.bucket(bucketName).file(blobName);
// Check if file exists first
const [exists] = await file.exists();
if (!exists) {
return null;
}
const [signedUrl] = await file.getSignedUrl({
version: 'v4',
action: 'read',
expires: Date.now() + expiresInMinutes * 60 * 1000,
});
return signedUrl;
} catch (error) {
logger.error(
{ error, bucketName, blobName },
'Failed to generate signed URL',
);
return null;
}
};
/**
* Generate a signed URL for a resume/CV file
*/
export const generateResumeSignedUrl = async (
blobName: string,
expiresInMinutes?: number,
): Promise<string | null> => {
const { bucketName } = gcsBucketMap.resume;
return generateSignedUrl({
bucketName,
blobName,
expiresInMinutes,
});
};
export const deleteFileFromBucket = async (
bucket: Bucket,
fileName: string,
) => {
const file = bucket.file(fileName);
try {
const [exists] = await file.exists();
if (exists) {
await file.delete();
return true;
}
} catch (e) {
logger.error(
{ bucketName: bucket.name, fileName, error: e },
'Failed to delete file from bucket',
);
}
return false;
};
export const deleteResumeByUserId = async (
userId: string,
): Promise<boolean> => {
const bucketName = RESUME_BUCKET_NAME;
if (!userId?.trim()) {
logger.warn('User ID is required to delete resume');
return false;
}
try {
const storage = new Storage();
const bucket = storage.bucket(bucketName);
await deleteFileFromBucket(bucket, userId);
logger.info(
{
userId,
acceptedResumeExtensions,
bucketName,
},
'deleted user resume',
);
return true;
} catch (error) {
logger.error(
{
userId,
acceptedResumeExtensions,
bucketName,
error,
},
'failed to delete user resume',
);
return false;
}
};
export const deleteBlobFromGCS = async ({
blobName,
bucketName,
logger,
}: {
blobName: string;
bucketName: string;
logger: FastifyBaseLogger;
}): Promise<boolean> => {
try {
const storage = new Storage();
const bucket = storage.bucket(bucketName);
await deleteFileFromBucket(bucket, blobName);
return true;
} catch (_err) {
const err = _err as Error;
logger.error(
{ err, bucketName, blobName },
'Failed to delete blob from GCS',
);
return false;
}
};
export const deleteEmploymentAgreementByUserId = async ({
userId,
logger,
}: {
userId: string;
logger: FastifyBaseLogger;
}): Promise<boolean> => {
const { bucketName, prefixedBlob } = gcsBucketMap.employmentAgreement;
return deleteBlobFromGCS({
blobName: prefixedBlob(userId),
bucketName,
logger,
});
};
export enum UserActiveState {
Active = '1',
InactiveSince6wAgo = '2',
InactiveSince12wAgo = '3',
NeverActive = '4',
}
export const userActiveStateQuery = `
with d as (
select u.primary_user_id,
min(last_app_timestamp) as last_app_timestamp,
min(registration_timestamp) as registration_timestamp,
min(
case
when period_end is null then '4'
when period_end between date(@previous_date - interval 6*7 day) and @previous_date then '1'
when period_end between date(@previous_date - interval 12*7 day) and date(@previous_date - interval 6*7 + 1 day) then '2'
when date(u.last_app_timestamp) < date(@previous_date - interval 12*7 day) then '3'
when date(u.registration_timestamp) < date(@previous_date - interval 12*7 day) then '3'
else '4' end
) as previous_state,
min(
case
when period_end is null then '4'
when period_end between date(@run_date - interval 6*7 day) and @run_date then '1'
when period_end between date(@run_date - interval 12*7 day) and date(@run_date - interval 6*7 + 1 day) then '2'
when date(u.last_app_timestamp) < date(@run_date - interval 12*7 day) then '3'
when date(u.registration_timestamp) < date(@run_date - interval 12*7 day) then '3'
else '4' end
) as current_state,
from analytics.user as u
left join analytics.user_state_sparse as uss on uss.primary_user_id = u.primary_user_id
and uss.period_end between date(@previous_date - interval 12* 7 day) and @run_date
and uss.period = 'daily'
and uss.app_active_state = 'active'
and uss.registration_state = 'registered'
where u.registration_timestamp is not null
and date(u.registration_timestamp) < @run_date
group by 1
)
select *
from d
where current_state != previous_state
and previous_state != '4'
`;
export const getUserActiveStateQuery = (
runDate: Date,
query = userActiveStateQuery,
): Query => {
const run_date = runDate.toISOString().split('T')[0];
const previous_date = subDays(runDate, 1).toISOString().split('T')[0];
return { query, params: { previous_date, run_date } };
};
export interface GetUsersActiveState {
inactiveUsers: string[];
downgradeUsers: string[];
reactivateUsers: string[];
}
export interface UserActiveStateData {
current_state: UserActiveState;
previous_state: UserActiveState;
primary_user_id: string;
}
const bigquery = new BigQuery();
export const queryFromBq = async (
query: Query,
): Promise<UserActiveStateData[]> => {
const [job] = await bigquery.createQueryJob(query);
const [rows] = await job.getQueryResults();
return rows;
};
export const sortUsersActiveState = (users: UserActiveStateData[]) => {
const inactiveUsers: string[] = [];
const downgradeUsers: string[] = [];
const reactivateUsers: string[] = [];
// sort users from bq into active, inactive, downgrade, and reactivate
for (const user of users) {
if (
user.current_state === UserActiveState.InactiveSince6wAgo &&
user.previous_state === UserActiveState.Active
) {
downgradeUsers.push(user.primary_user_id);
} else if (
user.current_state === UserActiveState.Active &&
user.previous_state !== UserActiveState.Active
) {
reactivateUsers.push(user.primary_user_id);
} else if (
user.current_state === UserActiveState.InactiveSince12wAgo &&
user.previous_state !== UserActiveState.InactiveSince12wAgo
) {
inactiveUsers.push(user.primary_user_id);
}
}
return { inactiveUsers, downgradeUsers, reactivateUsers };
};
export const getUsersActiveState = async (
runDate: Date,
): Promise<GetUsersActiveState> => {
const query = getUserActiveStateQuery(runDate);
const usersFromBq = await queryFromBq(query);
return sortUsersActiveState(usersFromBq);
};