Skip to content

Commit 4f4df2d

Browse files
vrtndvrtnd
committed
cache big query
1 parent 792a3e1 commit 4f4df2d

3 files changed

Lines changed: 176 additions & 312 deletions

File tree

‎src/handlers/getBridgeStatsOnDay.ts‎

Lines changed: 73 additions & 193 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,12 @@
11
import { IResponse, successResponse, errorResponse } from "../utils/lambda-response";
22
import wrap from "../utils/wrap";
33
import { getCurrentUnixTimestamp, getTimestampAtStartOfDay } from "../utils/date";
4-
import {
5-
queryAggregatedDailyTimestampRange,
6-
queryAggregatedTokensInRange,
7-
queryConfig,
8-
} from "../utils/wrappa/postgres/query";
4+
import { queryAggregatedTokenStatsTop30 } from "../utils/wrappa/postgres/query";
95
import { getLlamaPrices } from "../utils/prices";
106
import { importBridgeNetwork } from "../data/importBridgeNetwork";
11-
import BigNumber from "bignumber.js";
127
import { normalizeChain, normlizeTokenSymbol } from "../utils/normalizeChain";
138
import { getCache } from "../utils/cache";
149

15-
const numberOfTokensToReturn = 30; // also determines # of addresses returned
16-
17-
// the following 2 types should probably be combined
1810
type TokenRecord = {
1911
[token: string]: {
2012
amount: string;
@@ -24,234 +16,122 @@ type TokenRecord = {
2416
};
2517
};
2618

27-
type TokenRecordBn = {
28-
[token: string]: {
29-
amountBn: BigNumber;
30-
};
31-
};
32-
3319
type AddressRecord = {
3420
[address: string]: {
3521
usdValue: number;
3622
txs: number;
3723
};
3824
};
3925

40-
interface IAggregatedData {
41-
bridge_id: string;
42-
ts: Date;
43-
total_tokens_deposited: string[];
44-
total_tokens_withdrawn: string[];
45-
total_deposited_usd: string;
46-
total_withdrawn_usd: string;
47-
total_deposit_txs: number;
48-
total_withdrawal_txs: number;
49-
total_address_deposited: string[];
50-
total_address_withdrawn: string[];
51-
}
52-
53-
const sumTokenTxs = async (
54-
tokenTotals: string[],
55-
dailyTokensRecord: TokenRecord,
56-
prices: Record<string, any>,
57-
lzSymbols: Record<string, string>
58-
) => {
59-
if (!tokenTotals) return;
60-
let dailyTokensRecordBn = {} as TokenRecordBn;
61-
62-
63-
64-
tokenTotals.map((tokenString) => {
65-
const tokenData = tokenString.replace(/[('") ]/g, "").split(",");
66-
const token = tokenData[0];
67-
68-
const amountBn = BigNumber(tokenData[1]);
69-
const usdValue = parseFloat(tokenData[2]);
70-
dailyTokensRecordBn[token] = dailyTokensRecordBn[token] || {};
71-
dailyTokensRecordBn[token].amountBn = dailyTokensRecordBn[token].amountBn
72-
? dailyTokensRecordBn[token].amountBn.plus(amountBn)
73-
: BigNumber(amountBn);
74-
dailyTokensRecord[token] = dailyTokensRecord[token] || {};
75-
dailyTokensRecord[token].usdValue = (dailyTokensRecord[token].usdValue ?? 0) + usdValue;
76-
});
77-
Object.entries(dailyTokensRecordBn).map(([token, tokenData]) => {
78-
dailyTokensRecord[token].amount = tokenData.amountBn?.toFixed() ?? "0";
79-
const key = token?.toLowerCase?.() ?? token;
80-
const priceEntry = prices?.[key] ?? prices?.[token];
81-
const addrOnly = key.includes(":") ? key.split(":")[1] : key;
82-
const fallbackSymbol = lzSymbols?.[addrOnly] ?? lzSymbols?.[key] ?? "";
83-
const symbol = priceEntry?.symbol ?? fallbackSymbol ?? "";
84-
dailyTokensRecord[token].symbol = normlizeTokenSymbol(symbol);
85-
const decimalsVal = priceEntry?.decimals;
86-
dailyTokensRecord[token].decimals = typeof decimalsVal === "string" ? Number(decimalsVal) : decimalsVal ?? 0;
87-
});
26+
type StatsRow = {
27+
kind: "dt" | "wt" | "da" | "wa";
28+
key: string;
29+
amount: string | null;
30+
usd_value: string;
31+
txs: number | null;
8832
};
8933

90-
const sumAddressTxs = (addressTotals: string[], dailyAddresssRecord: AddressRecord) => {
91-
if (!addressTotals) return;
92-
addressTotals.map((addressString) => {
93-
const addressData = addressString.replace(/[('") ]/g, "").split(",");
94-
const address = addressData[0];
95-
const usdValue = parseFloat(addressData[1]);
96-
const txs = parseInt(addressData[2]);
97-
dailyAddresssRecord[address] = dailyAddresssRecord[address] || {};
98-
dailyAddresssRecord[address].usdValue = (dailyAddresssRecord[address].usdValue ?? 0) + usdValue;
99-
dailyAddresssRecord[address].txs = (dailyAddresssRecord[address].txs ?? 0) + txs;
100-
});
34+
const isValidPriceId = (id?: string) => {
35+
if (!id) return false;
36+
if (id.includes("\\") || id.includes("/") || id.includes(" ")) return false;
37+
const parts = id.split(":");
38+
if (parts.length !== 2) return false;
39+
if (!parts[0] || !parts[1]) return false;
40+
if (parts[1] === "\\N" || parts[1] === "\\n" || parts[1].toLowerCase() === "null" || parts[1].toLowerCase() === "undefined") return false;
41+
return true;
10142
};
10243

103-
// can also return total deposit/withdraw USD, deposit/withdraw #txs here if needed
104-
// don't let chain be 'all'
10544
const getBridgeStatsOnDay = async (timestamp: string = "0", chain: string, bridgeId?: string) => {
10645
let bridgeDbName = undefined as any;
10746
const queryChain = chain === "" ? "" : normalizeChain(chain);
108-
if (!bridgeId) {
109-
bridgeDbName = undefined;
110-
} else {
47+
if (bridgeId) {
11148
try {
11249
const bridgeNetwork = importBridgeNetwork(undefined, parseInt(bridgeId));
11350
if (!bridgeNetwork) {
11451
throw new Error("No bridge network found.");
11552
}
11653
({ bridgeDbName } = bridgeNetwork);
11754
} catch (e) {
118-
return errorResponse({
119-
message: "Invalid bridgeId entered.",
120-
});
55+
return errorResponse({ message: "Invalid bridgeId entered." });
12156
}
12257
}
12358

124-
const sourceChainConfigs = (await queryConfig(undefined, undefined, queryChain)).filter((config) => {
125-
if (bridgeId) {
126-
return config.bridge_name === bridgeDbName;
127-
}
128-
return true;
129-
});
130-
13159
const queryTimestamp = getTimestampAtStartOfDay(parseInt(timestamp));
13260
const maxEndTimestamp = queryTimestamp + 86400;
13361
const currentTimestamp = getCurrentUnixTimestamp();
13462
const endTimestamp = Math.max(queryTimestamp, Math.min(maxEndTimestamp, currentTimestamp));
13563

136-
let sourceChainsDailyData = [] as IAggregatedData[];
137-
await Promise.all(
138-
sourceChainConfigs.map(async (config) => {
139-
const sourceChainData = await queryAggregatedDailyTimestampRange(
140-
queryTimestamp,
141-
endTimestamp,
142-
config.chain,
143-
config.bridge_name
144-
);
145-
sourceChainsDailyData.push(...sourceChainData);
146-
})
147-
);
64+
const rows = (await queryAggregatedTokenStatsTop30(
65+
queryTimestamp,
66+
endTimestamp,
67+
queryChain,
68+
bridgeDbName
69+
)) as StatsRow[];
70+
71+
const dt = rows.filter((r) => r.kind === "dt");
72+
const wt = rows.filter((r) => r.kind === "wt");
73+
const da = rows.filter((r) => r.kind === "da");
74+
const wa = rows.filter((r) => r.kind === "wa");
75+
76+
const tokenIds = new Set<string>();
77+
for (const r of dt) {
78+
const id = r.key?.toLowerCase();
79+
if (isValidPriceId(id)) tokenIds.add(id);
80+
}
81+
for (const r of wt) {
82+
const id = r.key?.toLowerCase();
83+
if (isValidPriceId(id)) tokenIds.add(id);
84+
}
14885

149-
const dailyData = await queryAggregatedTokensInRange(queryTimestamp, endTimestamp, queryChain, bridgeDbName);
150-
let dailyTokensDeposited = {} as TokenRecord;
151-
let dailyTokensWithdrawn = {} as TokenRecord;
152-
let dailyAddressesDeposited = {} as AddressRecord;
153-
let dailyAddressesWithdrawn = {} as AddressRecord;
15486
const lzSymbols = ((await getCache("lz_token_symbols")) || {}) as Record<string, string>;
155-
156-
const allTokenIds = new Set<string>();
157-
const isValidPriceId = (id?: string) => {
158-
if (!id) return false;
159-
if (id.includes("\\") || id.includes("/") || id.includes(" ")) return false;
160-
const parts = id.split(":");
161-
if (parts.length !== 2) return false;
162-
if (!parts[0] || !parts[1]) return false;
163-
if (parts[1] === "\\N" || parts[1] === "\\n" || parts[1].toLowerCase() === "null" || parts[1].toLowerCase() === "undefined") return false;
164-
return true;
165-
};
166-
const collectTokenIds = (tokenTotals?: string[]) => {
167-
if (!tokenTotals) return;
168-
tokenTotals.forEach((tokenString) => {
169-
const tokenData = tokenString.replace(/[('\") ]/g, "").split(",");
170-
const tokenId = (tokenData[0] || "").toLowerCase();
171-
if (isValidPriceId(tokenId)) allTokenIds.add(tokenId);
172-
});
173-
};
174-
dailyData.forEach(({ total_tokens_deposited, total_tokens_withdrawn }) => {
175-
collectTokenIds(total_tokens_deposited);
176-
collectTokenIds(total_tokens_withdrawn);
177-
});
178-
sourceChainsDailyData.forEach(({ total_tokens_deposited, total_tokens_withdrawn }) => {
179-
collectTokenIds(total_tokens_deposited);
180-
collectTokenIds(total_tokens_withdrawn);
181-
});
182-
18387
let prices: Record<string, any> = {};
18488
try {
18589
prices = (await Promise.race([
186-
getLlamaPrices(Array.from(allTokenIds)),
90+
getLlamaPrices(Array.from(tokenIds)),
18791
new Promise((resolve) => setTimeout(() => resolve({}), 5000)),
18892
])) as Record<string, any>;
18993
} catch {
19094
prices = {};
19195
}
19296

193-
const dailyDataPromises = Promise.all(
194-
dailyData.map(async (dayData) => {
195-
const { total_tokens_deposited, total_tokens_withdrawn, total_address_deposited, total_address_withdrawn } =
196-
dayData;
197-
await sumTokenTxs(total_tokens_deposited, dailyTokensDeposited, prices, lzSymbols);
198-
await sumTokenTxs(total_tokens_withdrawn, dailyTokensWithdrawn, prices, lzSymbols);
199-
await sumAddressTxs(total_address_deposited, dailyAddressesDeposited);
200-
await sumAddressTxs(total_address_withdrawn, dailyAddressesWithdrawn);
201-
})
202-
);
203-
await dailyDataPromises;
204-
205-
const sourceChainsPromises = Promise.all(
206-
sourceChainsDailyData.map(async (dayData) => {
207-
const { total_tokens_deposited, total_tokens_withdrawn, total_address_deposited, total_address_withdrawn } =
208-
dayData;
209-
await sumTokenTxs(total_tokens_deposited, dailyTokensWithdrawn, prices, lzSymbols);
210-
await sumTokenTxs(total_tokens_withdrawn, dailyTokensDeposited, prices, lzSymbols);
211-
await sumAddressTxs(total_address_deposited, dailyAddressesWithdrawn);
212-
await sumAddressTxs(total_address_withdrawn, dailyAddressesDeposited);
213-
})
214-
);
215-
await sourceChainsPromises;
97+
const buildTokenRecord = (xs: typeof dt): TokenRecord => {
98+
const out: TokenRecord = {};
99+
for (const r of xs) {
100+
const token = r.key;
101+
const lower = token?.toLowerCase?.() ?? token;
102+
const priceEntry = prices?.[lower] ?? prices?.[token];
103+
const addrOnly = lower.includes(":") ? lower.split(":")[1] : lower;
104+
const fallbackSymbol = lzSymbols?.[addrOnly] ?? lzSymbols?.[lower] ?? "";
105+
const symbol = priceEntry?.symbol ?? fallbackSymbol ?? "";
106+
const decimalsVal = priceEntry?.decimals;
107+
out[token] = {
108+
amount: r.amount ?? "0",
109+
usdValue: Number(r.usd_value),
110+
symbol: normlizeTokenSymbol(symbol),
111+
decimals: typeof decimalsVal === "string" ? Number(decimalsVal) : decimalsVal ?? 0,
112+
};
113+
}
114+
return out;
115+
};
216116

217-
const sortedDailyTokensDeposited = Object.fromEntries(
218-
Object.entries(dailyTokensDeposited)
219-
.sort((a, b) => {
220-
return b[1].usdValue - a[1].usdValue;
221-
})
222-
.slice(0, numberOfTokensToReturn)
223-
);
224-
const sortedDailyTokensWithdrawn = Object.fromEntries(
225-
Object.entries(dailyTokensWithdrawn)
226-
.sort((a, b) => {
227-
return b[1].usdValue - a[1].usdValue;
228-
})
229-
.slice(0, numberOfTokensToReturn)
230-
);
231-
const sortedDailyAddressesDeposited = Object.fromEntries(
232-
Object.entries(dailyAddressesDeposited)
233-
.sort((a, b) => {
234-
return b[1].usdValue - a[1].usdValue;
235-
})
236-
.slice(0, numberOfTokensToReturn)
237-
);
238-
const sortedDailyAddressesWithdrawn = Object.fromEntries(
239-
Object.entries(dailyAddressesWithdrawn)
240-
.sort((a, b) => {
241-
return b[1].usdValue - a[1].usdValue;
242-
})
243-
.slice(0, numberOfTokensToReturn)
244-
);
117+
const buildAddressRecord = (xs: typeof da): AddressRecord => {
118+
const out: AddressRecord = {};
119+
for (const r of xs) {
120+
out[r.key] = {
121+
usdValue: Number(r.usd_value),
122+
txs: r.txs ?? 0,
123+
};
124+
}
125+
return out;
126+
};
245127

246-
const response = {
128+
return {
247129
date: queryTimestamp,
248-
totalTokensDeposited: sortedDailyTokensDeposited,
249-
totalTokensWithdrawn: sortedDailyTokensWithdrawn,
250-
totalAddressDeposited: sortedDailyAddressesDeposited,
251-
totalAddressWithdrawn: sortedDailyAddressesWithdrawn,
130+
totalTokensDeposited: buildTokenRecord(dt),
131+
totalTokensWithdrawn: buildTokenRecord(wt),
132+
totalAddressDeposited: buildAddressRecord(da),
133+
totalAddressWithdrawn: buildAddressRecord(wa),
252134
};
253-
254-
return response;
255135
};
256136

257137
const handler = async (event: AWSLambda.APIGatewayEvent): Promise<IResponse> => {

‎src/server/index.ts‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -139,6 +139,24 @@ const lambdaToFastify = (handler: Function) => async (request: any, reply: any)
139139
}
140140
};
141141

142+
const prewarmCriticalCaches = async () => {
143+
const targets: { routePath: string; pathParameters: Record<string, string>; handler: Function }[] = [
144+
{ routePath: "/bridgetxcounts/:chain", pathParameters: { chain: "all" }, handler: getBridgeTxCounts },
145+
];
146+
147+
for (const { routePath, pathParameters, handler } of targets) {
148+
const event = { pathParameters, queryStringParameters: {}, body: {}, routePath };
149+
const cacheKey = generateApiCacheKey(event);
150+
registerCacheHandler(cacheKey, () => handler(event));
151+
try {
152+
await warmCache(cacheKey);
153+
console.log(`[INFO] Prewarmed ${routePath} ${JSON.stringify(pathParameters)}`);
154+
} catch (error) {
155+
console.error(`[ERROR] Prewarm failed for ${routePath} ${JSON.stringify(pathParameters)}:`, error);
156+
}
157+
}
158+
};
159+
142160
const start = async () => {
143161
try {
144162
await server.register(cors, {
@@ -165,6 +183,8 @@ const start = async () => {
165183

166184
startHealthMonitoring();
167185

186+
prewarmCriticalCaches().catch((e) => console.error("[ERROR] Initial prewarm failed:", e));
187+
168188
setInterval(async () => {
169189
try {
170190
console.log("[INFO] Starting cache warming cycle");

0 commit comments

Comments
 (0)