diff --git a/packages/cache/src/decorators/index.ts b/packages/cache/src/decorators/index.ts index 31105051..239576d1 100644 --- a/packages/cache/src/decorators/index.ts +++ b/packages/cache/src/decorators/index.ts @@ -1,2 +1 @@ export * from './no.cache'; -export * from './scrollable'; diff --git a/packages/cache/src/decorators/scrollable.ts b/packages/cache/src/decorators/scrollable.ts deleted file mode 100644 index a21647b4..00000000 --- a/packages/cache/src/decorators/scrollable.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { DecoratorUtils } from "@multiversx/sdk-nestjs-common"; - -export class ScrollableOptions { - collection: string = ''; -} - -export const Scrollable = DecoratorUtils.registerMethodDecorator(ScrollableOptions); diff --git a/packages/cache/src/index.ts b/packages/cache/src/index.ts index d0f68460..bb4edfc2 100644 --- a/packages/cache/src/index.ts +++ b/packages/cache/src/index.ts @@ -5,6 +5,5 @@ export * from './cache'; export * from './jitter'; export * from './interceptors/caching.interceptor'; export * from './interceptors/guest.cache.interceptor'; -export * from './interceptors/scroll.interceptor'; export * from './guest-cache'; export * from './decorators'; diff --git a/packages/cache/src/interceptors/caching.interceptor.ts b/packages/cache/src/interceptors/caching.interceptor.ts index 73f3eff8..31b22735 100644 --- a/packages/cache/src/interceptors/caching.interceptor.ts +++ b/packages/cache/src/interceptors/caching.interceptor.ts @@ -43,12 +43,6 @@ export class CachingInterceptor implements NestInterceptor { return next.handle(); } - for (const paramName of Object.keys(request.query)) { - if (['scrollCreate', 'scrollAt', 'scrollAfter'].includes(paramName)) { - return next.handle(); - } - } - this.metricsService.setPendingRequestsCount(Object.keys(this.pendingRequestsDictionary).length); const cacheKey = this.getCacheKey(context); diff --git a/packages/cache/src/interceptors/scroll.interceptor.ts b/packages/cache/src/interceptors/scroll.interceptor.ts deleted file mode 100644 index 906ddea8..00000000 --- a/packages/cache/src/interceptors/scroll.interceptor.ts +++ /dev/null @@ -1,151 +0,0 @@ -import { Constants, ContextTracker, DecoratorUtils, ScrollableAfterSettings, ScrollableCreateSettings } from "@multiversx/sdk-nestjs-common"; -import { BadRequestException, CallHandler, ExecutionContext, Injectable, NestInterceptor } from "@nestjs/common"; -import { Observable, catchError, tap, throwError } from "rxjs"; -import { randomUUID } from "crypto"; -import { CacheService } from "../cache"; -import { ScrollableOptions } from "../decorators"; - -@Injectable() -export class ScrollInterceptor implements NestInterceptor { - guidRegex = /^[0-9a-f]{8}-[0-9a-f]{4}-[4][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; - constructor( - private readonly cacheService: CacheService, - ) { } - - async intercept(context: ExecutionContext, next: CallHandler): Promise> { - const httpContext = context.switchToHttp(); - const response = httpContext.getResponse(); - const request = httpContext.getRequest(); - - const scrollable = DecoratorUtils.getMethodDecorator(ScrollableOptions, context.getHandler()); - if (!scrollable) { - return next.handle(); - } - - const scrollCollection = scrollable.collection; - if (!scrollCollection) { - return next.handle(); - } - - const queryParams = JSON.parse(JSON.stringify(request.query)); - delete queryParams.scrollCreate; - delete queryParams.scrollAfter; - delete queryParams.scrollAt; - - let scrollId = this.handleScrollCreate(request, scrollCollection); - scrollId = await this.handleScrollAt(request, scrollCollection, queryParams); - scrollId = await this.handleScrollAfter(request, scrollCollection, queryParams); - - return next - .handle() - .pipe( - tap(async () => { - const contextObj = ContextTracker.get(); - const scrollResult = contextObj?.scrollResult; - if (scrollResult) { - if (scrollId) { - response.setHeader('X-Scroll-Id', scrollId); - } - - const scrollResultToSave = { - ...scrollResult, - queryParams, - }; - - await this.cacheService.set(`scrollInfo:${scrollId}`, scrollResultToSave, Constants.oneMinute() * 10); - } - }), - catchError((err) => { - return throwError(() => err); - }) - ); - } - - private handleScrollCreate(request: any, scrollCollection: string): string | undefined { - const scrollCreate = request.query.scrollCreate; - if (!scrollCreate) { - return; - } - - let scrollId = undefined; - - if (scrollCreate === 'true') { - scrollId = randomUUID(); - } else if (this.guidRegex.test(scrollCreate)) { - scrollId = scrollCreate; - } - - if (!scrollId) { - throw new Error('Invalid scrollCreate value'); - } - - ContextTracker.assign({ - scrollSettings: new ScrollableCreateSettings({ - collection: scrollCollection, - create: true, - }), - }); - - return scrollId; - } - - private async handleScrollAfter(request: any, scrollCollection: string, queryParams: any): Promise { - let scrollId = undefined; - - const scrollAfter = request.query.scrollAfter; - if (scrollAfter && this.guidRegex.test(scrollAfter)) { - scrollId = scrollAfter; - } else { - throw new Error('Invalid scrollAfter value'); - } - - const scrollInfo: any = await this.cacheService.get(`scrollInfo:${scrollId}`); - if (!scrollInfo) { - throw new BadRequestException(`Could not find scroll info for '${scrollId}'`); - } - - if (JSON.stringify(scrollInfo.queryParams) !== JSON.stringify(queryParams)) { - throw new BadRequestException('Invalid query params'); - } - - ContextTracker.assign({ - scrollSettings: new ScrollableAfterSettings({ - collection: scrollCollection, - after: scrollInfo.lastSort, - ids: scrollInfo.lastIds, - }), - }); - - return scrollId; - } - - private async handleScrollAt(request: any, scrollCollection: string, queryParams: any): Promise { - let scrollId = undefined; - - const scrollAt = request.query.scrollAt; - if (scrollAt && this.guidRegex.test(scrollAt)) { - scrollId = scrollAt; - } else { - throw new Error('Invalid scrollAt value'); - } - - const scrollInfo: any = await this.cacheService.get(`scrollInfo:${scrollId}`); - if (!scrollInfo) { - throw new BadRequestException(`Could not find scroll info for '${scrollId}'`); - } - - if (scrollInfo.queryParams.sort !== queryParams.sort) { - throw new BadRequestException('Invalid query params'); - } - - ContextTracker.assign({ - scrollSettings: new ScrollableAfterSettings({ - collection: scrollCollection, - after: scrollInfo.firstSort, - ids: scrollInfo.lastIds, - }), - }); - - return scrollId; - } -} diff --git a/packages/elastic/src/elastic.service.ts b/packages/elastic/src/elastic.service.ts index 7599e727..2678de4a 100644 --- a/packages/elastic/src/elastic.service.ts +++ b/packages/elastic/src/elastic.service.ts @@ -1,9 +1,8 @@ -import { forwardRef, Inject, Injectable } from "@nestjs/common"; +import { BadRequestException, forwardRef, Inject, Injectable } from "@nestjs/common"; import { ApiService } from "@multiversx/sdk-nestjs-http"; import { MetricsService, ElasticMetricType, PerformanceProfiler } from "@multiversx/sdk-nestjs-monitoring"; import { ElasticQuery } from "./entities/elastic.query"; import { ElasticModuleOptions } from "./entities/elastic.module.options"; -import { ContextTracker, ScrollableAfterSettings } from "@multiversx/sdk-nestjs-common"; @Injectable() export class ElasticService { @@ -52,129 +51,98 @@ export class ElasticService { } private formatItem(document: any, key: string) { - const { _id, _source } = document; + const { _id, _source, sort } = document; const item: any = {}; item[key] = _id; - return { ...item, ..._source }; - } - - private async getListResult(url: string, collection: string, elasticQuery: ElasticQuery) { - const scrollSettings = ContextTracker.get()?.scrollSettings; - - if (scrollSettings && scrollSettings.collection === collection) { - let documents: any[] = []; + const result = { ...item, ..._source }; - if (scrollSettings.after) { - documents = await this.getScrollAfterResult(url, elasticQuery, scrollSettings); - } else if (scrollSettings.create) { - documents = await this.getScrollCreateResult(url, elasticQuery); - } else { - throw new Error('Invalid scroll settings'); - } + if (sort !== undefined) { + result.searchAfter = this.encodeCursor(sort); + } - this.storeScrollResult(documents); + return result; + } - return documents; + private async getListResult(url: string, elasticQuery: ElasticQuery, searchAfter?: string | any[]) { + if (searchAfter) { + return this.getScrollAfterResult(url, elasticQuery, this.decodeCursor(searchAfter)); } const elasticQueryJson: any = elasticQuery.toJson(); + this.ensureUuidTieBreaker(elasticQueryJson); const result = await this.post(url, elasticQueryJson); return result.data.hits.hits; } - private async getScrollCreateResult(url: string, elasticQuery: ElasticQuery) { - const result = await this.post(url, elasticQuery.toJson()); - const documents = result.data.hits.hits; + private async getScrollAfterResult(url: string, elasticQuery: ElasticQuery, searchAfter: any[]) { + const elasticQueryJson: any = elasticQuery.toJson(); + this.ensureUuidTieBreaker(elasticQueryJson); - if (documents.length === 0) { - return documents; - } + elasticQueryJson.search_after = searchAfter; - return documents; + const queryResult = await this.post(url, elasticQueryJson); + return queryResult.data.hits.hits; } - private storeScrollResult(documents: any[]) { - const ids = this.getLastIds(documents); - const firstDocumentSort = documents[0].sort; - const lastDocumentSort = documents[documents.length - 1].sort; - - // and store this in cache on a specific key - ContextTracker.assign({ - scrollResult: { - lastIds: ids, - lastSort: lastDocumentSort, - firstSort: firstDocumentSort, - }, - }); + private formatDocuments(documents: any[], key: string): any[] { + return documents.map((document: any) => this.formatItem(document, key)); } - private async getScrollAfterResult(url: string, elasticQuery: ElasticQuery, scrollSettings: ScrollableAfterSettings) { - const elasticQueryJson: any = elasticQuery.toJson(); + private encodeCursor(sort: any[]): string { + return Buffer.from(JSON.stringify(sort), 'utf8').toString('base64'); + } - elasticQueryJson.search_after = scrollSettings.after; - this.excludeIds(elasticQueryJson, scrollSettings.ids); + private decodeCursor(searchAfter: string | any[]): any[] { + if (Array.isArray(searchAfter)) { + return searchAfter; + } - const queryResult = await this.post(url, elasticQueryJson); - return queryResult.data.hits.hits; - } + try { + const decoded = JSON.parse(Buffer.from(searchAfter, 'base64').toString('utf8')); + if (!Array.isArray(decoded)) { + throw new Error('Invalid cursor payload'); + } - private excludeIds(elasticQueryJson: any, ids: any[]) { - if (!elasticQueryJson.query) { - elasticQueryJson.query = {}; + return decoded; + } catch { + throw new BadRequestException('Invalid searchAfter'); } + } - if (!elasticQueryJson.query.bool) { - elasticQueryJson.query.bool = {}; + private ensureUuidTieBreaker(elasticQueryJson: any): void { + const sorts: any[] = elasticQueryJson.sort; + if (!Array.isArray(sorts) || sorts.length === 0) { + return; } - if (!elasticQueryJson.query.bool.must_not) { - elasticQueryJson.query.bool.must_not = []; + const hasUuidTieBreaker = sorts.some((sort) => Object.keys(sort)[0] === 'uuid.keyword'); + if (hasUuidTieBreaker) { + return; } - elasticQueryJson.query.bool.must_not.push({ - terms: { - _id: ids, + const lastSortKey = Object.keys(sorts[sorts.length - 1])[0]; + const lastSortOrder = sorts[sorts.length - 1][lastSortKey]?.order ?? 'desc'; + sorts.push({ + 'uuid.keyword': { + order: lastSortOrder, }, }); } - async getList(collection: string, key: string, elasticQuery: ElasticQuery, overrideUrl?: string): Promise { + async getList(collection: string, key: string, elasticQuery: ElasticQuery, overrideUrl?: string, searchAfter?: string | any[]): Promise { const url = `${overrideUrl ?? this.options.url}/${collection}/_search`; // attempt to get scroll settings const profiler = new PerformanceProfiler(); - const documents = await this.getListResult(url, collection, elasticQuery); + const documents = await this.getListResult(url, elasticQuery, searchAfter); profiler.stop(); this.metricsService.setElasticDuration(collection, ElasticMetricType.list, profiler.duration); - return documents.map((document: any) => this.formatItem(document, key)); - } - - private getLastIds(documents: any[]) { - const lastDocument = documents[documents.length - 1]; - const lastDocumentSort = lastDocument.sort; - const lastDocumentSortJson = JSON.stringify(lastDocumentSort); - - // then take the ids of all elements that have the same sort - const ids: string[] = []; - - for (let index = documents.length - 1; index >= 0; index--) { - const document = documents[index]; - - const documentSortJson = JSON.stringify(document.sort); - - if (documentSortJson === lastDocumentSortJson) { - ids.push(document._id); - } else { - break; - } - } - - return ids; + return this.formatDocuments(documents, key); } async getScrollableList(collection: string, key: string, elasticQuery: ElasticQuery, action: (items: any[]) => Promise, options?: { scrollTimeout?: string, delayBetweenScrolls?: number }): Promise { diff --git a/packages/http/src/interceptors/query.check.interceptor.ts b/packages/http/src/interceptors/query.check.interceptor.ts index ca4b476b..14ead00b 100644 --- a/packages/http/src/interceptors/query.check.interceptor.ts +++ b/packages/http/src/interceptors/query.check.interceptor.ts @@ -30,7 +30,7 @@ export class QueryCheckInterceptor implements NestInterceptor { const supportedQueryNames = Object.values(metadata).map((x: any) => x.data); for (const paramName of Object.keys(request.query)) { - if (!['fields', 'extract', 'excludeFields', 'scrollCreate', 'scrollAt', 'scrollAfter'].includes(paramName) && !supportedQueryNames.includes(paramName)) { + if (!['fields', 'extract', 'excludeFields'].includes(paramName) && !supportedQueryNames.includes(paramName)) { delete request.query[paramName]; // throw new BadRequestException(`Unsupported parameter '${paramName}'. Supported parameters are: ${supportedQueryNames.join(', ')}`); // const origin = request.headers['origin'];