diff --git a/src/core/domain/dtos/elastic/es-query.dto.ts b/src/core/domain/dtos/elastic/es-query.dto.ts index 0572933..693d24c 100644 --- a/src/core/domain/dtos/elastic/es-query.dto.ts +++ b/src/core/domain/dtos/elastic/es-query.dto.ts @@ -1,5 +1,5 @@ import { ApiExtraModels, ApiProperty, ApiPropertyOptional } from "@nestjs/swagger"; -import { IsArray, IsDefined, IsInt, IsNotEmpty, IsNumber, IsObject, IsOptional } from "class-validator"; +import { IsArray, IsDefined, IsInt, IsObject, IsOptional } from "class-validator"; import { EsPit } from "../../interfaces/elastic/es-pit.interface"; import { EsQuery } from "../../interfaces/elastic/es-query.interface" @@ -13,71 +13,80 @@ import { EsQuery } from "../../interfaces/elastic/es-query.interface" */ @ApiExtraModels() export class EsQueryDto { - /** - * Maximum number of elements returned by Elasticsearch - */ - @IsOptional() - @IsDefined() - @IsNumber() - @IsInt() - @ApiPropertyOptional({ - description: 'Maximum number of elements returned by Elasticsearch', - example: 30 - }) - size?: number; - - /** - * The search query object passed to Elasticsearch - */ - @IsDefined() - @IsObject() - @ApiProperty({ - description: 'Search query object passed to Elasticsearch', - example: {}, - }) - query: EsQuery; + /** + * Offset from the start of the list of hits + */ + @IsOptional() + @IsInt() + @ApiPropertyOptional({ + description: 'Offset from the start of the list of hits', + example: 5, + }) + from?: number; - /** - * Object, that stores PIT ID and time alive - */ - @IsOptional() - @IsObject() - @ApiPropertyOptional({ - description: 'PIT object', - example: {} - }) - pit?: EsPit; + /** + * Maximum number of elements returned by Elasticsearch + */ + @IsOptional() + @IsInt() + @ApiPropertyOptional({ + description: 'Maximum number of elements returned by Elasticsearch', + example: 30 + }) + size?: number; + + /** + * The search query object passed to Elasticsearch + */ + @IsDefined() + @IsObject() + @ApiProperty({ + description: 'Search query object passed to Elasticsearch', + example: {}, + }) + query: EsQuery; - /** - * Sorting info - */ - @IsOptional() - @IsArray() - @ApiPropertyOptional({ - description: '', - example: [] - }) - sort?: unknown[]; + /** + * Object, that stores PIT ID and time alive + */ + @IsOptional() + @IsObject() + @ApiPropertyOptional({ + description: 'PIT object', + example: {} + }) + pit?: EsPit; - /** - * Pagination info - */ - @IsOptional() - @IsArray() - @ApiPropertyOptional({ - description: '', - example: [] - }) - search_after?: unknown[]; + /** + * Sorting info + */ + @IsOptional() + @IsArray() + @ApiPropertyOptional({ + description: '', + example: [] + }) + sort?: unknown[]; - /** - * Constructs an empty object - */ - constructor() { - this.size = 10; - this.query = undefined; - this.pit = undefined; - this.sort = undefined; - this.search_after = undefined; - } + /** + * Pagination info + */ + @IsOptional() + @IsArray() + @ApiPropertyOptional({ + description: '', + example: [] + }) + search_after?: unknown[]; + + /** + * Constructs an empty object + */ + constructor() { + this.size = 10; + this.query = undefined; + this.pit = undefined; + this.sort = undefined; + this.search_after = undefined; + } } \ No newline at end of file diff --git a/src/core/domain/dtos/page-meta.dto.ts b/src/core/domain/dtos/page-meta.dto.ts index 2a89285..0e4c321 100644 --- a/src/core/domain/dtos/page-meta.dto.ts +++ b/src/core/domain/dtos/page-meta.dto.ts @@ -1,8 +1,6 @@ -import { ApiExtraModels, ApiProperty, PartialType } from "@nestjs/swagger"; +import { ApiExtraModels, ApiProperty } from "@nestjs/swagger"; import { IsArray } from "class-validator"; import { Order } from "../enums"; -import { PageMeta } from "../interfaces/page-meta.interface"; -import { PaperDto } from "./paper.dto"; /** * List of allowed properties in this DTO @@ -13,7 +11,7 @@ const allowedProperties = ['total', 'pagenum', 'order', 'hasNext', 'hasPrev', 'p * Page model for pagination */ @ApiExtraModels() -export class PageMetaDto implements PageMeta { +export class PageMetaDto { /** * Total number of hits (results) acquired from the search */ @@ -24,16 +22,6 @@ export class PageMetaDto implements PageMeta { }) total: number; - /** - * Current page number - */ - @ApiProperty({ - description: 'Current page number', - minimum: 1, - example: 3 - }) - pagenum: number; - /** * Order of the elements on the page */ @@ -42,32 +30,4 @@ export class PageMetaDto implements PageMeta { example: Order.DESC }) order: Order; - - /** - * Flag, that shows if there's a page following the current one - */ - @ApiProperty({ - description: 'Flag, that shows if there\'s a page following the current one', - example: true - }) - hasNext: boolean; - - /** - * Flag, that shows if there's a page preceding the current one - */ - @ApiProperty({ - description: 'Flag, that shows if there\'s a page preceding the current one', - example: true - }) - hasPrev: boolean; - - /** - * Maximum number of elements on the page - */ - @ApiProperty({ - description: 'Maximum number of elements on the page', - minimum: 1, - example: 20 - }) - pagesize: number; } \ No newline at end of file diff --git a/src/core/domain/dtos/page.dto.ts b/src/core/domain/dtos/page.dto.ts index 785b1e0..189f7db 100644 --- a/src/core/domain/dtos/page.dto.ts +++ b/src/core/domain/dtos/page.dto.ts @@ -1,7 +1,5 @@ import { ApiExtraModels, ApiProperty, PartialType } from "@nestjs/swagger"; import { IsArray } from "class-validator"; -import { Order } from "../enums"; -import { PageMeta } from "../interfaces/page-meta.interface"; import { PageMetaDto } from "./page-meta.dto"; import { PaperDto } from "./paper.dto"; @@ -22,7 +20,7 @@ export class PageDto { @ApiProperty({ description: 'All data (papers) the page contains', isArray: true, - type: PaperDto + type: PaperDto, }) readonly data: PaperDto[]; @@ -31,7 +29,7 @@ export class PageDto { */ @ApiProperty({ description: 'Metadata for the page', - // example: {}, + type: PageMetaDto, }) readonly meta: PageMetaDto; @@ -41,7 +39,7 @@ export class PageDto { * @param data * @param meta */ - constructor(data: PaperDto[], meta: PageMeta) { + constructor(data: PaperDto[], meta: PageMetaDto) { this.data = data; this.meta = meta; } diff --git a/src/core/domain/dtos/search-q.dto.ts b/src/core/domain/dtos/search-q.dto.ts index 8834657..ee4d929 100644 --- a/src/core/domain/dtos/search-q.dto.ts +++ b/src/core/domain/dtos/search-q.dto.ts @@ -1,4 +1,4 @@ -import { ApiExtraModels, ApiProperty } from "@nestjs/swagger"; +import { ApiExtraModels, ApiPropertyOptional } from "@nestjs/swagger"; import { IsDefined, IsInt, IsNotEmpty, IsOptional, IsString } from "class-validator"; /** @@ -12,63 +12,64 @@ const allowedProperties = ['query', 'pagen', 'limit', 'order']; @ApiExtraModels() export class SearchQueryDto { /** - * Given query string to perform the - * search on. + * Given query string to perform the search on. */ @IsDefined() @IsNotEmpty() @IsString() - @ApiProperty({ - description: 'query', - example: 'Particle Accelerator' + @ApiPropertyOptional({ + description: 'Given query string to perform the search on', + example: 'Particle Accelerator', }) query: string; - - /** - * Page number to display. - */ - @IsDefined() - @IsNotEmpty() - @IsInt() - @ApiProperty({ - description: 'page', - example: 3, - }) - page: number; /** * Limits the number of displayed elements. */ @IsOptional() @IsInt() - @ApiProperty({ - description: 'limit', + @ApiPropertyOptional({ + description: 'Limits the number of displayed elements', example: 10, }) - limit: number; + limit?: number; /** - * Limits the number of displayed elements. + * Offset from the start of the list of hits. + */ + @IsOptional() + @IsInt() + @ApiPropertyOptional({ + description: 'Offset from the start of the list of hits', + example: 0, + }) + offset?: number; + + /** + * Indicates in which order elements need to be displayed. */ @IsOptional() @IsString() - @ApiProperty({ - description: 'order', + @ApiPropertyOptional({ + description: 'Indicates in which order elements need to be displayed', example: 'asc', }) - order: string; + order?: string; - /** - * Constructs an object with provided parameters - * @param query - * @param page - * @param limit - * @param order - */ - constructor(query: string, page: number, limit: number, order: string) { + /** + * + */ + + /** + * Constructs an object with provided parameters + * @param query + * @param page + * @param limit + * @param order + */ + constructor(query: string, page: number, limit: number, order: string) { this.query = query; - this.page = page; this.limit = limit; this.order = order; - } + } } \ No newline at end of file diff --git a/src/core/domain/enums/page-order.enum.ts b/src/core/domain/enums/page-order.enum.ts index ff4a505..234f858 100644 --- a/src/core/domain/enums/page-order.enum.ts +++ b/src/core/domain/enums/page-order.enum.ts @@ -11,4 +11,11 @@ export enum Order { * Descending order */ DESC = 'desc', +} + +export function toOrder(str: string): Order { + switch (str) { + case 'asc': return Order.ASC; + case 'desc': return Order.DESC; + } } \ No newline at end of file diff --git a/src/core/domain/interfaces/index.ts b/src/core/domain/interfaces/index.ts index 6295a5f..3b9d09e 100644 --- a/src/core/domain/interfaces/index.ts +++ b/src/core/domain/interfaces/index.ts @@ -1,5 +1,4 @@ export * from './http-response.interface' -export * from './page-meta.interface' export * from './search-info.interface' export * from './elastic/es-query.interface' export * from './elastic/es-query-string.interface' diff --git a/src/core/domain/interfaces/page-meta.interface.ts b/src/core/domain/interfaces/page-meta.interface.ts deleted file mode 100644 index 62cc9a5..0000000 --- a/src/core/domain/interfaces/page-meta.interface.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { Order } from "../enums/page-order.enum"; - -/** - * Structure of page metadata - */ -export interface PageMeta { - /** - * Total search results - */ - total: number; - - /** - * Number of the page - */ - pagenum: number; - - /** - * Order of the elements on the page - */ - order: Order; - - /** - * Flag that indicates presence of the next page - */ - hasNext: boolean; - - /** - * Flag that indicates presence of the previous page - */ - hasPrev: boolean; - - /** - * Number of elements on the page - */ - pagesize: number; -} \ No newline at end of file diff --git a/src/core/interceptors/page.interceptor.ts b/src/core/interceptors/page.interceptor.ts index 01eda50..6efaf93 100644 --- a/src/core/interceptors/page.interceptor.ts +++ b/src/core/interceptors/page.interceptor.ts @@ -6,10 +6,10 @@ import { EsQueryDto } from "../domain/dtos/elastic/es-query.dto"; import { RequestDto } from "../domain/dtos/request.dto"; import { SearchQueryDto } from "../domain/dtos/search-q.dto"; import { EsTime } from "../domain/enums/es-time.enum"; -import { Order } from "../domain/enums/page-order.enum"; -import { PageMeta } from "../domain/interfaces"; +import { Order, toOrder } from "../domain/enums/page-order.enum"; import { EsPit } from "../domain/interfaces/elastic/es-pit.interface"; import { Cache } from 'cache-manager' +import { PageMetaDto } from "../domain/dtos/page-meta.dto"; /** * Pagination-implementing interceptor @@ -45,8 +45,12 @@ export class PageInterceptor implements NestInterceptor { async intercept(context: ExecutionContext, next: CallHandler): Promise> { const request: RequestDto = context.switchToHttp().getRequest(); const query: SearchQueryDto = request.query; - let reverse: boolean = false; + const offset = !query.offset ? 0 : query.offset; + const limit = !query.limit ? 10 : query.limit; + const order = !query.order ? Order.DESC : query.order; + + // Contruct a body for querying Elasticsearch request.es_query = new EsQueryDto(); request.es_query.query = { query_string: { @@ -54,63 +58,33 @@ export class PageInterceptor implements NestInterceptor { default_field: 'content', } }; + request.es_query.from = offset; + request.es_query.size = limit; request.es_query.sort = [ - { _score: { order: !query?.order ? Order.DESC : query.order } }, - { _shard_doc: 'desc' } - ]; + { "_score": { "order": order } }, + ]; - const limit = !query?.limit ? 10 : query.limit; - - if (await this.cacheManager.get('prevPage')) { - if (query.page == (await this.cacheManager.get('_pagenum'))) return await this.cacheManager.get('prevPage'); - - request.es_query.pit = await this.cacheManager.get('_pit'); - request.es_query.search_after = await this.cacheManager.get('_sa'); - request.es_query.size = limit * Math.abs(query.page - (await this.cacheManager.get('_pagenum'))); - - if (query.page < (await this.cacheManager.get('_pagenum'))) { - request.es_query.sort = [{ _score: { order: Order.ASC } }]; - request.es_query.size += limit - 1; - reverse = true; - } - } else { - request.es_query.pit = await this.getPIT(1); - request.es_query.size = limit * query.page; + const prev_page = await this.cacheManager.get('prev_page'); + if (prev_page) { + if (offset == prev_page[1] && limit == prev_page[2]) return prev_page[0]; } return next.handle().pipe( switchMap(async (res) => { // Setting the page meta-data - let meta: PageMeta = { + let meta: PageMetaDto = { total: res.hits.total.value, - pagenum: !query?.page ? 1 : +query.page, - order: query?.order?.toUpperCase() === Order.ASC ? Order.ASC : Order.DESC, - pagesize: !query?.limit ? 10 : query.limit, - hasNext: undefined, - hasPrev: undefined, + order: toOrder(order), }; - meta.hasNext = meta.pagenum * meta.pagesize < meta.total ? true : false; - meta.hasPrev = meta.pagenum != 1 ? true : false; - - // Saving the search info - await this.cacheManager.set('_pit', { id: res.pit_id, keep_alive: `1${EsTime.min}` }) - await this.cacheManager.set('_sa', res.hits.hits[res.hits.hits.length - 1]?.sort); - await this.cacheManager.set('_pagenum', query.page); // Check if the performed search is a backwards search - let data = res.hits.hits.slice(-meta.pagesize); - if (reverse) { - this.cacheManager.set('_sa', data[0]?.sort); - data.reverse(); - reverse = false; - } - + let data = res.hits.hits; // Omitting the redundant info and leaving only the document data = data.map((el) => el._source); // Cache and return the page const page: PageDto = new PageDto(data, meta); - await this.cacheManager.set('prevPage', page); + await this.cacheManager.set('prev_page', [page, offset, limit]); return page; }) ); @@ -121,19 +95,19 @@ export class PageInterceptor implements NestInterceptor { * @param alive, amount of time in minutes (defaults to 1). If time unit is not specified - defaults to minutes. * @returns PIT object containing PIT ID and keep_alive value */ - public async getPIT(alive: number, unit: EsTime = EsTime.min): Promise { - return new Promise((resolve, reject) => { - try { - this.httpService.post(`http://${this.ES_IP}:${this.ES_PORT}/papers/_pit?keep_alive=${alive+unit}`) - .pipe(take(1), map(axiosRes => axiosRes.data)) - .subscribe((res: EsPit) => { - res.keep_alive = alive + unit; - resolve(res); - }); - } catch (error) { - reject(error); - } - }); + public async getPIT(alive: number, unit: EsTime = EsTime.min): Promise { + return new Promise((resolve, reject) => { + try { + this.httpService.post(`http://${this.ES_IP}:${this.ES_PORT}/papers/_pit?keep_alive=${alive+unit}`) + .pipe(take(1), map(axiosRes => axiosRes.data)) + .subscribe((res: EsPit) => { + res.keep_alive = alive + unit; + resolve(res); + }); + } catch (error) { + reject(error); + } + }); } /** @@ -141,20 +115,20 @@ export class PageInterceptor implements NestInterceptor { * @param pitID, ID of the PIT, that would be deleted * @returns true/false, depending on the result of deletion of the PIT */ - async deletePIT(pitID: string): Promise { - return new Promise((resolve, reject) => { - try { - this.httpService.delete(`http://${this.ES_IP}:${this.ES_PORT}/_pit`, { - data: { id: pitID }, - headers: { 'Content-Type': 'application/json' }, - }) - .pipe(take(1), map(axiosRes => axiosRes.data)) - .subscribe((res) => { - resolve(res.succeeded); - }); - } catch (error) { - reject(error); - } - }) + async deletePIT(pitID: string): Promise { + return new Promise((resolve, reject) => { + try { + this.httpService.delete(`http://${this.ES_IP}:${this.ES_PORT}/_pit`, { + data: { id: pitID }, + headers: { 'Content-Type': 'application/json' }, + }) + .pipe(take(1), map(axiosRes => axiosRes.data)) + .subscribe((res) => { + resolve(res.succeeded); + }); + } catch (error) { + reject(error); + } + }) } } \ No newline at end of file diff --git a/src/test/e2e/papers.controller.e2e.spec.ts b/src/test/e2e/papers.controller.e2e.spec.ts index 2934683..d5bba5c 100644 --- a/src/test/e2e/papers.controller.e2e.spec.ts +++ b/src/test/e2e/papers.controller.e2e.spec.ts @@ -205,20 +205,8 @@ describe('E2E Testing of /papers', () => { expect(test.body.meta.total).toBeDefined(); expect(test.body.meta.total).toBe(2); - expect(test.body.meta.pagenum).toBeDefined(); - expect(test.body.meta.pagenum).toBe(1); - expect(test.body.meta.order).toBeDefined(); expect(test.body.meta.order).toBe(Order.DESC); - - expect(test.body.meta.pagesize).toBeDefined(); - expect(test.body.meta.pagesize).toBe(10); - - expect(test.body.meta.hasNext).toBeDefined(); - expect(test.body.meta.hasNext).toBe(false); - - expect(test.body.meta.hasPrev).toBeDefined(); - expect(test.body.meta.hasPrev).toBe(false); }); afterAll(async () => { diff --git a/src/test/page.interceptor.spec.ts b/src/test/page.interceptor.spec.ts index 5a60f02..d60c8f1 100644 --- a/src/test/page.interceptor.spec.ts +++ b/src/test/page.interceptor.spec.ts @@ -159,40 +159,7 @@ describe('Unit tests for PageInterceptor', () => { }); }); }); - }); - - it('Should reverse the search results', async () => { - execCtxMock.getRequest.mockReturnValueOnce({ - query: { - page: 1, - order: 'desc', - limit: 3 - } - }); - - await pageInter['cacheManager'].set('_pagenum', 3); - await pageInter['cacheManager'].set('prevPage', { set: "yes" }); - - callHandlerMock.handle.mockReturnValueOnce( - of({ - hits: { - total: { value: 1 }, - hits: [ - { sort: ['1', 'less relevant'], _source: '1' }, - { sort: ['2', 'average'], _source: '2' }, - { sort: ['3', 'most relevant'], _source: '3' } - ] - } - }) - ); - - pageInter.intercept(execCtxMock, callHandlerMock).then((res) => { - res.subscribe(async (page) => { - expect(await pageInter['cacheManager'].get('_sa')).toEqual(['1', 'less relevant']); - expect(page.data).toEqual(['3', '2', '1']); - }); - }); - }); + }); }); describe('getPIT()', () => {