danny-mhlv/scipaper
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Pagination redone

Browse Source
This commit is contained in:
danny-mhlv 2022-09-21 15:14:57 +03:00
parent 9e547a0a44
commit d996bf679d
10 changed files with 168 additions and 301 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

137
src/core/domain/dtos/elastic/es-query.dto.ts
View File

@ -1,5 +1,5 @@
import { ApiExtraModels, ApiProperty, ApiPropertyOptional } from "@nestjs/swagger";
import { IsArray, IsDefined, IsInt, IsNotEmpty, IsNumber, IsObject, IsOptional } from "class-validator";
import { IsArray, IsDefined, IsInt, IsObject, IsOptional } from "class-validator";
import { EsPit } from "../../interfaces/elastic/es-pit.interface";
import { EsQuery } from "../../interfaces/elastic/es-query.interface"
@ -13,71 +13,80 @@ import { EsQuery } from "../../interfaces/elastic/es-query.interface"
*/
@ApiExtraModels()
export class EsQueryDto {
/**
* Maximum number of elements returned by Elasticsearch
*/
@IsOptional()
@IsDefined()
@IsNumber()
@IsInt()
@ApiPropertyOptional({
description: 'Maximum number of elements returned by Elasticsearch',
example: 30
})
size?: number;
/**
* The search query object passed to Elasticsearch
*/
@IsDefined()
@IsObject()
@ApiProperty({
description: 'Search query object passed to Elasticsearch',
example: {},
})
query: EsQuery;
/**
* Offset from the start of the list of hits
*/
@IsOptional()
@IsInt()
@ApiPropertyOptional({
description: 'Offset from the start of the list of hits',
example: 5,
})
from?: number;
/**
* Object, that stores PIT ID and time alive
*/
@IsOptional()
@IsObject()
@ApiPropertyOptional({
description: 'PIT object',
example: {}
})
pit?: EsPit;
/**
* Maximum number of elements returned by Elasticsearch
*/
@IsOptional()
@IsInt()
@ApiPropertyOptional({
description: 'Maximum number of elements returned by Elasticsearch',
example: 30
})
size?: number;
/**
* The search query object passed to Elasticsearch
*/
@IsDefined()
@IsObject()
@ApiProperty({
description: 'Search query object passed to Elasticsearch',
example: {},
})
query: EsQuery;
/**
* Sorting info
*/
@IsOptional()
@IsArray()
@ApiPropertyOptional({
description: '',
example: []
})
sort?: unknown[];
/**
* Object, that stores PIT ID and time alive
*/
@IsOptional()
@IsObject()
@ApiPropertyOptional({
description: 'PIT object',
example: {}
})
pit?: EsPit;
/**
* Pagination info
*/
@IsOptional()
@IsArray()
@ApiPropertyOptional({
description: '',
example: []
})
search_after?: unknown[];
/**
* Sorting info
*/
@IsOptional()
@IsArray()
@ApiPropertyOptional({
description: '',
example: []
})
sort?: unknown[];
/**
* Constructs an empty object
*/
constructor() {
this.size = 10;
this.query = undefined;
this.pit = undefined;
this.sort = undefined;
this.search_after = undefined;
}
/**
* Pagination info
*/
@IsOptional()
@IsArray()
@ApiPropertyOptional({
description: '',
example: []
})
search_after?: unknown[];
/**
* Constructs an empty object
*/
constructor() {
this.size = 10;
this.query = undefined;
this.pit = undefined;
this.sort = undefined;
this.search_after = undefined;
}
}

44
src/core/domain/dtos/page-meta.dto.ts
View File

@ -1,8 +1,6 @@
import { ApiExtraModels, ApiProperty, PartialType } from "@nestjs/swagger";
import { ApiExtraModels, ApiProperty } from "@nestjs/swagger";
import { IsArray } from "class-validator";
import { Order } from "../enums";
import { PageMeta } from "../interfaces/page-meta.interface";
import { PaperDto } from "./paper.dto";
/**
* List of allowed properties in this DTO
@ -13,7 +11,7 @@ const allowedProperties = ['total', 'pagenum', 'order', 'hasNext', 'hasPrev', 'p
* Page model for pagination
*/
@ApiExtraModels()
export class PageMetaDto implements PageMeta {
export class PageMetaDto {
/**
* Total number of hits (results) acquired from the search
*/
@ -24,16 +22,6 @@ export class PageMetaDto implements PageMeta {
})
total: number;
/**
* Current page number
*/
@ApiProperty({
description: 'Current page number',
minimum: 1,
example: 3
})
pagenum: number;
/**
* Order of the elements on the page
*/
@ -42,32 +30,4 @@ export class PageMetaDto implements PageMeta {
example: Order.DESC
})
order: Order;
/**
* Flag, that shows if there's a page following the current one
*/
@ApiProperty({
description: 'Flag, that shows if there\'s a page following the current one',
example: true
})
hasNext: boolean;
/**
* Flag, that shows if there's a page preceding the current one
*/
@ApiProperty({
description: 'Flag, that shows if there\'s a page preceding the current one',
example: true
})
hasPrev: boolean;
/**
* Maximum number of elements on the page
*/
@ApiProperty({
description: 'Maximum number of elements on the page',
minimum: 1,
example: 20
})
pagesize: number;
}

8
src/core/domain/dtos/page.dto.ts
View File

@ -1,7 +1,5 @@
import { ApiExtraModels, ApiProperty, PartialType } from "@nestjs/swagger";
import { IsArray } from "class-validator";
import { Order } from "../enums";
import { PageMeta } from "../interfaces/page-meta.interface";
import { PageMetaDto } from "./page-meta.dto";
import { PaperDto } from "./paper.dto";
@ -22,7 +20,7 @@ export class PageDto {
@ApiProperty({
description: 'All data (papers) the page contains',
isArray: true,
type: PaperDto
type: PaperDto,
})
readonly data: PaperDto[];
@ -31,7 +29,7 @@ export class PageDto {
*/
@ApiProperty({
description: 'Metadata for the page',
// example: {},
type: PageMetaDto,
})
readonly meta: PageMetaDto;
@ -41,7 +39,7 @@ export class PageDto {
* @param data
* @param meta
*/
constructor(data: PaperDto[], meta: PageMeta) {
constructor(data: PaperDto[], meta: PageMetaDto) {
this.data = data;
this.meta = meta;
}

71
src/core/domain/dtos/search-q.dto.ts
View File

@ -1,4 +1,4 @@
import { ApiExtraModels, ApiProperty } from "@nestjs/swagger";
import { ApiExtraModels, ApiPropertyOptional } from "@nestjs/swagger";
import { IsDefined, IsInt, IsNotEmpty, IsOptional, IsString } from "class-validator";
/**
@ -12,63 +12,64 @@ const allowedProperties = ['query', 'pagen', 'limit', 'order'];
@ApiExtraModels()
export class SearchQueryDto {
/**
* Given query string to perform the
* search on.
* Given query string to perform the search on.
*/
@IsDefined()
@IsNotEmpty()
@IsString()
@ApiProperty({
description: 'query',
example: 'Particle Accelerator'
@ApiPropertyOptional({
description: 'Given query string to perform the search on',
example: 'Particle Accelerator',
})
query: string;
/**
* Page number to display.
*/
@IsDefined()
@IsNotEmpty()
@IsInt()
@ApiProperty({
description: 'page',
example: 3,
})
page: number;
/**
* Limits the number of displayed elements.
*/
@IsOptional()
@IsInt()
@ApiProperty({
description: 'limit',
@ApiPropertyOptional({
description: 'Limits the number of displayed elements',
example: 10,
})
limit: number;
limit?: number;
/**
* Limits the number of displayed elements.
* Offset from the start of the list of hits.
*/
@IsOptional()
@IsInt()
@ApiPropertyOptional({
description: 'Offset from the start of the list of hits',
example: 0,
})
offset?: number;
/**
* Indicates in which order elements need to be displayed.
*/
@IsOptional()
@IsString()
@ApiProperty({
description: 'order',
@ApiPropertyOptional({
description: 'Indicates in which order elements need to be displayed',
example: 'asc',
})
order: string;
order?: string;
/**
* Constructs an object with provided parameters
* @param query
* @param page
* @param limit
* @param order
*/
constructor(query: string, page: number, limit: number, order: string) {
/**
*
*/
/**
* Constructs an object with provided parameters
* @param query
* @param page
* @param limit
* @param order
*/
constructor(query: string, page: number, limit: number, order: string) {
this.query = query;
this.page = page;
this.limit = limit;
this.order = order;
}
}
}

7
src/core/domain/enums/page-order.enum.ts
View File

@ -11,4 +11,11 @@ export enum Order {
* Descending order
*/
DESC = 'desc',
}
export function toOrder(str: string): Order {
switch (str) {
case 'asc': return Order.ASC;
case 'desc': return Order.DESC;
}
}

1
src/core/domain/interfaces/index.ts
View File

@ -1,5 +1,4 @@
export * from './http-response.interface'
export * from './page-meta.interface'
export * from './search-info.interface'
export * from './elastic/es-query.interface'
export * from './elastic/es-query-string.interface'

36
src/core/domain/interfaces/page-meta.interface.ts
View File

@ -1,36 +0,0 @@
import { Order } from "../enums/page-order.enum";
/**
* Structure of page metadata
*/
export interface PageMeta {
/**
* Total search results
*/
total: number;
/**
* Number of the page
*/
pagenum: number;
/**
* Order of the elements on the page
*/
order: Order;
/**
* Flag that indicates presence of the next page
*/
hasNext: boolean;
/**
* Flag that indicates presence of the previous page
*/
hasPrev: boolean;
/**
* Number of elements on the page
*/
pagesize: number;
}

118
src/core/interceptors/page.interceptor.ts
View File

@ -6,10 +6,10 @@ import { EsQueryDto } from "../domain/dtos/elastic/es-query.dto";
import { RequestDto } from "../domain/dtos/request.dto";
import { SearchQueryDto } from "../domain/dtos/search-q.dto";
import { EsTime } from "../domain/enums/es-time.enum";
import { Order } from "../domain/enums/page-order.enum";
import { PageMeta } from "../domain/interfaces";
import { Order, toOrder } from "../domain/enums/page-order.enum";
import { EsPit } from "../domain/interfaces/elastic/es-pit.interface";
import { Cache } from 'cache-manager'
import { PageMetaDto } from "../domain/dtos/page-meta.dto";
/**
* Pagination-implementing interceptor
@ -45,8 +45,12 @@ export class PageInterceptor implements NestInterceptor {
async intercept(context: ExecutionContext, next: CallHandler<any>): Promise<Observable<PageDto>> {
const request: RequestDto = context.switchToHttp().getRequest<RequestDto>();
const query: SearchQueryDto = request.query;
let reverse: boolean = false;
const offset = !query.offset ? 0 : query.offset;
const limit = !query.limit ? 10 : query.limit;
const order = !query.order ? Order.DESC : query.order;
// Contruct a body for querying Elasticsearch
request.es_query = new EsQueryDto();
request.es_query.query = {
query_string: {
@ -54,63 +58,33 @@ export class PageInterceptor implements NestInterceptor {
default_field: 'content',
}
};
request.es_query.from = offset;
request.es_query.size = limit;
request.es_query.sort = [
{ _score: { order: !query?.order ? Order.DESC : query.order } },
{ _shard_doc: 'desc' }
];
{ "_score": { "order": order } },
];
const limit = !query?.limit ? 10 : query.limit;
if (await this.cacheManager.get('prevPage')) {
if (query.page == (await this.cacheManager.get('_pagenum'))) return await this.cacheManager.get('prevPage');
request.es_query.pit = await this.cacheManager.get('_pit');
request.es_query.search_after = await this.cacheManager.get('_sa');
request.es_query.size = limit * Math.abs(query.page - (await this.cacheManager.get('_pagenum')));
if (query.page < (await this.cacheManager.get('_pagenum'))) {
request.es_query.sort = [{ _score: { order: Order.ASC } }];
request.es_query.size += limit - 1;
reverse = true;
}
} else {
request.es_query.pit = await this.getPIT(1);
request.es_query.size = limit * query.page;
const prev_page = await this.cacheManager.get('prev_page');
if (prev_page) {
if (offset == prev_page[1] && limit == prev_page[2]) return prev_page[0];
}
return next.handle().pipe(
switchMap(async (res) => {
// Setting the page meta-data
let meta: PageMeta = {
let meta: PageMetaDto = {
total: res.hits.total.value,
pagenum: !query?.page ? 1 : +query.page,
order: query?.order?.toUpperCase() === Order.ASC ? Order.ASC : Order.DESC,
pagesize: !query?.limit ? 10 : query.limit,
hasNext: undefined,
hasPrev: undefined,
order: toOrder(order),
};
meta.hasNext = meta.pagenum * meta.pagesize < meta.total ? true : false;
meta.hasPrev = meta.pagenum != 1 ? true : false;
// Saving the search info
await this.cacheManager.set('_pit', { id: res.pit_id, keep_alive: `1${EsTime.min}` })
await this.cacheManager.set('_sa', res.hits.hits[res.hits.hits.length - 1]?.sort);
await this.cacheManager.set('_pagenum', query.page);
// Check if the performed search is a backwards search
let data = res.hits.hits.slice(-meta.pagesize);
if (reverse) {
this.cacheManager.set('_sa', data[0]?.sort);
data.reverse();
reverse = false;
}
let data = res.hits.hits;
// Omitting the redundant info and leaving only the document
data = data.map((el) => el._source);
// Cache and return the page
const page: PageDto = new PageDto(data, meta);
await this.cacheManager.set('prevPage', page);
await this.cacheManager.set('prev_page', [page, offset, limit]);
return page;
})
);
@ -121,19 +95,19 @@ export class PageInterceptor implements NestInterceptor {
* @param alive, amount of time in minutes (defaults to 1). If time unit is not specified - defaults to minutes.
* @returns PIT object <EsPit> containing PIT ID and keep_alive value
*/
public async getPIT(alive: number, unit: EsTime = EsTime.min): Promise<EsPit> {
return new Promise((resolve, reject) => {
try {
this.httpService.post<EsPit>(`http://${this.ES_IP}:${this.ES_PORT}/papers/_pit?keep_alive=${alive+unit}`)
.pipe(take(1), map(axiosRes => axiosRes.data))
.subscribe((res: EsPit) => {
res.keep_alive = alive + unit;
resolve(res);
});
} catch (error) {
reject(error);
}
});
public async getPIT(alive: number, unit: EsTime = EsTime.min): Promise<EsPit> {
return new Promise((resolve, reject) => {
try {
this.httpService.post<EsPit>(`http://${this.ES_IP}:${this.ES_PORT}/papers/_pit?keep_alive=${alive+unit}`)
.pipe(take(1), map(axiosRes => axiosRes.data))
.subscribe((res: EsPit) => {
res.keep_alive = alive + unit;
resolve(res);
});
} catch (error) {
reject(error);
}
});
}
/**
@ -141,20 +115,20 @@ export class PageInterceptor implements NestInterceptor {
* @param pitID, ID of the PIT, that would be deleted
* @returns true/false, depending on the result of deletion of the PIT
*/
async deletePIT(pitID: string): Promise<boolean> {
return new Promise((resolve, reject) => {
try {
this.httpService.delete(`http://${this.ES_IP}:${this.ES_PORT}/_pit`, {
data: { id: pitID },
headers: { 'Content-Type': 'application/json' },
})
.pipe(take(1), map(axiosRes => axiosRes.data))
.subscribe((res) => {
resolve(res.succeeded);
});
} catch (error) {
reject(error);
}
})
async deletePIT(pitID: string): Promise<boolean> {
return new Promise((resolve, reject) => {
try {
this.httpService.delete(`http://${this.ES_IP}:${this.ES_PORT}/_pit`, {
data: { id: pitID },
headers: { 'Content-Type': 'application/json' },
})
.pipe(take(1), map(axiosRes => axiosRes.data))
.subscribe((res) => {
resolve(res.succeeded);
});
} catch (error) {
reject(error);
}
})
}
}

12
src/test/e2e/papers.controller.e2e.spec.ts
View File

@ -205,20 +205,8 @@ describe('E2E Testing of /papers', () => {
expect(test.body.meta.total).toBeDefined();
expect(test.body.meta.total).toBe(2);
expect(test.body.meta.pagenum).toBeDefined();
expect(test.body.meta.pagenum).toBe(1);
expect(test.body.meta.order).toBeDefined();
expect(test.body.meta.order).toBe(Order.DESC);
expect(test.body.meta.pagesize).toBeDefined();
expect(test.body.meta.pagesize).toBe(10);
expect(test.body.meta.hasNext).toBeDefined();
expect(test.body.meta.hasNext).toBe(false);
expect(test.body.meta.hasPrev).toBeDefined();
expect(test.body.meta.hasPrev).toBe(false);
});
afterAll(async () => {

35
src/test/page.interceptor.spec.ts
View File

@ -159,40 +159,7 @@ describe('Unit tests for PageInterceptor', () => {
});
});
});
});
it('Should reverse the search results', async () => {
execCtxMock.getRequest.mockReturnValueOnce({
query: {
page: 1,
order: 'desc',
limit: 3
}
});
await pageInter['cacheManager'].set('_pagenum', 3);
await pageInter['cacheManager'].set('prevPage', { set: "yes" });
callHandlerMock.handle.mockReturnValueOnce(
of({
hits: {
total: { value: 1 },
hits: [
{ sort: ['1', 'less relevant'], _source: '1' },
{ sort: ['2', 'average'], _source: '2' },
{ sort: ['3', 'most relevant'], _source: '3' }
]
}
})
);
pageInter.intercept(execCtxMock, callHandlerMock).then((res) => {
res.subscribe(async (page) => {
expect(await pageInter['cacheManager'].get('_sa')).toEqual(['1', 'less relevant']);
expect(page.data).toEqual(['3', '2', '1']);
});
});
});
});
});
describe('getPIT()', () => {