danny-mhlv/scipaper
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Implemented In-Memory-Caching with default CacheModule

Browse Source
This commit is contained in:
danny-mhlv 2022-09-14 13:12:51 +03:00
parent 3a3737dd85
commit b6287509ad
6 changed files with 108 additions and 161 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/application/controller/papers.controller.ts
View File

@ -1,4 +1,4 @@
import { Controller, Get, HttpCode, Param, ParseUUIDPipe, Req, UseInterceptors } from "@nestjs/common";
import { CacheInterceptor, Controller, Get, HttpCode, Inject, Param, ParseUUIDPipe, Req, UseInterceptors } from "@nestjs/common";
import { SearchService } from "../../core/services/common/search.service";
import { PageInterceptor } from "../../core/interceptors/page.interceptor";
import { ApiExtraModels, ApiGatewayTimeoutResponse, ApiOperation, ApiResponse, ApiTags, getSchemaPath } from "@nestjs/swagger";
@ -13,6 +13,7 @@ import { EsHitDto, EsResponseDto, PageDto, PaperDto } from "../../core/domain";
path: 'papers',
})
@ApiExtraModels(RequestDto, EsHitDto, EsResponseDto)
// @UseInterceptors(CacheInterceptor)
export class PapersController {
constructor(private searchService: SearchService) {}

234
src/core/interceptors/page.interceptor.ts
View File

@ -1,6 +1,6 @@
import { HttpService } from "@nestjs/axios";
import { CallHandler, ExecutionContext, Injectable, NestInterceptor } from "@nestjs/common";
import { Observable, map, take } from "rxjs";
import { CACHE_MANAGER, CallHandler, ExecutionContext, Inject, Injectable, NestInterceptor } from "@nestjs/common";
import { Observable, map, take, switchMap } from "rxjs";
import { PageDto } from "../domain/dtos";
import { EsQueryDto } from "../domain/dtos/elastic/es-query.dto";
import { RequestDto } from "../domain/dtos/request.dto";
@ -9,62 +9,7 @@ import { EsTime } from "../domain/enums/es-time.enum";
import { Order } from "../domain/enums/page-order.enum";
import { PageMeta } from "../domain/interfaces";
import { EsPit } from "../domain/interfaces/elastic/es-pit.interface";
/**
* Previous search data storage
*/
class PrevSearch {
/**
* Constructs an uninitialized object
*/
constructor() {
this.pit = undefined;
this.tiebreaker = undefined;
this.prevPage = -1;
}
/**
* PIT object of the previous search
*/
private pit: EsPit;
set _pit(pit: EsPit) {
this.pit = pit;
}
get _pit(): EsPit {
return this.pit;
}
/**
* Tiebreaker and sort parameters
*/
private tiebreaker: unknown[];
set _tiebreaker(tiebreaker: unknown[]) {
this.tiebreaker = tiebreaker;
}
get _tiebreaker(): unknown[] {
return this.tiebreaker;
}
/**
* Number of the previous page
*/
private prevPage: number;
set _prevPage(page: number) {
this.prevPage = page;
}
get _prevPage(): number {
return this.prevPage;
}
/**
* Checks if there was the search before current one
* @returns true/false, showing whether or not there was another search before
*/
public isSet(): boolean {
if (this.pit && this.tiebreaker && this.prevPage !== -1) return true;
return false;
}
}
import { Cache } from 'cache-manager'
/**
* Pagination-implementing interceptor
@ -73,95 +18,13 @@ class PrevSearch {
export class PageInterceptor implements NestInterceptor {
/**
* Injects needed dependencies and instantiates the storage object
* @param httpService
* @param searchService
* @param httpService
* @param searchService
*/
constructor(private readonly httpService: HttpService) {
this.prevSearch = new PrevSearch;
}
/**
* Override of intercept() method, specified in NestInterceptor interface
* @param context
* @param next
* @returns Page with content and metadata
*/
async intercept(context: ExecutionContext, next: CallHandler<any>): Promise<Observable<PageDto>> {
let request: RequestDto = context.switchToHttp().getRequest<RequestDto>();
const query: SearchQueryDto = request.query;
let reverse: boolean = false;
request.es_query = new EsQueryDto();
request.es_query.query = {
query_string: {
query: query.query,
default_field: 'content',
}
};
request.es_query.sort = [
{ _score: { order: !query?.order ? Order.DESC : query.order } },
{ _shard_doc: 'desc' }
];
if (this.prevSearch.isSet()) {
request.es_query.pit = this.prevSearch._pit;
request.es_query.search_after = this.prevSearch._tiebreaker;
let limit = !query?.limit ? 10 : query.limit;
request.es_query.size = limit * Math.abs(query.page - this.prevSearch._prevPage);
if (query.page < this.prevSearch._prevPage) {
request.es_query.sort = [{ _score: { order: 'asc' } }];
request.es_query.size += limit - 1;
reverse = true;
} else if (query.page == this.prevSearch._prevPage) {
// Caching should be HERE
request.es_query.sort = [{ _score: { order: 'asc' } }];
reverse = true;
}
} else {
this.prevSearch._pit = request.es_query.pit = await this.getPIT(1);
let limit = !query?.limit ? 10 : query.limit;
request.es_query.size = limit * query.page;
}
return next.handle().pipe(
map((res) => {
// Setting the page meta-data
let meta: PageMeta = {
total: res.hits.total.value,
pagenum: !query?.page ? 1 : +query.page,
order: query?.order?.toUpperCase() === Order.ASC ? Order.ASC : Order.DESC,
pagesize: !query?.limit ? 10 : query.limit,
hasNext: undefined,
hasPrev: undefined,
};
meta.hasNext = meta.pagenum * meta.pagesize < meta.total ? true : false;
meta.hasPrev = meta.pagenum != 1 ? true : false;
// Saving the search info
this.prevSearch._pit.id = res.pit_id;
this.prevSearch._tiebreaker = res.hits.hits[res.hits.hits.length - 1]?.sort;
this.prevSearch._prevPage = query.page;
// Check if the performed search is a backwards search
let data = res.hits.hits.slice(-meta.pagesize);
if (reverse) {
this.prevSearch._tiebreaker = data[0]?.sort;
data.reverse();
reverse = false;
}
// Omitting the redundant info and leaving only the document
data = data.map((el) => el._source);
// Return the page
return new PageDto(data, meta);
})
);
}
constructor(
private readonly httpService: HttpService,
@Inject(CACHE_MANAGER) private cacheManager: Cache
) {}
/**
* Elastichsearch server port-number
@ -174,9 +37,84 @@ export class PageInterceptor implements NestInterceptor {
private readonly ES_IP = process.env.ES_CONTAINER_NAME;
/**
* Info about previously completed search
* Override of intercept() method, specified in NestInterceptor interface
* @param context
* @param next
* @returns Page with content and metadata
*/
private prevSearch: PrevSearch;
async intercept(context: ExecutionContext, next: CallHandler<any>): Promise<Observable<PageDto>> {
const request: RequestDto = context.switchToHttp().getRequest<RequestDto>();
const query: SearchQueryDto = request.query;
let reverse: boolean = false;
request.es_query = new EsQueryDto();
request.es_query.query = {
query_string: {
query: query.query,
default_field: 'content',
}
};
request.es_query.sort = [
{ _score: { order: !query?.order ? Order.DESC : query.order } },
{ _shard_doc: 'desc' }
];
const limit = !query?.limit ? 10 : query.limit;
if (await this.cacheManager.get('prevPage')) {
if (query.page == (await this.cacheManager.get('_pagenum'))) return await this.cacheManager.get('prevPage');
request.es_query.pit = await this.cacheManager.get('_pit');
request.es_query.search_after = await this.cacheManager.get('_sa');
request.es_query.size = limit * Math.abs(query.page - (await this.cacheManager.get('_pagenum')));
if (query.page < (await this.cacheManager.get('_pagenum'))) {
request.es_query.sort = [{ _score: { order: Order.ASC } }];
request.es_query.size += limit - 1;
reverse = true;
}
} else {
request.es_query.pit = await this.getPIT(1);
request.es_query.size = limit * query.page;
}
return next.handle().pipe(
switchMap(async (res) => {
// Setting the page meta-data
let meta: PageMeta = {
total: res.hits.total.value,
pagenum: !query?.page ? 1 : +query.page,
order: query?.order?.toUpperCase() === Order.ASC ? Order.ASC : Order.DESC,
pagesize: !query?.limit ? 10 : query.limit,
hasNext: undefined,
hasPrev: undefined,
};
meta.hasNext = meta.pagenum * meta.pagesize < meta.total ? true : false;
meta.hasPrev = meta.pagenum != 1 ? true : false;
// Saving the search info
await this.cacheManager.set('_pit', { id: res.pit_id, keep_alive: `1${EsTime.min}` })
await this.cacheManager.set('_sa', res.hits.hits[res.hits.hits.length - 1]?.sort);
await this.cacheManager.set('_pagenum', query.page);
// Check if the performed search is a backwards search
let data = res.hits.hits.slice(-meta.pagesize);
if (reverse) {
this.cacheManager.set('_sa', data[0]?.sort);
data.reverse();
reverse = false;
}
// Omitting the redundant info and leaving only the document
data = data.map((el) => el._source);
// Cache and return the page
const page: PageDto = new PageDto(data, meta);
await this.cacheManager.set('prevPage', page);
return page;
})
);
}
/**
* Acquires a PIT ID from Elasticsearch, needed for a request

3
src/infrastructure/modules/search.module.ts
View File

@ -1,5 +1,5 @@
import { HttpModule } from "@nestjs/axios";
import { Module } from "@nestjs/common";
import { CacheModule, Module } from "@nestjs/common";
import { PapersController } from "../../application";
import { SearchService } from "../../core/services/common/search.service";
@ -9,6 +9,7 @@ import { SearchService } from "../../core/services/common/search.service";
@Module({
imports: [
HttpModule,
CacheModule.register(),
],
exports: [SearchService],
providers: [SearchService],

2
src/test/e2e/papers.controller.e2e.spec.ts
View File

@ -75,7 +75,7 @@ describe('E2E Testing of /papers', () => {
let httpGetSpy = jest.spyOn(httpService, 'get').mockReturnValueOnce(axiosRes);
const test = await request(app.getHttpServer())
.get('/papers/2d3dc418-7778-abab-b33f-3d63aa25db41') // ??? Fetch a random object from DB
.get('/papers/2d3dc418-7778-abab-b33f-3d63aa25db41')
.expect(200);
// Expect HttpService.get() method to be touched

20
src/test/page.interceptor.spec.ts
View File

@ -1,4 +1,5 @@
import { HttpService } from "@nestjs/axios";
import { CacheModule, CACHE_MANAGER } from "@nestjs/common";
import { ConfigModule } from "@nestjs/config";
import { Test } from "@nestjs/testing";
import { Observable, of } from "rxjs";
@ -44,7 +45,8 @@ describe('Unit tests for PageInterceptor', () => {
isGlobal: true,
cache: true,
expandVariables: true,
})
}),
CacheModule.register()
],
}).compile();
@ -159,7 +161,7 @@ describe('Unit tests for PageInterceptor', () => {
});
});
it('Should reverse the search results', () => {
it('Should reverse the search results', async () => {
execCtxMock.getRequest.mockReturnValueOnce({
query: {
page: 1,
@ -168,10 +170,8 @@ describe('Unit tests for PageInterceptor', () => {
}
});
pageInter['prevSearch']._prevPage = 3;
pageInter['prevSearch'].isSet = jest.fn().mockImplementationOnce(() => {
return true;
})
await pageInter['cacheManager'].set('_pagenum', 3);
await pageInter['cacheManager'].set('prevPage', { set: "yes" });
callHandlerMock.handle.mockReturnValueOnce(
of({
@ -187,8 +187,8 @@ describe('Unit tests for PageInterceptor', () => {
);
pageInter.intercept(execCtxMock, callHandlerMock).then((res) => {
res.subscribe((page) => {
expect(pageInter['prevSearch']._tiebreaker).toEqual(['1', 'less relevant']);
res.subscribe(async (page) => {
expect(await pageInter['cacheManager'].get('_sa')).toEqual(['1', 'less relevant']);
expect(page.data).toEqual(['3', '2', '1']);
});
});
@ -315,4 +315,8 @@ describe('Unit tests for PageInterceptor', () => {
expect(pageInter.deletePIT('')).resolves.toBe(true);
});
});
afterEach(() => {
pageInter['cacheManager'].reset();
})
});

7
src/test/papers.controller.spec.ts
View File

@ -1,5 +1,5 @@
import { HttpModule } from "@nestjs/axios";
import { NotFoundException } from "@nestjs/common";
import { CacheModule, NotFoundException } from "@nestjs/common";
import { Test } from "@nestjs/testing";
import { PapersController } from "src/application";
import { SearchService } from "src/core/services/common/search.service";
@ -21,7 +21,10 @@ describe('Unit tests for PapersController', () => {
}
}
],
imports: [HttpModule]
imports: [
HttpModule,
CacheModule.register()
]
}).compile();
papersController = moduleRef.get(PapersController);