Implemented In-Memory-Caching with default CacheModule

This commit is contained in:
danny-mhlv 2022-09-14 13:12:51 +03:00
parent 3a3737dd85
commit b6287509ad
6 changed files with 108 additions and 161 deletions

View File

@ -1,4 +1,4 @@
import { Controller, Get, HttpCode, Param, ParseUUIDPipe, Req, UseInterceptors } from "@nestjs/common"; import { CacheInterceptor, Controller, Get, HttpCode, Inject, Param, ParseUUIDPipe, Req, UseInterceptors } from "@nestjs/common";
import { SearchService } from "../../core/services/common/search.service"; import { SearchService } from "../../core/services/common/search.service";
import { PageInterceptor } from "../../core/interceptors/page.interceptor"; import { PageInterceptor } from "../../core/interceptors/page.interceptor";
import { ApiExtraModels, ApiGatewayTimeoutResponse, ApiOperation, ApiResponse, ApiTags, getSchemaPath } from "@nestjs/swagger"; import { ApiExtraModels, ApiGatewayTimeoutResponse, ApiOperation, ApiResponse, ApiTags, getSchemaPath } from "@nestjs/swagger";
@ -13,6 +13,7 @@ import { EsHitDto, EsResponseDto, PageDto, PaperDto } from "../../core/domain";
path: 'papers', path: 'papers',
}) })
@ApiExtraModels(RequestDto, EsHitDto, EsResponseDto) @ApiExtraModels(RequestDto, EsHitDto, EsResponseDto)
// @UseInterceptors(CacheInterceptor)
export class PapersController { export class PapersController {
constructor(private searchService: SearchService) {} constructor(private searchService: SearchService) {}

View File

@ -1,6 +1,6 @@
import { HttpService } from "@nestjs/axios"; import { HttpService } from "@nestjs/axios";
import { CallHandler, ExecutionContext, Injectable, NestInterceptor } from "@nestjs/common"; import { CACHE_MANAGER, CallHandler, ExecutionContext, Inject, Injectable, NestInterceptor } from "@nestjs/common";
import { Observable, map, take } from "rxjs"; import { Observable, map, take, switchMap } from "rxjs";
import { PageDto } from "../domain/dtos"; import { PageDto } from "../domain/dtos";
import { EsQueryDto } from "../domain/dtos/elastic/es-query.dto"; import { EsQueryDto } from "../domain/dtos/elastic/es-query.dto";
import { RequestDto } from "../domain/dtos/request.dto"; import { RequestDto } from "../domain/dtos/request.dto";
@ -9,62 +9,7 @@ import { EsTime } from "../domain/enums/es-time.enum";
import { Order } from "../domain/enums/page-order.enum"; import { Order } from "../domain/enums/page-order.enum";
import { PageMeta } from "../domain/interfaces"; import { PageMeta } from "../domain/interfaces";
import { EsPit } from "../domain/interfaces/elastic/es-pit.interface"; import { EsPit } from "../domain/interfaces/elastic/es-pit.interface";
import { Cache } from 'cache-manager'
/**
* Previous search data storage
*/
class PrevSearch {
/**
* Constructs an uninitialized object
*/
constructor() {
this.pit = undefined;
this.tiebreaker = undefined;
this.prevPage = -1;
}
/**
* PIT object of the previous search
*/
private pit: EsPit;
set _pit(pit: EsPit) {
this.pit = pit;
}
get _pit(): EsPit {
return this.pit;
}
/**
* Tiebreaker and sort parameters
*/
private tiebreaker: unknown[];
set _tiebreaker(tiebreaker: unknown[]) {
this.tiebreaker = tiebreaker;
}
get _tiebreaker(): unknown[] {
return this.tiebreaker;
}
/**
* Number of the previous page
*/
private prevPage: number;
set _prevPage(page: number) {
this.prevPage = page;
}
get _prevPage(): number {
return this.prevPage;
}
/**
* Checks if there was the search before current one
* @returns true/false, showing whether or not there was another search before
*/
public isSet(): boolean {
if (this.pit && this.tiebreaker && this.prevPage !== -1) return true;
return false;
}
}
/** /**
* Pagination-implementing interceptor * Pagination-implementing interceptor
@ -76,9 +21,20 @@ export class PageInterceptor implements NestInterceptor {
* @param httpService * @param httpService
* @param searchService * @param searchService
*/ */
constructor(private readonly httpService: HttpService) { constructor(
this.prevSearch = new PrevSearch; private readonly httpService: HttpService,
} @Inject(CACHE_MANAGER) private cacheManager: Cache
) {}
/**
* Elastichsearch server port-number
*/
private readonly ES_PORT = process.env.ES_PORT;
/**
* Elastichsearch IP address
*/
private readonly ES_IP = process.env.ES_CONTAINER_NAME;
/** /**
* Override of intercept() method, specified in NestInterceptor interface * Override of intercept() method, specified in NestInterceptor interface
@ -87,12 +43,11 @@ export class PageInterceptor implements NestInterceptor {
* @returns Page with content and metadata * @returns Page with content and metadata
*/ */
async intercept(context: ExecutionContext, next: CallHandler<any>): Promise<Observable<PageDto>> { async intercept(context: ExecutionContext, next: CallHandler<any>): Promise<Observable<PageDto>> {
let request: RequestDto = context.switchToHttp().getRequest<RequestDto>(); const request: RequestDto = context.switchToHttp().getRequest<RequestDto>();
const query: SearchQueryDto = request.query; const query: SearchQueryDto = request.query;
let reverse: boolean = false; let reverse: boolean = false;
request.es_query = new EsQueryDto(); request.es_query = new EsQueryDto();
request.es_query.query = { request.es_query.query = {
query_string: { query_string: {
query: query.query, query: query.query,
@ -104,31 +59,27 @@ export class PageInterceptor implements NestInterceptor {
{ _shard_doc: 'desc' } { _shard_doc: 'desc' }
]; ];
if (this.prevSearch.isSet()) { const limit = !query?.limit ? 10 : query.limit;
request.es_query.pit = this.prevSearch._pit;
request.es_query.search_after = this.prevSearch._tiebreaker;
let limit = !query?.limit ? 10 : query.limit; if (await this.cacheManager.get('prevPage')) {
request.es_query.size = limit * Math.abs(query.page - this.prevSearch._prevPage); if (query.page == (await this.cacheManager.get('_pagenum'))) return await this.cacheManager.get('prevPage');
if (query.page < this.prevSearch._prevPage) { request.es_query.pit = await this.cacheManager.get('_pit');
request.es_query.sort = [{ _score: { order: 'asc' } }]; request.es_query.search_after = await this.cacheManager.get('_sa');
request.es_query.size = limit * Math.abs(query.page - (await this.cacheManager.get('_pagenum')));
if (query.page < (await this.cacheManager.get('_pagenum'))) {
request.es_query.sort = [{ _score: { order: Order.ASC } }];
request.es_query.size += limit - 1; request.es_query.size += limit - 1;
reverse = true; reverse = true;
} else if (query.page == this.prevSearch._prevPage) {
// Caching should be HERE
request.es_query.sort = [{ _score: { order: 'asc' } }];
reverse = true;
} }
} else { } else {
this.prevSearch._pit = request.es_query.pit = await this.getPIT(1); request.es_query.pit = await this.getPIT(1);
let limit = !query?.limit ? 10 : query.limit;
request.es_query.size = limit * query.page; request.es_query.size = limit * query.page;
} }
return next.handle().pipe( return next.handle().pipe(
map((res) => { switchMap(async (res) => {
// Setting the page meta-data // Setting the page meta-data
let meta: PageMeta = { let meta: PageMeta = {
total: res.hits.total.value, total: res.hits.total.value,
@ -142,14 +93,14 @@ export class PageInterceptor implements NestInterceptor {
meta.hasPrev = meta.pagenum != 1 ? true : false; meta.hasPrev = meta.pagenum != 1 ? true : false;
// Saving the search info // Saving the search info
this.prevSearch._pit.id = res.pit_id; await this.cacheManager.set('_pit', { id: res.pit_id, keep_alive: `1${EsTime.min}` })
this.prevSearch._tiebreaker = res.hits.hits[res.hits.hits.length - 1]?.sort; await this.cacheManager.set('_sa', res.hits.hits[res.hits.hits.length - 1]?.sort);
this.prevSearch._prevPage = query.page; await this.cacheManager.set('_pagenum', query.page);
// Check if the performed search is a backwards search // Check if the performed search is a backwards search
let data = res.hits.hits.slice(-meta.pagesize); let data = res.hits.hits.slice(-meta.pagesize);
if (reverse) { if (reverse) {
this.prevSearch._tiebreaker = data[0]?.sort; this.cacheManager.set('_sa', data[0]?.sort);
data.reverse(); data.reverse();
reverse = false; reverse = false;
} }
@ -157,27 +108,14 @@ export class PageInterceptor implements NestInterceptor {
// Omitting the redundant info and leaving only the document // Omitting the redundant info and leaving only the document
data = data.map((el) => el._source); data = data.map((el) => el._source);
// Return the page // Cache and return the page
return new PageDto(data, meta); const page: PageDto = new PageDto(data, meta);
await this.cacheManager.set('prevPage', page);
return page;
}) })
); );
} }
/**
* Elastichsearch server port-number
*/
private readonly ES_PORT = process.env.ES_PORT;
/**
* Elastichsearch IP address
*/
private readonly ES_IP = process.env.ES_CONTAINER_NAME;
/**
* Info about previously completed search
*/
private prevSearch: PrevSearch;
/** /**
* Acquires a PIT ID from Elasticsearch, needed for a request * Acquires a PIT ID from Elasticsearch, needed for a request
* @param alive, amount of time in minutes (defaults to 1). If time unit is not specified - defaults to minutes. * @param alive, amount of time in minutes (defaults to 1). If time unit is not specified - defaults to minutes.

View File

@ -1,5 +1,5 @@
import { HttpModule } from "@nestjs/axios"; import { HttpModule } from "@nestjs/axios";
import { Module } from "@nestjs/common"; import { CacheModule, Module } from "@nestjs/common";
import { PapersController } from "../../application"; import { PapersController } from "../../application";
import { SearchService } from "../../core/services/common/search.service"; import { SearchService } from "../../core/services/common/search.service";
@ -9,6 +9,7 @@ import { SearchService } from "../../core/services/common/search.service";
@Module({ @Module({
imports: [ imports: [
HttpModule, HttpModule,
CacheModule.register(),
], ],
exports: [SearchService], exports: [SearchService],
providers: [SearchService], providers: [SearchService],

View File

@ -75,7 +75,7 @@ describe('E2E Testing of /papers', () => {
let httpGetSpy = jest.spyOn(httpService, 'get').mockReturnValueOnce(axiosRes); let httpGetSpy = jest.spyOn(httpService, 'get').mockReturnValueOnce(axiosRes);
const test = await request(app.getHttpServer()) const test = await request(app.getHttpServer())
.get('/papers/2d3dc418-7778-abab-b33f-3d63aa25db41') // ??? Fetch a random object from DB .get('/papers/2d3dc418-7778-abab-b33f-3d63aa25db41')
.expect(200); .expect(200);
// Expect HttpService.get() method to be touched // Expect HttpService.get() method to be touched

View File

@ -1,4 +1,5 @@
import { HttpService } from "@nestjs/axios"; import { HttpService } from "@nestjs/axios";
import { CacheModule, CACHE_MANAGER } from "@nestjs/common";
import { ConfigModule } from "@nestjs/config"; import { ConfigModule } from "@nestjs/config";
import { Test } from "@nestjs/testing"; import { Test } from "@nestjs/testing";
import { Observable, of } from "rxjs"; import { Observable, of } from "rxjs";
@ -44,7 +45,8 @@ describe('Unit tests for PageInterceptor', () => {
isGlobal: true, isGlobal: true,
cache: true, cache: true,
expandVariables: true, expandVariables: true,
}) }),
CacheModule.register()
], ],
}).compile(); }).compile();
@ -159,7 +161,7 @@ describe('Unit tests for PageInterceptor', () => {
}); });
}); });
it('Should reverse the search results', () => { it('Should reverse the search results', async () => {
execCtxMock.getRequest.mockReturnValueOnce({ execCtxMock.getRequest.mockReturnValueOnce({
query: { query: {
page: 1, page: 1,
@ -168,10 +170,8 @@ describe('Unit tests for PageInterceptor', () => {
} }
}); });
pageInter['prevSearch']._prevPage = 3; await pageInter['cacheManager'].set('_pagenum', 3);
pageInter['prevSearch'].isSet = jest.fn().mockImplementationOnce(() => { await pageInter['cacheManager'].set('prevPage', { set: "yes" });
return true;
})
callHandlerMock.handle.mockReturnValueOnce( callHandlerMock.handle.mockReturnValueOnce(
of({ of({
@ -187,8 +187,8 @@ describe('Unit tests for PageInterceptor', () => {
); );
pageInter.intercept(execCtxMock, callHandlerMock).then((res) => { pageInter.intercept(execCtxMock, callHandlerMock).then((res) => {
res.subscribe((page) => { res.subscribe(async (page) => {
expect(pageInter['prevSearch']._tiebreaker).toEqual(['1', 'less relevant']); expect(await pageInter['cacheManager'].get('_sa')).toEqual(['1', 'less relevant']);
expect(page.data).toEqual(['3', '2', '1']); expect(page.data).toEqual(['3', '2', '1']);
}); });
}); });
@ -315,4 +315,8 @@ describe('Unit tests for PageInterceptor', () => {
expect(pageInter.deletePIT('')).resolves.toBe(true); expect(pageInter.deletePIT('')).resolves.toBe(true);
}); });
}); });
afterEach(() => {
pageInter['cacheManager'].reset();
})
}); });

View File

@ -1,5 +1,5 @@
import { HttpModule } from "@nestjs/axios"; import { HttpModule } from "@nestjs/axios";
import { NotFoundException } from "@nestjs/common"; import { CacheModule, NotFoundException } from "@nestjs/common";
import { Test } from "@nestjs/testing"; import { Test } from "@nestjs/testing";
import { PapersController } from "src/application"; import { PapersController } from "src/application";
import { SearchService } from "src/core/services/common/search.service"; import { SearchService } from "src/core/services/common/search.service";
@ -21,7 +21,10 @@ describe('Unit tests for PapersController', () => {
} }
} }
], ],
imports: [HttpModule] imports: [
HttpModule,
CacheModule.register()
]
}).compile(); }).compile();
papersController = moduleRef.get(PapersController); papersController = moduleRef.get(PapersController);