From b6287509ad9e5e4ae4c07f0351606634a0cacad2 Mon Sep 17 00:00:00 2001
From: danny-mhlv <danny.mhlv@gmail.com>
Date: Wed, 14 Sep 2022 13:12:51 +0300
Subject: [PATCH] Implemented In-Memory-Caching with default CacheModule
---
.../controller/papers.controller.ts | 3 +-
src/core/interceptors/page.interceptor.ts | 234 +++++++-----------
src/infrastructure/modules/search.module.ts | 3 +-
src/test/e2e/papers.controller.e2e.spec.ts | 2 +-
src/test/page.interceptor.spec.ts | 20 +-
src/test/papers.controller.spec.ts | 7 +-
6 files changed, 108 insertions(+), 161 deletions(-)
diff --git a/src/application/controller/papers.controller.ts b/src/application/controller/papers.controller.ts
index 74efcdf..3151334 100644
--- a/src/application/controller/papers.controller.ts
+++ b/src/application/controller/papers.controller.ts
@@ -1,4 +1,4 @@
-import { Controller, Get, HttpCode, Param, ParseUUIDPipe, Req, UseInterceptors } from "@nestjs/common";
+import { CacheInterceptor, Controller, Get, HttpCode, Inject, Param, ParseUUIDPipe, Req, UseInterceptors } from "@nestjs/common";
import { SearchService } from "../../core/services/common/search.service";
import { PageInterceptor } from "../../core/interceptors/page.interceptor";
import { ApiExtraModels, ApiGatewayTimeoutResponse, ApiOperation, ApiResponse, ApiTags, getSchemaPath } from "@nestjs/swagger";
@@ -13,6 +13,7 @@ import { EsHitDto, EsResponseDto, PageDto, PaperDto } from "../../core/domain";
path: 'papers',
})
@ApiExtraModels(RequestDto, EsHitDto, EsResponseDto)
+// @UseInterceptors(CacheInterceptor)
export class PapersController {
constructor(private searchService: SearchService) {}
diff --git a/src/core/interceptors/page.interceptor.ts b/src/core/interceptors/page.interceptor.ts
index 76e1934..01eda50 100644
--- a/src/core/interceptors/page.interceptor.ts
+++ b/src/core/interceptors/page.interceptor.ts
@@ -1,6 +1,6 @@
import { HttpService } from "@nestjs/axios";
-import { CallHandler, ExecutionContext, Injectable, NestInterceptor } from "@nestjs/common";
-import { Observable, map, take } from "rxjs";
+import { CACHE_MANAGER, CallHandler, ExecutionContext, Inject, Injectable, NestInterceptor } from "@nestjs/common";
+import { Observable, map, take, switchMap } from "rxjs";
import { PageDto } from "../domain/dtos";
import { EsQueryDto } from "../domain/dtos/elastic/es-query.dto";
import { RequestDto } from "../domain/dtos/request.dto";
@@ -9,62 +9,7 @@ import { EsTime } from "../domain/enums/es-time.enum";
import { Order } from "../domain/enums/page-order.enum";
import { PageMeta } from "../domain/interfaces";
import { EsPit } from "../domain/interfaces/elastic/es-pit.interface";
-
-/**
- * Previous search data storage
- */
-class PrevSearch {
- /**
- * Constructs an uninitialized object
- */
- constructor() {
- this.pit = undefined;
- this.tiebreaker = undefined;
- this.prevPage = -1;
- }
-
- /**
- * PIT object of the previous search
- */
- private pit: EsPit;
- set _pit(pit: EsPit) {
- this.pit = pit;
- }
- get _pit(): EsPit {
- return this.pit;
- }
-
- /**
- * Tiebreaker and sort parameters
- */
- private tiebreaker: unknown[];
- set _tiebreaker(tiebreaker: unknown[]) {
- this.tiebreaker = tiebreaker;
- }
- get _tiebreaker(): unknown[] {
- return this.tiebreaker;
- }
-
- /**
- * Number of the previous page
- */
- private prevPage: number;
- set _prevPage(page: number) {
- this.prevPage = page;
- }
- get _prevPage(): number {
- return this.prevPage;
- }
-
- /**
- * Checks if there was the search before current one
- * @returns true/false, showing whether or not there was another search before
- */
- public isSet(): boolean {
- if (this.pit && this.tiebreaker && this.prevPage !== -1) return true;
- return false;
- }
-}
+import { Cache } from 'cache-manager'
/**
* Pagination-implementing interceptor
@@ -73,95 +18,13 @@ class PrevSearch {
export class PageInterceptor implements NestInterceptor {
/**
* Injects needed dependencies and instantiates the storage object
- * @param httpService
- * @param searchService
+ * @param httpService
+ * @param searchService
*/
- constructor(private readonly httpService: HttpService) {
- this.prevSearch = new PrevSearch;
- }
-
- /**
- * Override of intercept() method, specified in NestInterceptor interface
- * @param context
- * @param next
- * @returns Page with content and metadata
- */
- async intercept(context: ExecutionContext, next: CallHandler<any>): Promise<Observable<PageDto>> {
- let request: RequestDto = context.switchToHttp().getRequest<RequestDto>();
- const query: SearchQueryDto = request.query;
- let reverse: boolean = false;
-
- request.es_query = new EsQueryDto();
-
- request.es_query.query = {
- query_string: {
- query: query.query,
- default_field: 'content',
- }
- };
- request.es_query.sort = [
- { _score: { order: !query?.order ? Order.DESC : query.order } },
- { _shard_doc: 'desc' }
- ];
-
- if (this.prevSearch.isSet()) {
- request.es_query.pit = this.prevSearch._pit;
- request.es_query.search_after = this.prevSearch._tiebreaker;
-
- let limit = !query?.limit ? 10 : query.limit;
- request.es_query.size = limit * Math.abs(query.page - this.prevSearch._prevPage);
-
- if (query.page < this.prevSearch._prevPage) {
- request.es_query.sort = [{ _score: { order: 'asc' } }];
- request.es_query.size += limit - 1;
- reverse = true;
- } else if (query.page == this.prevSearch._prevPage) {
- // Caching should be HERE
- request.es_query.sort = [{ _score: { order: 'asc' } }];
- reverse = true;
- }
- } else {
- this.prevSearch._pit = request.es_query.pit = await this.getPIT(1);
-
- let limit = !query?.limit ? 10 : query.limit;
- request.es_query.size = limit * query.page;
- }
-
- return next.handle().pipe(
- map((res) => {
- // Setting the page meta-data
- let meta: PageMeta = {
- total: res.hits.total.value,
- pagenum: !query?.page ? 1 : +query.page,
- order: query?.order?.toUpperCase() === Order.ASC ? Order.ASC : Order.DESC,
- pagesize: !query?.limit ? 10 : query.limit,
- hasNext: undefined,
- hasPrev: undefined,
- };
- meta.hasNext = meta.pagenum * meta.pagesize < meta.total ? true : false;
- meta.hasPrev = meta.pagenum != 1 ? true : false;
-
- // Saving the search info
- this.prevSearch._pit.id = res.pit_id;
- this.prevSearch._tiebreaker = res.hits.hits[res.hits.hits.length - 1]?.sort;
- this.prevSearch._prevPage = query.page;
-
- // Check if the performed search is a backwards search
- let data = res.hits.hits.slice(-meta.pagesize);
- if (reverse) {
- this.prevSearch._tiebreaker = data[0]?.sort;
- data.reverse();
- reverse = false;
- }
-
- // Omitting the redundant info and leaving only the document
- data = data.map((el) => el._source);
-
- // Return the page
- return new PageDto(data, meta);
- })
- );
- }
+ constructor(
+ private readonly httpService: HttpService,
+ @Inject(CACHE_MANAGER) private cacheManager: Cache
+ ) {}
/**
* Elastichsearch server port-number
@@ -174,9 +37,84 @@ export class PageInterceptor implements NestInterceptor {
private readonly ES_IP = process.env.ES_CONTAINER_NAME;
/**
- * Info about previously completed search
+ * Override of intercept() method, specified in NestInterceptor interface
+ * @param context
+ * @param next
+ * @returns Page with content and metadata
*/
- private prevSearch: PrevSearch;
+ async intercept(context: ExecutionContext, next: CallHandler<any>): Promise<Observable<PageDto>> {
+ const request: RequestDto = context.switchToHttp().getRequest<RequestDto>();
+ const query: SearchQueryDto = request.query;
+ let reverse: boolean = false;
+
+ request.es_query = new EsQueryDto();
+ request.es_query.query = {
+ query_string: {
+ query: query.query,
+ default_field: 'content',
+ }
+ };
+ request.es_query.sort = [
+ { _score: { order: !query?.order ? Order.DESC : query.order } },
+ { _shard_doc: 'desc' }
+ ];
+
+ const limit = !query?.limit ? 10 : query.limit;
+
+ if (await this.cacheManager.get('prevPage')) {
+ if (query.page == (await this.cacheManager.get('_pagenum'))) return await this.cacheManager.get('prevPage');
+
+ request.es_query.pit = await this.cacheManager.get('_pit');
+ request.es_query.search_after = await this.cacheManager.get('_sa');
+ request.es_query.size = limit * Math.abs(query.page - (await this.cacheManager.get('_pagenum')));
+
+ if (query.page < (await this.cacheManager.get('_pagenum'))) {
+ request.es_query.sort = [{ _score: { order: Order.ASC } }];
+ request.es_query.size += limit - 1;
+ reverse = true;
+ }
+ } else {
+ request.es_query.pit = await this.getPIT(1);
+ request.es_query.size = limit * query.page;
+ }
+
+ return next.handle().pipe(
+ switchMap(async (res) => {
+ // Setting the page meta-data
+ let meta: PageMeta = {
+ total: res.hits.total.value,
+ pagenum: !query?.page ? 1 : +query.page,
+ order: query?.order?.toUpperCase() === Order.ASC ? Order.ASC : Order.DESC,
+ pagesize: !query?.limit ? 10 : query.limit,
+ hasNext: undefined,
+ hasPrev: undefined,
+ };
+ meta.hasNext = meta.pagenum * meta.pagesize < meta.total ? true : false;
+ meta.hasPrev = meta.pagenum != 1 ? true : false;
+
+ // Saving the search info
+ await this.cacheManager.set('_pit', { id: res.pit_id, keep_alive: `1${EsTime.min}` })
+ await this.cacheManager.set('_sa', res.hits.hits[res.hits.hits.length - 1]?.sort);
+ await this.cacheManager.set('_pagenum', query.page);
+
+ // Check if the performed search is a backwards search
+ let data = res.hits.hits.slice(-meta.pagesize);
+ if (reverse) {
+ this.cacheManager.set('_sa', data[0]?.sort);
+ data.reverse();
+ reverse = false;
+ }
+
+ // Omitting the redundant info and leaving only the document
+ data = data.map((el) => el._source);
+
+ // Cache and return the page
+ const page: PageDto = new PageDto(data, meta);
+ await this.cacheManager.set('prevPage', page);
+ return page;
+ })
+ );
+ }
/**
* Acquires a PIT ID from Elasticsearch, needed for a request
diff --git a/src/infrastructure/modules/search.module.ts b/src/infrastructure/modules/search.module.ts
index 526c7d7..8fd2f60 100644
--- a/src/infrastructure/modules/search.module.ts
+++ b/src/infrastructure/modules/search.module.ts
@@ -1,5 +1,5 @@
import { HttpModule } from "@nestjs/axios";
-import { Module } from "@nestjs/common";
+import { CacheModule, Module } from "@nestjs/common";
import { PapersController } from "../../application";
import { SearchService } from "../../core/services/common/search.service";
@@ -9,6 +9,7 @@ import { SearchService } from "../../core/services/common/search.service";
@Module({
imports: [
HttpModule,
+ CacheModule.register(),
],
exports: [SearchService],
providers: [SearchService],
diff --git a/src/test/e2e/papers.controller.e2e.spec.ts b/src/test/e2e/papers.controller.e2e.spec.ts
index 922bdc0..2934683 100644
--- a/src/test/e2e/papers.controller.e2e.spec.ts
+++ b/src/test/e2e/papers.controller.e2e.spec.ts
@@ -75,7 +75,7 @@ describe('E2E Testing of /papers', () => {
let httpGetSpy = jest.spyOn(httpService, 'get').mockReturnValueOnce(axiosRes);
const test = await request(app.getHttpServer())
- .get('/papers/2d3dc418-7778-abab-b33f-3d63aa25db41') // ??? Fetch a random object from DB
+ .get('/papers/2d3dc418-7778-abab-b33f-3d63aa25db41')
.expect(200);
// Expect HttpService.get() method to be touched
diff --git a/src/test/page.interceptor.spec.ts b/src/test/page.interceptor.spec.ts
index e3fda46..5a60f02 100644
--- a/src/test/page.interceptor.spec.ts
+++ b/src/test/page.interceptor.spec.ts
@@ -1,4 +1,5 @@
import { HttpService } from "@nestjs/axios";
+import { CacheModule, CACHE_MANAGER } from "@nestjs/common";
import { ConfigModule } from "@nestjs/config";
import { Test } from "@nestjs/testing";
import { Observable, of } from "rxjs";
@@ -44,7 +45,8 @@ describe('Unit tests for PageInterceptor', () => {
isGlobal: true,
cache: true,
expandVariables: true,
- })
+ }),
+ CacheModule.register()
],
}).compile();
@@ -159,7 +161,7 @@ describe('Unit tests for PageInterceptor', () => {
});
});
- it('Should reverse the search results', () => {
+ it('Should reverse the search results', async () => {
execCtxMock.getRequest.mockReturnValueOnce({
query: {
page: 1,
@@ -168,10 +170,8 @@ describe('Unit tests for PageInterceptor', () => {
}
});
- pageInter['prevSearch']._prevPage = 3;
- pageInter['prevSearch'].isSet = jest.fn().mockImplementationOnce(() => {
- return true;
- })
+ await pageInter['cacheManager'].set('_pagenum', 3);
+ await pageInter['cacheManager'].set('prevPage', { set: "yes" });
callHandlerMock.handle.mockReturnValueOnce(
of({
@@ -187,8 +187,8 @@ describe('Unit tests for PageInterceptor', () => {
);
pageInter.intercept(execCtxMock, callHandlerMock).then((res) => {
- res.subscribe((page) => {
- expect(pageInter['prevSearch']._tiebreaker).toEqual(['1', 'less relevant']);
+ res.subscribe(async (page) => {
+ expect(await pageInter['cacheManager'].get('_sa')).toEqual(['1', 'less relevant']);
expect(page.data).toEqual(['3', '2', '1']);
});
});
@@ -315,4 +315,8 @@ describe('Unit tests for PageInterceptor', () => {
expect(pageInter.deletePIT('')).resolves.toBe(true);
});
});
+
+ afterEach(() => {
+ pageInter['cacheManager'].reset();
+ })
});
\ No newline at end of file
diff --git a/src/test/papers.controller.spec.ts b/src/test/papers.controller.spec.ts
index 5a1ff79..36c4eb4 100644
--- a/src/test/papers.controller.spec.ts
+++ b/src/test/papers.controller.spec.ts
@@ -1,5 +1,5 @@
import { HttpModule } from "@nestjs/axios";
-import { NotFoundException } from "@nestjs/common";
+import { CacheModule, NotFoundException } from "@nestjs/common";
import { Test } from "@nestjs/testing";
import { PapersController } from "src/application";
import { SearchService } from "src/core/services/common/search.service";
@@ -21,7 +21,10 @@ describe('Unit tests for PapersController', () => {
}
}
],
- imports: [HttpModule]
+ imports: [
+ HttpModule,
+ CacheModule.register()
+ ]
}).compile();
papersController = moduleRef.get(PapersController);