From d283cc22f13cb5f4af90f4d07316b6886297e230 Mon Sep 17 00:00:00 2001 From: danny-mhlv Date: Thu, 25 Aug 2022 18:12:27 +0300 Subject: [PATCH 01/13] Dockerfile changes --- .env.develop | 7 ++++++- Dockerfile | 3 +++ docker-compose.yaml | 15 +++++++-------- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/.env.develop b/.env.develop index 868ed15..b7c3f27 100644 --- a/.env.develop +++ b/.env.develop @@ -8,4 +8,9 @@ CONTAINER_NAME="container-name" ES_PORT=9200 ES_IMAGE_VERSION=8.3.2 ES_IMAGE_NAME="image-name" -ES_CONTAINER_NAME="container-name" \ No newline at end of file +ES_CONTAINER_NAME="container-name" + +MDB_PORT=27017 +MDB_IMAGE_NAME="image-name" +MDB_IMAGE_VERSION="latest" +MDB_CONTAINER_NAME="container-name" \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 22be726..1c66e2a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,6 +8,9 @@ ENV MS_SCRIPTS="${MS_HOME}/scripts" ENV USER_NAME=node USER_UID=1000 GROUP_NAME=node GROUP_UID=1000 +ENV ES_PORT=9200 +ENV MDB_PORT=27017 + WORKDIR "${MS_HOME}" # Build diff --git a/docker-compose.yaml b/docker-compose.yaml index 46a132b..85a9ae7 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -10,12 +10,12 @@ services: environment: - xpack.security.enabled=false - discovery.type=single-node - - - - - - + mongo: + image: ${MDB_IMAGE_NAME}:${MDB_IMAGE_VERSION} + container_name: ${MDB_CONTAINER_NAME} + restart: always + ports: + - ${MDB_PORT}:${MDB_PORT} freeland: image: ${IMAGE_NAME}:${IMAGE_VERSION} build: @@ -29,5 +29,4 @@ services: - "${LOCAL_PORT}:${NODE_PORT}" environment: - NODE_ENV= ${NODE_ENV} - - NODE_PORT= ${NODE_PORT} - \ No newline at end of file + - NODE_PORT= ${NODE_PORT} \ No newline at end of file -- 2.39.5 From 9f430dc54d88689afae634eb26f550fd5b2fb9f4 Mon Sep 17 00:00:00 2001 From: danny-mhlv Date: Wed, 14 Sep 2022 12:26:03 +0300 Subject: [PATCH 02/13] Added config script for Elastcsearch --- elastic/scripts/es_boot.sh | 208 +++++++++++++++++++++++++++++++++++++ 1 file changed, 208 insertions(+) create mode 100755 elastic/scripts/es_boot.sh diff --git a/elastic/scripts/es_boot.sh b/elastic/scripts/es_boot.sh new file mode 100755 index 0000000..5e5f2eb --- /dev/null +++ b/elastic/scripts/es_boot.sh @@ -0,0 +1,208 @@ +#!/usr/bin/env bash + +__isint='^[0-9]+$' +__isvalidstr='^[a-z0-9]+$' +__isvalidaddr='^[a-z]+$|^((25[0-5]|2[0-4]?[0-9]|1[0-9]?[0-9]|[3-9][0-9]|[0-9])\.){3}(25[0-5]|2[0-4]?[0-9]|1[0-9]?[0-9]|[3-9][0-9]|[0-9])$' + +create_template() { + curl -H "Content-Type: application/json" -X PUT "$1:$2/_index_template/papers_t" -d ' + { + "index_patterns": ["papers*"], + "priority": 1, + "template": { + "aliases": { + "papers": {} + }, + "mappings": { + "properties": { + "id": { + "type": "keyword" + }, + "title": { + "type": "text", + "analyzer": "title_analyzer" + }, + "authors": { + "type": "text" + }, + "topic": { + "type": "text" + }, + "summary": { + "type": "text" + }, + "tags": { + "type": "keyword" + }, + "content": { + "type": "text" + }, + "publisher": { + "type": "text" + } + } + }, + "settings": { + "analysis": { + "analyzer": { + "title_analyzer": { + "type": "custom", + "tokenizer": "title_engram_tokenizer" + }, + "content_analyzer_i": { + "type": "custom", + "tokenizer": "content_onchar_tokenizer", + "char_filter": [ + "markdown_token_filter" + ] + }, + "content_analyzer_s": { + "type": "custom", + "tokenizer": "content_onchar_tokenizer", + "char_filter": [ + "markdown_token_filter" + ], + "filter": [ + ] + }, + "basic_analyzer": { + + } + }, + "tokenizer": { + "title_engram_tokenizer": { + "type": "edge_ngram", + "min_gram": 2, + "max_gram": 10, + "token_chars": [ + "letter", + "digit" + ] + }, + "content_onchar_tokenizer": { + "type": "char_group", + "tokenize_on_chars": [ + "whitespace", + ".", ",", "(", ")", "-", "[", "]", "{", + "}", "#", ":", ";", "`", "!", "*" + ] + } + }, + "char_filter": { + "markdown_token_filter": { + "type": "pattern_replace", + "pattern": "[[a-z][0-9]]*://[[a-z][0-9]]*.[a-z]*", + "replacement": "" + } + }, + "filter": { + } + } + } + } + } + ' +} + +#============================================================================================================================================================================= + +create_index() { + curl -X PUT "$1:$2/papers-$3?pretty" +} + +#============================================================================================================================================================================= + +__usage=" + Usage: $(basename $0) + --------------------------------------------------------------------------- + | -c, --create-only | Skip template initialization and only create | + | | specified index. Result index name will be | + | | 'papers-{specified name}' | + | | + | -h | Help information | + | | + | -a | Specifies the address | + | | + | -p, --port | Specifies the port | + | | + | -i, --index-name | Specifies the index name: | + | | Must be lowercase, cannot include [\/*?\"<>| ,#:], | + | | cannot start with [.-_+], cannot be \".\" or \"..\" | + | | cannot be longer than 255 bytes (note: multi-byte | + | | characters will count towards the limit faster) | + | | Result index name will be 'papers-{specified name}' | + --------------------------------------------------------------------------- +" +#============================================================================================================================================================================= + +CTRL=0 + +if [[ "$1" == "-h" ]]; then + echo "$__usage" +else + while [[ $# -gt 0 ]]; do + case "$1" in + -p | --port) + if [[ -n "$2" && $2 =~ $__isint && "$2" -ge 1 && "$2" -le 65535 ]]; then + PORT="$2" + shift + else + echo "Invalid port number!" + fi + ;; + -a | --address) + if [[ -n "$2" ]]; then + IP="$2" + shift + else + echo "Address is not specified!" + fi + ;; + -i | --index-name) + if [[ -n "$2" && $2 =~ $__isvalidstr ]]; then + IND="$2" + shift + else + echo "Index name is not specified!" + fi + ;; + -c | --create-only) + CTRL=2 + ;; + -*) + echo "Option '$1' is not supported" + exit + ;; + *) + if [[ $1 =~ $__isvalidaddr ]]; then + IP="$1" + elif [[ $1 =~ $__isint && "$1" -ge 1 && "$1" -le 65535 ]]; then + PORT="$1" + elif [[ $1 =~ $__isvalidstr ]]; then + IND="$1" + else + echo "Invalid argument!" + exit; + fi + ;; + esac + shift + done + + echo "Specified: $IP:$PORT | Index name: $IND" + + case $CTRL in + 0) # Default behaviour - full initialization (template creation and index creation) + create_template "$IP" "$PORT" + echo "Elasticsearch index template created" + create_index "$IP" "$PORT" "$IND" + echo "Elasticsearch index (papers-$IND) created" + exit + ;; + 2) # Create index, skip creating the template + create_index "$IP" "$PORT" "$IND" + echo "Elasticsearch index (papers-$IND) created" + exit + ;; + esac +fi \ No newline at end of file -- 2.39.5 From b6287509ad9e5e4ae4c07f0351606634a0cacad2 Mon Sep 17 00:00:00 2001 From: danny-mhlv Date: Wed, 14 Sep 2022 13:12:51 +0300 Subject: [PATCH 03/13] Implemented In-Memory-Caching with default CacheModule --- .../controller/papers.controller.ts | 3 +- src/core/interceptors/page.interceptor.ts | 234 +++++++----------- src/infrastructure/modules/search.module.ts | 3 +- src/test/e2e/papers.controller.e2e.spec.ts | 2 +- src/test/page.interceptor.spec.ts | 20 +- src/test/papers.controller.spec.ts | 7 +- 6 files changed, 108 insertions(+), 161 deletions(-) diff --git a/src/application/controller/papers.controller.ts b/src/application/controller/papers.controller.ts index 74efcdf..3151334 100644 --- a/src/application/controller/papers.controller.ts +++ b/src/application/controller/papers.controller.ts @@ -1,4 +1,4 @@ -import { Controller, Get, HttpCode, Param, ParseUUIDPipe, Req, UseInterceptors } from "@nestjs/common"; +import { CacheInterceptor, Controller, Get, HttpCode, Inject, Param, ParseUUIDPipe, Req, UseInterceptors } from "@nestjs/common"; import { SearchService } from "../../core/services/common/search.service"; import { PageInterceptor } from "../../core/interceptors/page.interceptor"; import { ApiExtraModels, ApiGatewayTimeoutResponse, ApiOperation, ApiResponse, ApiTags, getSchemaPath } from "@nestjs/swagger"; @@ -13,6 +13,7 @@ import { EsHitDto, EsResponseDto, PageDto, PaperDto } from "../../core/domain"; path: 'papers', }) @ApiExtraModels(RequestDto, EsHitDto, EsResponseDto) +// @UseInterceptors(CacheInterceptor) export class PapersController { constructor(private searchService: SearchService) {} diff --git a/src/core/interceptors/page.interceptor.ts b/src/core/interceptors/page.interceptor.ts index 76e1934..01eda50 100644 --- a/src/core/interceptors/page.interceptor.ts +++ b/src/core/interceptors/page.interceptor.ts @@ -1,6 +1,6 @@ import { HttpService } from "@nestjs/axios"; -import { CallHandler, ExecutionContext, Injectable, NestInterceptor } from "@nestjs/common"; -import { Observable, map, take } from "rxjs"; +import { CACHE_MANAGER, CallHandler, ExecutionContext, Inject, Injectable, NestInterceptor } from "@nestjs/common"; +import { Observable, map, take, switchMap } from "rxjs"; import { PageDto } from "../domain/dtos"; import { EsQueryDto } from "../domain/dtos/elastic/es-query.dto"; import { RequestDto } from "../domain/dtos/request.dto"; @@ -9,62 +9,7 @@ import { EsTime } from "../domain/enums/es-time.enum"; import { Order } from "../domain/enums/page-order.enum"; import { PageMeta } from "../domain/interfaces"; import { EsPit } from "../domain/interfaces/elastic/es-pit.interface"; - -/** - * Previous search data storage - */ -class PrevSearch { - /** - * Constructs an uninitialized object - */ - constructor() { - this.pit = undefined; - this.tiebreaker = undefined; - this.prevPage = -1; - } - - /** - * PIT object of the previous search - */ - private pit: EsPit; - set _pit(pit: EsPit) { - this.pit = pit; - } - get _pit(): EsPit { - return this.pit; - } - - /** - * Tiebreaker and sort parameters - */ - private tiebreaker: unknown[]; - set _tiebreaker(tiebreaker: unknown[]) { - this.tiebreaker = tiebreaker; - } - get _tiebreaker(): unknown[] { - return this.tiebreaker; - } - - /** - * Number of the previous page - */ - private prevPage: number; - set _prevPage(page: number) { - this.prevPage = page; - } - get _prevPage(): number { - return this.prevPage; - } - - /** - * Checks if there was the search before current one - * @returns true/false, showing whether or not there was another search before - */ - public isSet(): boolean { - if (this.pit && this.tiebreaker && this.prevPage !== -1) return true; - return false; - } -} +import { Cache } from 'cache-manager' /** * Pagination-implementing interceptor @@ -73,95 +18,13 @@ class PrevSearch { export class PageInterceptor implements NestInterceptor { /** * Injects needed dependencies and instantiates the storage object - * @param httpService - * @param searchService + * @param httpService + * @param searchService */ - constructor(private readonly httpService: HttpService) { - this.prevSearch = new PrevSearch; - } - - /** - * Override of intercept() method, specified in NestInterceptor interface - * @param context - * @param next - * @returns Page with content and metadata - */ - async intercept(context: ExecutionContext, next: CallHandler): Promise> { - let request: RequestDto = context.switchToHttp().getRequest(); - const query: SearchQueryDto = request.query; - let reverse: boolean = false; - - request.es_query = new EsQueryDto(); - - request.es_query.query = { - query_string: { - query: query.query, - default_field: 'content', - } - }; - request.es_query.sort = [ - { _score: { order: !query?.order ? Order.DESC : query.order } }, - { _shard_doc: 'desc' } - ]; - - if (this.prevSearch.isSet()) { - request.es_query.pit = this.prevSearch._pit; - request.es_query.search_after = this.prevSearch._tiebreaker; - - let limit = !query?.limit ? 10 : query.limit; - request.es_query.size = limit * Math.abs(query.page - this.prevSearch._prevPage); - - if (query.page < this.prevSearch._prevPage) { - request.es_query.sort = [{ _score: { order: 'asc' } }]; - request.es_query.size += limit - 1; - reverse = true; - } else if (query.page == this.prevSearch._prevPage) { - // Caching should be HERE - request.es_query.sort = [{ _score: { order: 'asc' } }]; - reverse = true; - } - } else { - this.prevSearch._pit = request.es_query.pit = await this.getPIT(1); - - let limit = !query?.limit ? 10 : query.limit; - request.es_query.size = limit * query.page; - } - - return next.handle().pipe( - map((res) => { - // Setting the page meta-data - let meta: PageMeta = { - total: res.hits.total.value, - pagenum: !query?.page ? 1 : +query.page, - order: query?.order?.toUpperCase() === Order.ASC ? Order.ASC : Order.DESC, - pagesize: !query?.limit ? 10 : query.limit, - hasNext: undefined, - hasPrev: undefined, - }; - meta.hasNext = meta.pagenum * meta.pagesize < meta.total ? true : false; - meta.hasPrev = meta.pagenum != 1 ? true : false; - - // Saving the search info - this.prevSearch._pit.id = res.pit_id; - this.prevSearch._tiebreaker = res.hits.hits[res.hits.hits.length - 1]?.sort; - this.prevSearch._prevPage = query.page; - - // Check if the performed search is a backwards search - let data = res.hits.hits.slice(-meta.pagesize); - if (reverse) { - this.prevSearch._tiebreaker = data[0]?.sort; - data.reverse(); - reverse = false; - } - - // Omitting the redundant info and leaving only the document - data = data.map((el) => el._source); - - // Return the page - return new PageDto(data, meta); - }) - ); - } + constructor( + private readonly httpService: HttpService, + @Inject(CACHE_MANAGER) private cacheManager: Cache + ) {} /** * Elastichsearch server port-number @@ -174,9 +37,84 @@ export class PageInterceptor implements NestInterceptor { private readonly ES_IP = process.env.ES_CONTAINER_NAME; /** - * Info about previously completed search + * Override of intercept() method, specified in NestInterceptor interface + * @param context + * @param next + * @returns Page with content and metadata */ - private prevSearch: PrevSearch; + async intercept(context: ExecutionContext, next: CallHandler): Promise> { + const request: RequestDto = context.switchToHttp().getRequest(); + const query: SearchQueryDto = request.query; + let reverse: boolean = false; + + request.es_query = new EsQueryDto(); + request.es_query.query = { + query_string: { + query: query.query, + default_field: 'content', + } + }; + request.es_query.sort = [ + { _score: { order: !query?.order ? Order.DESC : query.order } }, + { _shard_doc: 'desc' } + ]; + + const limit = !query?.limit ? 10 : query.limit; + + if (await this.cacheManager.get('prevPage')) { + if (query.page == (await this.cacheManager.get('_pagenum'))) return await this.cacheManager.get('prevPage'); + + request.es_query.pit = await this.cacheManager.get('_pit'); + request.es_query.search_after = await this.cacheManager.get('_sa'); + request.es_query.size = limit * Math.abs(query.page - (await this.cacheManager.get('_pagenum'))); + + if (query.page < (await this.cacheManager.get('_pagenum'))) { + request.es_query.sort = [{ _score: { order: Order.ASC } }]; + request.es_query.size += limit - 1; + reverse = true; + } + } else { + request.es_query.pit = await this.getPIT(1); + request.es_query.size = limit * query.page; + } + + return next.handle().pipe( + switchMap(async (res) => { + // Setting the page meta-data + let meta: PageMeta = { + total: res.hits.total.value, + pagenum: !query?.page ? 1 : +query.page, + order: query?.order?.toUpperCase() === Order.ASC ? Order.ASC : Order.DESC, + pagesize: !query?.limit ? 10 : query.limit, + hasNext: undefined, + hasPrev: undefined, + }; + meta.hasNext = meta.pagenum * meta.pagesize < meta.total ? true : false; + meta.hasPrev = meta.pagenum != 1 ? true : false; + + // Saving the search info + await this.cacheManager.set('_pit', { id: res.pit_id, keep_alive: `1${EsTime.min}` }) + await this.cacheManager.set('_sa', res.hits.hits[res.hits.hits.length - 1]?.sort); + await this.cacheManager.set('_pagenum', query.page); + + // Check if the performed search is a backwards search + let data = res.hits.hits.slice(-meta.pagesize); + if (reverse) { + this.cacheManager.set('_sa', data[0]?.sort); + data.reverse(); + reverse = false; + } + + // Omitting the redundant info and leaving only the document + data = data.map((el) => el._source); + + // Cache and return the page + const page: PageDto = new PageDto(data, meta); + await this.cacheManager.set('prevPage', page); + return page; + }) + ); + } /** * Acquires a PIT ID from Elasticsearch, needed for a request diff --git a/src/infrastructure/modules/search.module.ts b/src/infrastructure/modules/search.module.ts index 526c7d7..8fd2f60 100644 --- a/src/infrastructure/modules/search.module.ts +++ b/src/infrastructure/modules/search.module.ts @@ -1,5 +1,5 @@ import { HttpModule } from "@nestjs/axios"; -import { Module } from "@nestjs/common"; +import { CacheModule, Module } from "@nestjs/common"; import { PapersController } from "../../application"; import { SearchService } from "../../core/services/common/search.service"; @@ -9,6 +9,7 @@ import { SearchService } from "../../core/services/common/search.service"; @Module({ imports: [ HttpModule, + CacheModule.register(), ], exports: [SearchService], providers: [SearchService], diff --git a/src/test/e2e/papers.controller.e2e.spec.ts b/src/test/e2e/papers.controller.e2e.spec.ts index 922bdc0..2934683 100644 --- a/src/test/e2e/papers.controller.e2e.spec.ts +++ b/src/test/e2e/papers.controller.e2e.spec.ts @@ -75,7 +75,7 @@ describe('E2E Testing of /papers', () => { let httpGetSpy = jest.spyOn(httpService, 'get').mockReturnValueOnce(axiosRes); const test = await request(app.getHttpServer()) - .get('/papers/2d3dc418-7778-abab-b33f-3d63aa25db41') // ??? Fetch a random object from DB + .get('/papers/2d3dc418-7778-abab-b33f-3d63aa25db41') .expect(200); // Expect HttpService.get() method to be touched diff --git a/src/test/page.interceptor.spec.ts b/src/test/page.interceptor.spec.ts index e3fda46..5a60f02 100644 --- a/src/test/page.interceptor.spec.ts +++ b/src/test/page.interceptor.spec.ts @@ -1,4 +1,5 @@ import { HttpService } from "@nestjs/axios"; +import { CacheModule, CACHE_MANAGER } from "@nestjs/common"; import { ConfigModule } from "@nestjs/config"; import { Test } from "@nestjs/testing"; import { Observable, of } from "rxjs"; @@ -44,7 +45,8 @@ describe('Unit tests for PageInterceptor', () => { isGlobal: true, cache: true, expandVariables: true, - }) + }), + CacheModule.register() ], }).compile(); @@ -159,7 +161,7 @@ describe('Unit tests for PageInterceptor', () => { }); }); - it('Should reverse the search results', () => { + it('Should reverse the search results', async () => { execCtxMock.getRequest.mockReturnValueOnce({ query: { page: 1, @@ -168,10 +170,8 @@ describe('Unit tests for PageInterceptor', () => { } }); - pageInter['prevSearch']._prevPage = 3; - pageInter['prevSearch'].isSet = jest.fn().mockImplementationOnce(() => { - return true; - }) + await pageInter['cacheManager'].set('_pagenum', 3); + await pageInter['cacheManager'].set('prevPage', { set: "yes" }); callHandlerMock.handle.mockReturnValueOnce( of({ @@ -187,8 +187,8 @@ describe('Unit tests for PageInterceptor', () => { ); pageInter.intercept(execCtxMock, callHandlerMock).then((res) => { - res.subscribe((page) => { - expect(pageInter['prevSearch']._tiebreaker).toEqual(['1', 'less relevant']); + res.subscribe(async (page) => { + expect(await pageInter['cacheManager'].get('_sa')).toEqual(['1', 'less relevant']); expect(page.data).toEqual(['3', '2', '1']); }); }); @@ -315,4 +315,8 @@ describe('Unit tests for PageInterceptor', () => { expect(pageInter.deletePIT('')).resolves.toBe(true); }); }); + + afterEach(() => { + pageInter['cacheManager'].reset(); + }) }); \ No newline at end of file diff --git a/src/test/papers.controller.spec.ts b/src/test/papers.controller.spec.ts index 5a1ff79..36c4eb4 100644 --- a/src/test/papers.controller.spec.ts +++ b/src/test/papers.controller.spec.ts @@ -1,5 +1,5 @@ import { HttpModule } from "@nestjs/axios"; -import { NotFoundException } from "@nestjs/common"; +import { CacheModule, NotFoundException } from "@nestjs/common"; import { Test } from "@nestjs/testing"; import { PapersController } from "src/application"; import { SearchService } from "src/core/services/common/search.service"; @@ -21,7 +21,10 @@ describe('Unit tests for PapersController', () => { } } ], - imports: [HttpModule] + imports: [ + HttpModule, + CacheModule.register() + ] }).compile(); papersController = moduleRef.get(PapersController); -- 2.39.5 From 090ecb4ff7aeb3f240d81086855c45bfc11f2350 Mon Sep 17 00:00:00 2001 From: danny-mhlv Date: Tue, 20 Sep 2022 14:38:26 +0300 Subject: [PATCH 04/13] Elasticsearch config-s[Ccript functionality implemented --- elastic/samples/template.json | 95 +++++++++++++++++ elastic/scripts/es_boot.sh | 191 ++++++++++++++++------------------ 2 files changed, 182 insertions(+), 104 deletions(-) create mode 100644 elastic/samples/template.json diff --git a/elastic/samples/template.json b/elastic/samples/template.json new file mode 100644 index 0000000..b9fcdcc --- /dev/null +++ b/elastic/samples/template.json @@ -0,0 +1,95 @@ +{ + "index_patterns": ["papers*"], + "priority": 1, + "template": { + "aliases": { + "papers": {} + }, + "mappings": { + "properties": { + "id": { + "type": "keyword" + }, + "title": { + "type": "text", + "analyzer": "title_analyzer" + }, + "authors": { + "type": "text" + }, + "topic": { + "type": "text" + }, + "summary": { + "type": "text" + }, + "tags": { + "type": "keyword" + }, + "content": { + "type": "text" + }, + "publisher": { + "type": "text" + } + } + }, + "settings": { + "analysis": { + "analyzer": { + "title_analyzer": { + "type": "custom", + "tokenizer": "title_engram_tokenizer" + }, + "content_analyzer_i": { + "type": "custom", + "tokenizer": "content_onchar_tokenizer", + "char_filter": [ + "markdown_token_filter" + ] + }, + "content_analyzer_s": { + "type": "custom", + "tokenizer": "content_onchar_tokenizer", + "char_filter": [ + "markdown_token_filter" + ], + "filter": [ + ] + }, + "basic_analyzer": { + + } + }, + "tokenizer": { + "title_engram_tokenizer": { + "type": "edge_ngram", + "min_gram": 2, + "max_gram": 10, + "token_chars": [ + "letter", + "digit" + ] + }, + "content_onchar_tokenizer": { + "type": "char_group", + "tokenize_on_chars": [ + "whitespace", + ".", ",", "(", ")", "-", "[", "]", "{", + "}", "#", ":", ";", "`", "!", "*" + ] + } + }, + "char_filter": { + "markdown_token_filter": { + "type": "pattern_replace", + "pattern": "[[a-z][0-9]]*://[[a-z][0-9]]*.[a-z]*", + "replacement": "" + } + }, + "filter": { + } + } + } + } +} \ No newline at end of file diff --git a/elastic/scripts/es_boot.sh b/elastic/scripts/es_boot.sh index 5e5f2eb..77c9650 100755 --- a/elastic/scripts/es_boot.sh +++ b/elastic/scripts/es_boot.sh @@ -3,105 +3,10 @@ __isint='^[0-9]+$' __isvalidstr='^[a-z0-9]+$' __isvalidaddr='^[a-z]+$|^((25[0-5]|2[0-4]?[0-9]|1[0-9]?[0-9]|[3-9][0-9]|[0-9])\.){3}(25[0-5]|2[0-4]?[0-9]|1[0-9]?[0-9]|[3-9][0-9]|[0-9])$' +__isJSONfile='^[a-z0-9\_\-]+(\.json)$' create_template() { - curl -H "Content-Type: application/json" -X PUT "$1:$2/_index_template/papers_t" -d ' - { - "index_patterns": ["papers*"], - "priority": 1, - "template": { - "aliases": { - "papers": {} - }, - "mappings": { - "properties": { - "id": { - "type": "keyword" - }, - "title": { - "type": "text", - "analyzer": "title_analyzer" - }, - "authors": { - "type": "text" - }, - "topic": { - "type": "text" - }, - "summary": { - "type": "text" - }, - "tags": { - "type": "keyword" - }, - "content": { - "type": "text" - }, - "publisher": { - "type": "text" - } - } - }, - "settings": { - "analysis": { - "analyzer": { - "title_analyzer": { - "type": "custom", - "tokenizer": "title_engram_tokenizer" - }, - "content_analyzer_i": { - "type": "custom", - "tokenizer": "content_onchar_tokenizer", - "char_filter": [ - "markdown_token_filter" - ] - }, - "content_analyzer_s": { - "type": "custom", - "tokenizer": "content_onchar_tokenizer", - "char_filter": [ - "markdown_token_filter" - ], - "filter": [ - ] - }, - "basic_analyzer": { - - } - }, - "tokenizer": { - "title_engram_tokenizer": { - "type": "edge_ngram", - "min_gram": 2, - "max_gram": 10, - "token_chars": [ - "letter", - "digit" - ] - }, - "content_onchar_tokenizer": { - "type": "char_group", - "tokenize_on_chars": [ - "whitespace", - ".", ",", "(", ")", "-", "[", "]", "{", - "}", "#", ":", ";", "`", "!", "*" - ] - } - }, - "char_filter": { - "markdown_token_filter": { - "type": "pattern_replace", - "pattern": "[[a-z][0-9]]*://[[a-z][0-9]]*.[a-z]*", - "replacement": "" - } - }, - "filter": { - } - } - } - } - } - ' + curl -H "Content-Type: application/json" -X PUT "$1:$2/_index_template/papers_t" -d @"$3" } #============================================================================================================================================================================= @@ -112,13 +17,45 @@ create_index() { #============================================================================================================================================================================= +upload_files() { + _file="$4" + _dir="$5" + + if [[ -n "$_file" ]]; then + curl -X POST "$1:$2/papers-$3/_doc/" \ + -H "Content-Type: application/json" \ + -d @"$_file" + elif [[ -n "$_dir" ]]; then + for file in "$_dir"/*; do + if [[ ! file =~ $__isJSONfile ]]; then + echo "$file is not identified as JSON. Skipping..." + continue; + fi + + curl -X POST "$1:$2/papers-$3/_doc/" \ + -H "Content-Type: application/json" \ + -d @"$file" + done + fi +} + +#============================================================================================================================================================================= + __usage=" - Usage: $(basename $0) - --------------------------------------------------------------------------- + Usage: $(basename "$0") + #-------------------------------------------------------------------------# + | *MODES* | + | | + | Note: 2 modes cannot be specified in one call. | + | | | -c, --create-only | Skip template initialization and only create | | | specified index. Result index name will be | | | 'papers-{specified name}' | | | + | -u, --updload | Uploads the specified file(s) to specified index | + |-------------------------------------------------------------------------| + | *OPTIONS* | + | | | -h | Help information | | | | -a | Specifies the address | @@ -131,7 +68,14 @@ __usage=" | | cannot be longer than 255 bytes (note: multi-byte | | | characters will count towards the limit faster) | | | Result index name will be 'papers-{specified name}' | - --------------------------------------------------------------------------- + | | + | -f, --file | Specify a JSON file that either a config or a | + | | document | + | | + | -d, --dir | Specify a directory containing documents to be | + | | uploaded to the index | + | | + #-------------------------------------------------------------------------# " #============================================================================================================================================================================= @@ -139,6 +83,8 @@ CTRL=0 if [[ "$1" == "-h" ]]; then echo "$__usage" +elif [[ -z "$1" ]]; then + echo "Use -h or --help for usage information." else while [[ $# -gt 0 ]]; do case "$1" in @@ -166,9 +112,32 @@ else echo "Index name is not specified!" fi ;; + -f | --file) + if [[ -n "$2" && $2 =~ $__isJSONfile ]]; then + FPATH="$2" + shift + else + echo "Invalid file name!" + fi + ;; + -d | --dir) + if [[ -n "$2" && -d "$2" ]]; then + DIRPATH="$2" + shift + fi + ;; -c | --create-only) + if [[ CTRL -ne 0 ]]; then echo "Incorrect use of modes. Use -h or --help."; + exit; + fi CTRL=2 ;; + -u | --upload) + if [[ CTRL -ne 0 ]]; then echo "Incorrect use of modes. Use -h or --help."; + exit; + fi + CTRL=3 + ;; -*) echo "Option '$1' is not supported" exit @@ -180,6 +149,8 @@ else PORT="$1" elif [[ $1 =~ $__isvalidstr ]]; then IND="$1" + elif [[ $1 =~ $__isJSONfile ]]; then + FPATH="$1" else echo "Invalid argument!" exit; @@ -190,18 +161,30 @@ else done echo "Specified: $IP:$PORT | Index name: $IND" + if [[ -z "$IP" || -z "$PORT" || -z "$IND" ]]; then + echo "Failed to identify target!" + exit + fi case $CTRL in 0) # Default behaviour - full initialization (template creation and index creation) - create_template "$IP" "$PORT" - echo "Elasticsearch index template created" + if [[ -z "$FPATH" ]]; then + echo "Template config-file was not specified." + fi + create_template "$IP" "$PORT" "$FPATH" + echo -e "\nElasticsearch index template created" create_index "$IP" "$PORT" "$IND" - echo "Elasticsearch index (papers-$IND) created" + echo -e "\nElasticsearch index (papers-$IND) created" exit ;; 2) # Create index, skip creating the template create_index "$IP" "$PORT" "$IND" - echo "Elasticsearch index (papers-$IND) created" + echo -e "\nElasticsearch index (papers-$IND) created" + exit + ;; + 3) # Uploads the specified file(s) to specified index + upload_files "$IP" "$PORT" "$IND" "$FPATH" "$DIRPATH" + echo -e "\nFinished uploading to index (papers-$IND)!" exit ;; esac -- 2.39.5 From d996bf679dbe3ff46924e2d3c3f9089faa80bca5 Mon Sep 17 00:00:00 2001 From: danny-mhlv Date: Wed, 21 Sep 2022 15:14:57 +0300 Subject: [PATCH 05/13] Pagination redone --- src/core/domain/dtos/elastic/es-query.dto.ts | 137 ++++++++++-------- src/core/domain/dtos/page-meta.dto.ts | 44 +----- src/core/domain/dtos/page.dto.ts | 8 +- src/core/domain/dtos/search-q.dto.ts | 71 ++++----- src/core/domain/enums/page-order.enum.ts | 7 + src/core/domain/interfaces/index.ts | 1 - .../domain/interfaces/page-meta.interface.ts | 36 ----- src/core/interceptors/page.interceptor.ts | 118 ++++++--------- src/test/e2e/papers.controller.e2e.spec.ts | 12 -- src/test/page.interceptor.spec.ts | 35 +---- 10 files changed, 168 insertions(+), 301 deletions(-) delete mode 100644 src/core/domain/interfaces/page-meta.interface.ts diff --git a/src/core/domain/dtos/elastic/es-query.dto.ts b/src/core/domain/dtos/elastic/es-query.dto.ts index 0572933..693d24c 100644 --- a/src/core/domain/dtos/elastic/es-query.dto.ts +++ b/src/core/domain/dtos/elastic/es-query.dto.ts @@ -1,5 +1,5 @@ import { ApiExtraModels, ApiProperty, ApiPropertyOptional } from "@nestjs/swagger"; -import { IsArray, IsDefined, IsInt, IsNotEmpty, IsNumber, IsObject, IsOptional } from "class-validator"; +import { IsArray, IsDefined, IsInt, IsObject, IsOptional } from "class-validator"; import { EsPit } from "../../interfaces/elastic/es-pit.interface"; import { EsQuery } from "../../interfaces/elastic/es-query.interface" @@ -13,71 +13,80 @@ import { EsQuery } from "../../interfaces/elastic/es-query.interface" */ @ApiExtraModels() export class EsQueryDto { - /** - * Maximum number of elements returned by Elasticsearch - */ - @IsOptional() - @IsDefined() - @IsNumber() - @IsInt() - @ApiPropertyOptional({ - description: 'Maximum number of elements returned by Elasticsearch', - example: 30 - }) - size?: number; - - /** - * The search query object passed to Elasticsearch - */ - @IsDefined() - @IsObject() - @ApiProperty({ - description: 'Search query object passed to Elasticsearch', - example: {}, - }) - query: EsQuery; + /** + * Offset from the start of the list of hits + */ + @IsOptional() + @IsInt() + @ApiPropertyOptional({ + description: 'Offset from the start of the list of hits', + example: 5, + }) + from?: number; - /** - * Object, that stores PIT ID and time alive - */ - @IsOptional() - @IsObject() - @ApiPropertyOptional({ - description: 'PIT object', - example: {} - }) - pit?: EsPit; + /** + * Maximum number of elements returned by Elasticsearch + */ + @IsOptional() + @IsInt() + @ApiPropertyOptional({ + description: 'Maximum number of elements returned by Elasticsearch', + example: 30 + }) + size?: number; + + /** + * The search query object passed to Elasticsearch + */ + @IsDefined() + @IsObject() + @ApiProperty({ + description: 'Search query object passed to Elasticsearch', + example: {}, + }) + query: EsQuery; - /** - * Sorting info - */ - @IsOptional() - @IsArray() - @ApiPropertyOptional({ - description: '', - example: [] - }) - sort?: unknown[]; + /** + * Object, that stores PIT ID and time alive + */ + @IsOptional() + @IsObject() + @ApiPropertyOptional({ + description: 'PIT object', + example: {} + }) + pit?: EsPit; - /** - * Pagination info - */ - @IsOptional() - @IsArray() - @ApiPropertyOptional({ - description: '', - example: [] - }) - search_after?: unknown[]; + /** + * Sorting info + */ + @IsOptional() + @IsArray() + @ApiPropertyOptional({ + description: '', + example: [] + }) + sort?: unknown[]; - /** - * Constructs an empty object - */ - constructor() { - this.size = 10; - this.query = undefined; - this.pit = undefined; - this.sort = undefined; - this.search_after = undefined; - } + /** + * Pagination info + */ + @IsOptional() + @IsArray() + @ApiPropertyOptional({ + description: '', + example: [] + }) + search_after?: unknown[]; + + /** + * Constructs an empty object + */ + constructor() { + this.size = 10; + this.query = undefined; + this.pit = undefined; + this.sort = undefined; + this.search_after = undefined; + } } \ No newline at end of file diff --git a/src/core/domain/dtos/page-meta.dto.ts b/src/core/domain/dtos/page-meta.dto.ts index 2a89285..0e4c321 100644 --- a/src/core/domain/dtos/page-meta.dto.ts +++ b/src/core/domain/dtos/page-meta.dto.ts @@ -1,8 +1,6 @@ -import { ApiExtraModels, ApiProperty, PartialType } from "@nestjs/swagger"; +import { ApiExtraModels, ApiProperty } from "@nestjs/swagger"; import { IsArray } from "class-validator"; import { Order } from "../enums"; -import { PageMeta } from "../interfaces/page-meta.interface"; -import { PaperDto } from "./paper.dto"; /** * List of allowed properties in this DTO @@ -13,7 +11,7 @@ const allowedProperties = ['total', 'pagenum', 'order', 'hasNext', 'hasPrev', 'p * Page model for pagination */ @ApiExtraModels() -export class PageMetaDto implements PageMeta { +export class PageMetaDto { /** * Total number of hits (results) acquired from the search */ @@ -24,16 +22,6 @@ export class PageMetaDto implements PageMeta { }) total: number; - /** - * Current page number - */ - @ApiProperty({ - description: 'Current page number', - minimum: 1, - example: 3 - }) - pagenum: number; - /** * Order of the elements on the page */ @@ -42,32 +30,4 @@ export class PageMetaDto implements PageMeta { example: Order.DESC }) order: Order; - - /** - * Flag, that shows if there's a page following the current one - */ - @ApiProperty({ - description: 'Flag, that shows if there\'s a page following the current one', - example: true - }) - hasNext: boolean; - - /** - * Flag, that shows if there's a page preceding the current one - */ - @ApiProperty({ - description: 'Flag, that shows if there\'s a page preceding the current one', - example: true - }) - hasPrev: boolean; - - /** - * Maximum number of elements on the page - */ - @ApiProperty({ - description: 'Maximum number of elements on the page', - minimum: 1, - example: 20 - }) - pagesize: number; } \ No newline at end of file diff --git a/src/core/domain/dtos/page.dto.ts b/src/core/domain/dtos/page.dto.ts index 785b1e0..189f7db 100644 --- a/src/core/domain/dtos/page.dto.ts +++ b/src/core/domain/dtos/page.dto.ts @@ -1,7 +1,5 @@ import { ApiExtraModels, ApiProperty, PartialType } from "@nestjs/swagger"; import { IsArray } from "class-validator"; -import { Order } from "../enums"; -import { PageMeta } from "../interfaces/page-meta.interface"; import { PageMetaDto } from "./page-meta.dto"; import { PaperDto } from "./paper.dto"; @@ -22,7 +20,7 @@ export class PageDto { @ApiProperty({ description: 'All data (papers) the page contains', isArray: true, - type: PaperDto + type: PaperDto, }) readonly data: PaperDto[]; @@ -31,7 +29,7 @@ export class PageDto { */ @ApiProperty({ description: 'Metadata for the page', - // example: {}, + type: PageMetaDto, }) readonly meta: PageMetaDto; @@ -41,7 +39,7 @@ export class PageDto { * @param data * @param meta */ - constructor(data: PaperDto[], meta: PageMeta) { + constructor(data: PaperDto[], meta: PageMetaDto) { this.data = data; this.meta = meta; } diff --git a/src/core/domain/dtos/search-q.dto.ts b/src/core/domain/dtos/search-q.dto.ts index 8834657..ee4d929 100644 --- a/src/core/domain/dtos/search-q.dto.ts +++ b/src/core/domain/dtos/search-q.dto.ts @@ -1,4 +1,4 @@ -import { ApiExtraModels, ApiProperty } from "@nestjs/swagger"; +import { ApiExtraModels, ApiPropertyOptional } from "@nestjs/swagger"; import { IsDefined, IsInt, IsNotEmpty, IsOptional, IsString } from "class-validator"; /** @@ -12,63 +12,64 @@ const allowedProperties = ['query', 'pagen', 'limit', 'order']; @ApiExtraModels() export class SearchQueryDto { /** - * Given query string to perform the - * search on. + * Given query string to perform the search on. */ @IsDefined() @IsNotEmpty() @IsString() - @ApiProperty({ - description: 'query', - example: 'Particle Accelerator' + @ApiPropertyOptional({ + description: 'Given query string to perform the search on', + example: 'Particle Accelerator', }) query: string; - - /** - * Page number to display. - */ - @IsDefined() - @IsNotEmpty() - @IsInt() - @ApiProperty({ - description: 'page', - example: 3, - }) - page: number; /** * Limits the number of displayed elements. */ @IsOptional() @IsInt() - @ApiProperty({ - description: 'limit', + @ApiPropertyOptional({ + description: 'Limits the number of displayed elements', example: 10, }) - limit: number; + limit?: number; /** - * Limits the number of displayed elements. + * Offset from the start of the list of hits. + */ + @IsOptional() + @IsInt() + @ApiPropertyOptional({ + description: 'Offset from the start of the list of hits', + example: 0, + }) + offset?: number; + + /** + * Indicates in which order elements need to be displayed. */ @IsOptional() @IsString() - @ApiProperty({ - description: 'order', + @ApiPropertyOptional({ + description: 'Indicates in which order elements need to be displayed', example: 'asc', }) - order: string; + order?: string; - /** - * Constructs an object with provided parameters - * @param query - * @param page - * @param limit - * @param order - */ - constructor(query: string, page: number, limit: number, order: string) { + /** + * + */ + + /** + * Constructs an object with provided parameters + * @param query + * @param page + * @param limit + * @param order + */ + constructor(query: string, page: number, limit: number, order: string) { this.query = query; - this.page = page; this.limit = limit; this.order = order; - } + } } \ No newline at end of file diff --git a/src/core/domain/enums/page-order.enum.ts b/src/core/domain/enums/page-order.enum.ts index ff4a505..234f858 100644 --- a/src/core/domain/enums/page-order.enum.ts +++ b/src/core/domain/enums/page-order.enum.ts @@ -11,4 +11,11 @@ export enum Order { * Descending order */ DESC = 'desc', +} + +export function toOrder(str: string): Order { + switch (str) { + case 'asc': return Order.ASC; + case 'desc': return Order.DESC; + } } \ No newline at end of file diff --git a/src/core/domain/interfaces/index.ts b/src/core/domain/interfaces/index.ts index 6295a5f..3b9d09e 100644 --- a/src/core/domain/interfaces/index.ts +++ b/src/core/domain/interfaces/index.ts @@ -1,5 +1,4 @@ export * from './http-response.interface' -export * from './page-meta.interface' export * from './search-info.interface' export * from './elastic/es-query.interface' export * from './elastic/es-query-string.interface' diff --git a/src/core/domain/interfaces/page-meta.interface.ts b/src/core/domain/interfaces/page-meta.interface.ts deleted file mode 100644 index 62cc9a5..0000000 --- a/src/core/domain/interfaces/page-meta.interface.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { Order } from "../enums/page-order.enum"; - -/** - * Structure of page metadata - */ -export interface PageMeta { - /** - * Total search results - */ - total: number; - - /** - * Number of the page - */ - pagenum: number; - - /** - * Order of the elements on the page - */ - order: Order; - - /** - * Flag that indicates presence of the next page - */ - hasNext: boolean; - - /** - * Flag that indicates presence of the previous page - */ - hasPrev: boolean; - - /** - * Number of elements on the page - */ - pagesize: number; -} \ No newline at end of file diff --git a/src/core/interceptors/page.interceptor.ts b/src/core/interceptors/page.interceptor.ts index 01eda50..6efaf93 100644 --- a/src/core/interceptors/page.interceptor.ts +++ b/src/core/interceptors/page.interceptor.ts @@ -6,10 +6,10 @@ import { EsQueryDto } from "../domain/dtos/elastic/es-query.dto"; import { RequestDto } from "../domain/dtos/request.dto"; import { SearchQueryDto } from "../domain/dtos/search-q.dto"; import { EsTime } from "../domain/enums/es-time.enum"; -import { Order } from "../domain/enums/page-order.enum"; -import { PageMeta } from "../domain/interfaces"; +import { Order, toOrder } from "../domain/enums/page-order.enum"; import { EsPit } from "../domain/interfaces/elastic/es-pit.interface"; import { Cache } from 'cache-manager' +import { PageMetaDto } from "../domain/dtos/page-meta.dto"; /** * Pagination-implementing interceptor @@ -45,8 +45,12 @@ export class PageInterceptor implements NestInterceptor { async intercept(context: ExecutionContext, next: CallHandler): Promise> { const request: RequestDto = context.switchToHttp().getRequest(); const query: SearchQueryDto = request.query; - let reverse: boolean = false; + const offset = !query.offset ? 0 : query.offset; + const limit = !query.limit ? 10 : query.limit; + const order = !query.order ? Order.DESC : query.order; + + // Contruct a body for querying Elasticsearch request.es_query = new EsQueryDto(); request.es_query.query = { query_string: { @@ -54,63 +58,33 @@ export class PageInterceptor implements NestInterceptor { default_field: 'content', } }; + request.es_query.from = offset; + request.es_query.size = limit; request.es_query.sort = [ - { _score: { order: !query?.order ? Order.DESC : query.order } }, - { _shard_doc: 'desc' } - ]; + { "_score": { "order": order } }, + ]; - const limit = !query?.limit ? 10 : query.limit; - - if (await this.cacheManager.get('prevPage')) { - if (query.page == (await this.cacheManager.get('_pagenum'))) return await this.cacheManager.get('prevPage'); - - request.es_query.pit = await this.cacheManager.get('_pit'); - request.es_query.search_after = await this.cacheManager.get('_sa'); - request.es_query.size = limit * Math.abs(query.page - (await this.cacheManager.get('_pagenum'))); - - if (query.page < (await this.cacheManager.get('_pagenum'))) { - request.es_query.sort = [{ _score: { order: Order.ASC } }]; - request.es_query.size += limit - 1; - reverse = true; - } - } else { - request.es_query.pit = await this.getPIT(1); - request.es_query.size = limit * query.page; + const prev_page = await this.cacheManager.get('prev_page'); + if (prev_page) { + if (offset == prev_page[1] && limit == prev_page[2]) return prev_page[0]; } return next.handle().pipe( switchMap(async (res) => { // Setting the page meta-data - let meta: PageMeta = { + let meta: PageMetaDto = { total: res.hits.total.value, - pagenum: !query?.page ? 1 : +query.page, - order: query?.order?.toUpperCase() === Order.ASC ? Order.ASC : Order.DESC, - pagesize: !query?.limit ? 10 : query.limit, - hasNext: undefined, - hasPrev: undefined, + order: toOrder(order), }; - meta.hasNext = meta.pagenum * meta.pagesize < meta.total ? true : false; - meta.hasPrev = meta.pagenum != 1 ? true : false; - - // Saving the search info - await this.cacheManager.set('_pit', { id: res.pit_id, keep_alive: `1${EsTime.min}` }) - await this.cacheManager.set('_sa', res.hits.hits[res.hits.hits.length - 1]?.sort); - await this.cacheManager.set('_pagenum', query.page); // Check if the performed search is a backwards search - let data = res.hits.hits.slice(-meta.pagesize); - if (reverse) { - this.cacheManager.set('_sa', data[0]?.sort); - data.reverse(); - reverse = false; - } - + let data = res.hits.hits; // Omitting the redundant info and leaving only the document data = data.map((el) => el._source); // Cache and return the page const page: PageDto = new PageDto(data, meta); - await this.cacheManager.set('prevPage', page); + await this.cacheManager.set('prev_page', [page, offset, limit]); return page; }) ); @@ -121,19 +95,19 @@ export class PageInterceptor implements NestInterceptor { * @param alive, amount of time in minutes (defaults to 1). If time unit is not specified - defaults to minutes. * @returns PIT object containing PIT ID and keep_alive value */ - public async getPIT(alive: number, unit: EsTime = EsTime.min): Promise { - return new Promise((resolve, reject) => { - try { - this.httpService.post(`http://${this.ES_IP}:${this.ES_PORT}/papers/_pit?keep_alive=${alive+unit}`) - .pipe(take(1), map(axiosRes => axiosRes.data)) - .subscribe((res: EsPit) => { - res.keep_alive = alive + unit; - resolve(res); - }); - } catch (error) { - reject(error); - } - }); + public async getPIT(alive: number, unit: EsTime = EsTime.min): Promise { + return new Promise((resolve, reject) => { + try { + this.httpService.post(`http://${this.ES_IP}:${this.ES_PORT}/papers/_pit?keep_alive=${alive+unit}`) + .pipe(take(1), map(axiosRes => axiosRes.data)) + .subscribe((res: EsPit) => { + res.keep_alive = alive + unit; + resolve(res); + }); + } catch (error) { + reject(error); + } + }); } /** @@ -141,20 +115,20 @@ export class PageInterceptor implements NestInterceptor { * @param pitID, ID of the PIT, that would be deleted * @returns true/false, depending on the result of deletion of the PIT */ - async deletePIT(pitID: string): Promise { - return new Promise((resolve, reject) => { - try { - this.httpService.delete(`http://${this.ES_IP}:${this.ES_PORT}/_pit`, { - data: { id: pitID }, - headers: { 'Content-Type': 'application/json' }, - }) - .pipe(take(1), map(axiosRes => axiosRes.data)) - .subscribe((res) => { - resolve(res.succeeded); - }); - } catch (error) { - reject(error); - } - }) + async deletePIT(pitID: string): Promise { + return new Promise((resolve, reject) => { + try { + this.httpService.delete(`http://${this.ES_IP}:${this.ES_PORT}/_pit`, { + data: { id: pitID }, + headers: { 'Content-Type': 'application/json' }, + }) + .pipe(take(1), map(axiosRes => axiosRes.data)) + .subscribe((res) => { + resolve(res.succeeded); + }); + } catch (error) { + reject(error); + } + }) } } \ No newline at end of file diff --git a/src/test/e2e/papers.controller.e2e.spec.ts b/src/test/e2e/papers.controller.e2e.spec.ts index 2934683..d5bba5c 100644 --- a/src/test/e2e/papers.controller.e2e.spec.ts +++ b/src/test/e2e/papers.controller.e2e.spec.ts @@ -205,20 +205,8 @@ describe('E2E Testing of /papers', () => { expect(test.body.meta.total).toBeDefined(); expect(test.body.meta.total).toBe(2); - expect(test.body.meta.pagenum).toBeDefined(); - expect(test.body.meta.pagenum).toBe(1); - expect(test.body.meta.order).toBeDefined(); expect(test.body.meta.order).toBe(Order.DESC); - - expect(test.body.meta.pagesize).toBeDefined(); - expect(test.body.meta.pagesize).toBe(10); - - expect(test.body.meta.hasNext).toBeDefined(); - expect(test.body.meta.hasNext).toBe(false); - - expect(test.body.meta.hasPrev).toBeDefined(); - expect(test.body.meta.hasPrev).toBe(false); }); afterAll(async () => { diff --git a/src/test/page.interceptor.spec.ts b/src/test/page.interceptor.spec.ts index 5a60f02..d60c8f1 100644 --- a/src/test/page.interceptor.spec.ts +++ b/src/test/page.interceptor.spec.ts @@ -159,40 +159,7 @@ describe('Unit tests for PageInterceptor', () => { }); }); }); - }); - - it('Should reverse the search results', async () => { - execCtxMock.getRequest.mockReturnValueOnce({ - query: { - page: 1, - order: 'desc', - limit: 3 - } - }); - - await pageInter['cacheManager'].set('_pagenum', 3); - await pageInter['cacheManager'].set('prevPage', { set: "yes" }); - - callHandlerMock.handle.mockReturnValueOnce( - of({ - hits: { - total: { value: 1 }, - hits: [ - { sort: ['1', 'less relevant'], _source: '1' }, - { sort: ['2', 'average'], _source: '2' }, - { sort: ['3', 'most relevant'], _source: '3' } - ] - } - }) - ); - - pageInter.intercept(execCtxMock, callHandlerMock).then((res) => { - res.subscribe(async (page) => { - expect(await pageInter['cacheManager'].get('_sa')).toEqual(['1', 'less relevant']); - expect(page.data).toEqual(['3', '2', '1']); - }); - }); - }); + }); }); describe('getPIT()', () => { -- 2.39.5 From f4f01fe8c07fdd7b1939fa11939a10fdf1cde3bb Mon Sep 17 00:00:00 2001 From: danny-mhlv Date: Thu, 22 Sep 2022 13:31:22 +0300 Subject: [PATCH 06/13] Fixed order-change --- src/core/interceptors/page.interceptor.ts | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/src/core/interceptors/page.interceptor.ts b/src/core/interceptors/page.interceptor.ts index 6efaf93..73c2089 100644 --- a/src/core/interceptors/page.interceptor.ts +++ b/src/core/interceptors/page.interceptor.ts @@ -1,6 +1,6 @@ import { HttpService } from "@nestjs/axios"; import { CACHE_MANAGER, CallHandler, ExecutionContext, Inject, Injectable, NestInterceptor } from "@nestjs/common"; -import { Observable, map, take, switchMap } from "rxjs"; +import { Observable, map, take, switchMap, of } from "rxjs"; import { PageDto } from "../domain/dtos"; import { EsQueryDto } from "../domain/dtos/elastic/es-query.dto"; import { RequestDto } from "../domain/dtos/request.dto"; @@ -50,6 +50,11 @@ export class PageInterceptor implements NestInterceptor { const limit = !query.limit ? 10 : query.limit; const order = !query.order ? Order.DESC : query.order; + const prev_page = await this.cacheManager.get('prev_page'); + if (prev_page) { + if (offset == prev_page[1] && limit == prev_page[2] && order == prev_page[3]) return of(prev_page[0]); + } + // Contruct a body for querying Elasticsearch request.es_query = new EsQueryDto(); request.es_query.query = { @@ -60,14 +65,6 @@ export class PageInterceptor implements NestInterceptor { }; request.es_query.from = offset; request.es_query.size = limit; - request.es_query.sort = [ - { "_score": { "order": order } }, - ]; - - const prev_page = await this.cacheManager.get('prev_page'); - if (prev_page) { - if (offset == prev_page[1] && limit == prev_page[2]) return prev_page[0]; - } return next.handle().pipe( switchMap(async (res) => { @@ -81,10 +78,12 @@ export class PageInterceptor implements NestInterceptor { let data = res.hits.hits; // Omitting the redundant info and leaving only the document data = data.map((el) => el._source); + // Change the order if set + if (order == Order.ASC) data.reverse(); // Cache and return the page const page: PageDto = new PageDto(data, meta); - await this.cacheManager.set('prev_page', [page, offset, limit]); + await this.cacheManager.set('prev_page', [page, offset, limit, order]); return page; }) ); -- 2.39.5 From e0596d409f366def101d724bcd94125edf65b3ad Mon Sep 17 00:00:00 2001 From: danny-mhlv Date: Thu, 22 Sep 2022 17:28:14 +0300 Subject: [PATCH 07/13] Implemented HTTP-exceptions filter --- documentation/classes/EsQueryDto.html | 197 ++++++---- .../classes/HttpExceptionFilter.html | 316 ++++++++++++++++ documentation/classes/PageDto.html | 22 +- documentation/classes/PageMetaDto.html | 230 +----------- documentation/classes/SearchQueryDto.html | 182 +++++----- .../controllers/PapersController.html | 17 +- documentation/coverage.html | 54 +-- documentation/graph/dependencies.svg | 174 ++++----- .../injectables/PageInterceptor.html | 336 ++++++------------ documentation/injectables/SearchService.html | 16 +- documentation/js/menu-wc.js | 21 +- documentation/js/menu-wc_es5.js | 2 +- documentation/js/search/search_index.js | 4 +- documentation/miscellaneous/functions.html | 76 ++++ documentation/modules/HttpResponseModule.html | 8 +- documentation/modules/LoggerModule.html | 8 +- .../modules/LoggerModule/dependencies.svg | 8 +- documentation/modules/SearchModule.html | 3 +- .../modules/SearchModule/dependencies.svg | 8 +- documentation/overview.html | 2 +- .../controller/papers.controller.ts | 9 +- src/core/domain/enums/page-order.enum.ts | 8 +- src/core/filters/http-exception.filter.ts | 23 ++ src/core/pipes/query-str.pipe.ts | 13 + src/core/services/common/search.service.ts | 6 +- src/infrastructure/modules/app.module.ts | 3 +- src/main.ts | 3 +- src/test/page.interceptor.spec.ts | 4 - src/test/search.service.spec.ts | 34 +- 29 files changed, 978 insertions(+), 809 deletions(-) create mode 100644 documentation/classes/HttpExceptionFilter.html create mode 100644 src/core/filters/http-exception.filter.ts create mode 100644 src/core/pipes/query-str.pipe.ts diff --git a/documentation/classes/EsQueryDto.html b/documentation/classes/EsQueryDto.html index 3064425..2ba0be8 100644 --- a/documentation/classes/EsQueryDto.html +++ b/documentation/classes/EsQueryDto.html @@ -90,6 +90,10 @@
    +
  • + Optional + from +
  • Optional pit @@ -133,7 +137,7 @@ - + @@ -152,6 +156,48 @@

    Properties

    + + + + + + + + + + + + + + + + + + + + +
    + + + Optional + from + + +
    + Type : number + +
    + Decorators : +
    + + @IsOptional()
    @IsInt()
    @ApiPropertyOptional({description: 'Offset from the start of the list of hits', example: 5})
    +
    +
    + +
    +

    Offset from the start of the list of hits

    +
    +
    @@ -181,7 +227,7 @@ @@ -222,7 +268,7 @@ @@ -264,7 +310,7 @@ @@ -300,13 +346,13 @@ Decorators :
    - @IsOptional()
    @IsDefined()
    @IsNumber()
    @IsInt()
    @ApiPropertyOptional({description: 'Maximum number of elements returned by Elasticsearch', example: 30})
    + @IsOptional()
    @IsInt()
    @ApiPropertyOptional({description: 'Maximum number of elements returned by Elasticsearch', example: 30})
    @@ -348,7 +394,7 @@ @@ -374,7 +420,7 @@
    import { ApiExtraModels, ApiProperty, ApiPropertyOptional } from "@nestjs/swagger";
    -import { IsArray, IsDefined, IsInt, IsNotEmpty, IsNumber, IsObject, IsOptional } from "class-validator";
    +import { IsArray, IsDefined, IsInt, IsObject, IsOptional } from "class-validator";
     import { EsPit } from "../../interfaces/elastic/es-pit.interface";
     import { EsQuery } from "../../interfaces/elastic/es-query.interface"
     
    @@ -388,73 +434,82 @@ import { EsQuery } from "../../interfaces/elastic/es-query.interface"
       */
      @ApiExtraModels()
      export class EsQueryDto {
    -     /**
    -      * Maximum number of elements returned by Elasticsearch
    -      */
    -     @IsOptional()
    -     @IsDefined()
    -     @IsNumber()
    -     @IsInt()
    -     @ApiPropertyOptional({
    -         description: 'Maximum number of elements returned by Elasticsearch',
    -         example: 30
    -     })
    -     size?: number;
    -     
    -     /**
    -      * The search query object passed to Elasticsearch
    -      */
    -     @IsDefined()
    -     @IsObject()
    -     @ApiProperty({
    -         description: 'Search query object passed to Elasticsearch',
    -         example: {},
    -     })
    -     query: EsQuery;
    +   /**
    +    * Offset from the start of the list of hits
    +    */
    +   @IsOptional()
    +   @IsInt()
    +   @ApiPropertyOptional({
    +      description: 'Offset from the start of the list of hits',
    +      example: 5,
    +   })
    +   from?: number;
     
    -     /**
    -      * Object, that stores PIT ID and time alive
    -      */
    -     @IsOptional()
    -     @IsObject()
    -     @ApiPropertyOptional({
    -        description: 'PIT object',
    -        example: {}
    -     })
    -     pit?: EsPit;
    +   /**
    +    * Maximum number of elements returned by Elasticsearch
    +    */
    +    @IsOptional()
    +    @IsInt()
    +    @ApiPropertyOptional({
    +        description: 'Maximum number of elements returned by Elasticsearch',
    +        example: 30
    +    })
    +    size?: number;
    +    
    +    /**
    +     * The search query object passed to Elasticsearch
    +     */
    +    @IsDefined()
    +    @IsObject()
    +    @ApiProperty({
    +        description: 'Search query object passed to Elasticsearch',
    +        example: {},
    +    })
    +    query: EsQuery;
     
    -     /**
    -      * Sorting info
    -      */
    -     @IsOptional()
    -     @IsArray()
    -     @ApiPropertyOptional({
    -        description: '',
    -        example: []
    -     })
    -     sort?: unknown[];
    +    /**
    +     * Object, that stores PIT ID and time alive
    +     */
    +    @IsOptional()
    +    @IsObject()
    +    @ApiPropertyOptional({
    +       description: 'PIT object',
    +       example: {}
    +    })
    +    pit?: EsPit;
     
    -     /**
    -      * Pagination info
    -      */
    -     @IsOptional()
    -     @IsArray()
    -     @ApiPropertyOptional({
    -        description: '',
    -        example: []
    -     })
    -     search_after?: unknown[];
    +    /**
    +     * Sorting info
    +     */
    +    @IsOptional()
    +    @IsArray()
    +    @ApiPropertyOptional({
    +       description: '',
    +       example: []
    +    })
    +    sort?: unknown[];
     
    -     /**
    -      * Constructs an empty object
    -      */
    -     constructor() {
    -        this.size = 10;
    -        this.query = undefined;
    -        this.pit = undefined;
    -        this.sort = undefined;
    -        this.search_after = undefined;
    -     }
    +    /**
    +     * Pagination info
    +     */
    +    @IsOptional()
    +    @IsArray()
    +    @ApiPropertyOptional({
    +       description: '',
    +       example: []
    +    })
    +    search_after?: unknown[];
    +
    +    /**
    +     * Constructs an empty object
    +     */
    +    constructor() {
    +       this.size = 10;
    +       this.query = undefined;
    +       this.pit = undefined;
    +       this.sort = undefined;
    +       this.search_after = undefined;
    +    }
      }
    diff --git a/documentation/classes/HttpExceptionFilter.html b/documentation/classes/HttpExceptionFilter.html new file mode 100644 index 0000000..45f65ed --- /dev/null +++ b/documentation/classes/HttpExceptionFilter.html @@ -0,0 +1,316 @@ + + + + + + hometask documentation + + + + + + + + + + + + + + +
    +
    + + +
    +
    + + + + + + + + + + + + + + + + +
    +
    +

    +

    File

    +

    +

    + src/core/filters/http-exception.filter.ts +

    + + +

    +

    Description

    +

    +

    +

    Basic filter for processing unhabdled HTTP exceptions

    + +

    + + +

    +

    Implements

    +

    +

    + ExceptionFilter +

    + + +
    +

    Index

    +
    - +
    - +
    - +
    - +
    - +
    + + + + + + + + + + + + + + +
    +
    Methods
    +
    + +
    + + + + +
    + +

    + Methods +

    + + + + + + + + + + + + + + + + + + + +
    + + + catch + + +
    +catch(exception: HttpException, host: ArgumentsHost) +
    + +
    +

    Exception handling method

    +
    + +
    + Parameters : + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeOptionalDescription
    exception + HttpException + + No + +

    Execution object currently being processed

    + +
    host + ArgumentsHost + + No + +

    Arguments host utility object

    + +
    +
    +
    +
    +
    + Returns : void + +
    +
    + +
    +
    +
    + + + + + + + + +
    +
    import { ArgumentsHost, Catch, ExceptionFilter, HttpException } from "@nestjs/common";
    +
    +/**
    + * Basic filter for processing unhabdled HTTP exceptions
    + */
    +@Catch(HttpException)
    +export class HttpExceptionFilter implements ExceptionFilter {
    +    /**
    +     * Exception handling method
    +     * @param exception Execution object currently being processed
    +     * @param host Arguments host utility object
    +     */
    +    catch(exception: HttpException, host: ArgumentsHost) {
    +        const ctx = host.switchToHttp();
    +        const response = ctx.getResponse();
    +        const status = exception.getStatus();
    +
    +        response.status(status).json({
    +            status: status,
    +            message: exception.message,
    +        });
    +    }
    +}
    +
    + + + + + + + + + + +
    +
    +

    results matching ""

    +
      +
      +
      +

      No results matching ""

      +
      +
      + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/documentation/classes/PageDto.html b/documentation/classes/PageDto.html index 18a1357..0bb8e47 100644 --- a/documentation/classes/PageDto.html +++ b/documentation/classes/PageDto.html @@ -117,12 +117,12 @@ -constructor(data: PaperDto[], meta: PageMeta) +constructor(data: PaperDto[], meta: PageMetaDto) - + @@ -157,7 +157,7 @@ meta - PageMeta + PageMetaDto @@ -208,7 +208,7 @@ - + @@ -235,7 +235,7 @@ - Type : PageMetaDto + Type : PageMetaDto @@ -244,13 +244,13 @@ Decorators :
      - @ApiProperty({description: 'Metadata for the page'})
      + @ApiProperty({description: 'Metadata for the page', type: PageMetaDto})
      - + @@ -277,8 +277,6 @@
      import { ApiExtraModels, ApiProperty, PartialType } from "@nestjs/swagger";
       import { IsArray } from "class-validator";
      -import { Order } from "../enums";
      -import { PageMeta } from "../interfaces/page-meta.interface";
       import { PageMetaDto } from "./page-meta.dto";
       import { PaperDto } from "./paper.dto";
       
      @@ -299,7 +297,7 @@ export class PageDto {
           @ApiProperty({
               description: 'All data (papers) the page contains',
               isArray: true,
      -        type: PaperDto
      +        type: PaperDto,
           })
           readonly data: PaperDto[];
       
      @@ -308,7 +306,7 @@ export class PageDto {
            */
           @ApiProperty({
               description: 'Metadata for the page',
      -        // example: {},
      +        type: PageMetaDto,
               
           })
           readonly meta: PageMetaDto;
      @@ -318,7 +316,7 @@ export class PageDto {
            * @param data 
            * @param meta 
            */
      -    constructor(data: PaperDto[], meta: PageMeta) {
      +    constructor(data: PaperDto[], meta: PageMetaDto) {
               this.data = data;
               this.meta = meta;
           }
      diff --git a/documentation/classes/PageMetaDto.html b/documentation/classes/PageMetaDto.html
      index 17aa765..e1bd77c 100644
      --- a/documentation/classes/PageMetaDto.html
      +++ b/documentation/classes/PageMetaDto.html
      @@ -76,12 +76,6 @@
                   

      -

      -

      Implements

      -

      -

      - PageMeta -

      @@ -96,21 +90,9 @@
        -
      • - hasNext -
      • -
      • - hasPrev -
      • order
      • -
      • - pagenum -
      • -
      • - pagesize -
      • total
      • @@ -133,88 +115,6 @@

        Properties

        - - - - - - - - - - - - - - - - - - - - -
        - - - hasNext - - -
        - Type : boolean - -
        - Decorators : -
        - - @ApiProperty({description: 'Flag, that shows if there's a page following the current one', example: true})
        -
        -
        - -
        -

        Flag, that shows if there's a page following the current one

        -
        -
        - - - - - - - - - - - - - - - - - - - - -
        - - - hasPrev - - -
        - Type : boolean - -
        - Decorators : -
        - - @ApiProperty({description: 'Flag, that shows if there's a page preceding the current one', example: true})
        -
        -
        - -
        -

        Flag, that shows if there's a page preceding the current one

        -
        -
        @@ -243,7 +143,7 @@ @@ -256,88 +156,6 @@
        - +
        - - - - - - - - - - - - - - - - - - - - -
        - - - pagenum - - -
        - Type : number - -
        - Decorators : -
        - - @ApiProperty({description: 'Current page number', minimum: 1, example: 3})
        -
        -
        - -
        -

        Current page number

        -
        -
        - - - - - - - - - - - - - - - - - - - - -
        - - - pagesize - - -
        - Type : number - -
        - Decorators : -
        - - @ApiProperty({description: 'Maximum number of elements on the page', minimum: 1, example: 20})
        -
        -
        - -
        -

        Maximum number of elements on the page

        -
        -
        @@ -366,7 +184,7 @@ @@ -391,11 +209,9 @@
        -
        import { ApiExtraModels, ApiProperty, PartialType } from "@nestjs/swagger";
        +        
        import { ApiExtraModels, ApiProperty } from "@nestjs/swagger";
         import { IsArray } from "class-validator";
         import { Order } from "../enums";
        -import { PageMeta } from "../interfaces/page-meta.interface";
        -import { PaperDto } from "./paper.dto";
         
         /**
          * List of allowed properties in this DTO
        @@ -406,7 +222,7 @@ const allowedProperties = ['total', 'pagenum', 'or
          * Page model for pagination
          */
         @ApiExtraModels()
        -export class PageMetaDto implements PageMeta {
        +export class PageMetaDto {
             /**
              * Total number of hits (results) acquired from the search
              */
        @@ -417,16 +233,6 @@ export class PageMetaDto implements PageMeta {
             })
             total: number;
         
        -    /**
        -     * Current page number
        -     */
        -    @ApiProperty({
        -        description: 'Current page number',
        -        minimum: 1,
        -        example: 3
        -    })
        -    pagenum: number;
        -
             /**
              * Order of the elements on the page
              */
        @@ -435,34 +241,6 @@ export class PageMetaDto implements PageMeta {
                 example: Order.DESC
             })
             order: Order;
        -
        -    /**
        -     * Flag, that shows if there's a page following the current one
        -     */
        -    @ApiProperty({
        -        description: 'Flag, that shows if there\'s a page following the current one',
        -        example: true
        -    })
        -    hasNext: boolean;
        -
        -    /**
        -     * Flag, that shows if there's a page preceding the current one
        -     */
        -    @ApiProperty({
        -        description: 'Flag, that shows if there\'s a page preceding the current one',
        -        example: true
        -    })
        -    hasPrev: boolean;
        -
        -    /**
        -     * Maximum number of elements on the page
        -     */
        -    @ApiProperty({
        -        description: 'Maximum number of elements on the page',
        -        minimum: 1,
        -        example: 20
        -    })
        -    pagesize: number;
         }
        diff --git a/documentation/classes/SearchQueryDto.html b/documentation/classes/SearchQueryDto.html index e30b687..f60eb1c 100644 --- a/documentation/classes/SearchQueryDto.html +++ b/documentation/classes/SearchQueryDto.html @@ -91,13 +91,16 @@ @@ -213,6 +216,7 @@ @@ -248,12 +252,55 @@
        - +
        - +
        + Optional limit @@ -229,13 +233,13 @@ Decorators :
        - @IsOptional()
        @IsInt()
        @ApiProperty({description: 'limit', example: 10})
        + @IsOptional()
        @IsInt()
        @ApiPropertyOptional({description: 'Limits the number of displayed elements', example: 10})
        - +
        + + + + + + + + + + + + + + + + + + + + +
        + + + Optional + offset + + +
        + Type : number + +
        + Decorators : +
        + + @IsOptional()
        @IsInt()
        @ApiPropertyOptional({description: 'Offset from the start of the list of hits', example: 0})
        +
        +
        + +
        +

        Offset from the start of the list of hits.

        +
        +
        - - - -
        + Optional order @@ -270,60 +317,19 @@ Decorators :
        - @IsOptional()
        @IsString()
        @ApiProperty({description: 'order', example: 'asc'})
        + @IsOptional()
        @IsString()
        @ApiPropertyOptional({description: 'Indicates in which order elements need to be displayed', example: 'asc'})
        - +
        -

        Limits the number of displayed elements.

        -
        -
        - - - - - - - - - - - - - - - - - @@ -352,20 +358,19 @@ Decorators :
        - @IsDefined()
        @IsNotEmpty()
        @IsString()
        @ApiProperty({description: 'query', example: 'Particle Accelerator'})
        + @IsDefined()
        @IsNotEmpty()
        @IsString()
        @ApiPropertyOptional({description: 'Given query string to perform the search on', example: 'Particle Accelerator'})
        @@ -384,7 +389,7 @@ search on.

        -
        import { ApiExtraModels, ApiProperty } from "@nestjs/swagger";
        +        
        import { ApiExtraModels, ApiPropertyOptional } from "@nestjs/swagger";
         import { IsDefined, IsInt, IsNotEmpty, IsOptional, IsString } from "class-validator";
         
         /**
        @@ -398,65 +403,66 @@ const allowedProperties = ['query', 'pagen', 'limi
         @ApiExtraModels()
         export class SearchQueryDto {
             /**
        -     * Given query string to perform the
        -     * search on.
        +     * Given query string to perform the search on.
              */
             @IsDefined()
             @IsNotEmpty()
             @IsString()
        -    @ApiProperty({
        -        description: 'query',
        -        example: 'Particle Accelerator'
        +    @ApiPropertyOptional({
        +        description: 'Given query string to perform the search on',
        +        example: 'Particle Accelerator',
             })
             query: string;
        -    
        -    /**
        -     * Page number to display.
        -     */
        -    @IsDefined()
        -    @IsNotEmpty()
        -    @IsInt()
        -    @ApiProperty({
        -        description: 'page',
        -        example: 3,
        -    })
        -    page: number;
         
             /**
              * Limits the number of displayed elements.
              */
             @IsOptional()
             @IsInt()
        -    @ApiProperty({
        -        description: 'limit',
        +    @ApiPropertyOptional({
        +        description: 'Limits the number of displayed elements',
                 example: 10,
             })
        -    limit: number;
        +    limit?: number;
         
             /**
        -     * Limits the number of displayed elements.
        +     * Offset from the start of the list of hits.
        +     */
        +    @IsOptional()
        +    @IsInt()
        +    @ApiPropertyOptional({
        +        description: 'Offset from the start of the list of hits',
        +        example: 0,
        +    })
        +    offset?: number;
        +
        +    /**
        +     * Indicates in which order elements need to be displayed.
              */
              @IsOptional()
              @IsString()
        -     @ApiProperty({
        -         description: 'order',
        +     @ApiPropertyOptional({
        +         description: 'Indicates in which order elements need to be displayed',
                  example: 'asc',
              })
        -     order: string;
        +     order?: string;
         
        -     /**
        -      * Constructs an object with provided parameters
        -      * @param query 
        -      * @param page 
        -      * @param limit 
        -      * @param order 
        -      */
        -     constructor(query: string, page: number, limit: number, order: string) {
        +    /**
        +     * 
        +     */
        +
        +    /**
        +     * Constructs an object with provided parameters
        +     * @param query 
        +     * @param page 
        +     * @param limit 
        +     * @param order 
        +     */
        +    constructor(query: string, page: number, limit: number, order: string) {
                 this.query = query;
        -        this.page = page;
                 this.limit = limit;
                 this.order = order;
        -     }
        +    }
         }
        diff --git a/documentation/controllers/PapersController.html b/documentation/controllers/PapersController.html index 4de01ed..7eb4ad9 100644 --- a/documentation/controllers/PapersController.html +++ b/documentation/controllers/PapersController.html @@ -135,8 +135,8 @@ @@ -214,8 +214,8 @@ @@ -270,21 +270,24 @@
        -
        import { Controller, Get, HttpCode, Param, ParseUUIDPipe, Req, UseInterceptors } from "@nestjs/common";
        +        
        import { Controller, Get, HttpCode, Param, ParseUUIDPipe, Req, UseFilters, UseInterceptors } from "@nestjs/common";
         import { SearchService } from "../../core/services/common/search.service";
         import { PageInterceptor } from "../../core/interceptors/page.interceptor";
        -import { ApiExtraModels, ApiGatewayTimeoutResponse, ApiOperation, ApiResponse, ApiTags, getSchemaPath } from "@nestjs/swagger";
        +import { ApiExtraModels, ApiGatewayTimeoutResponse, ApiOperation, ApiResponse, ApiTags } from "@nestjs/swagger";
         import { RequestDto } from "../../core/domain/dtos/request.dto";
         import { EsHitDto, EsResponseDto, PageDto, PaperDto } from "../../core/domain";
        +import { HttpExceptionFilter } from "src/core/filters/http-exception.filter";
         
         /**
          * /papers/ route controller
          */
        +@UseFilters(HttpExceptionFilter)
         @Controller({
             version: '1',
             path: 'papers',
         })
         @ApiExtraModels(RequestDto, EsHitDto, EsResponseDto)
        +// @UseInterceptors(CacheInterceptor)
         export class PapersController {
             constructor(private searchService: SearchService) {}
         
        @@ -344,7 +347,7 @@ export class PapersController {
             getByID(@Param('uuid', ParseUUIDPipe) uuid: string): Promise<PaperDto> {
                 return this.searchService.findByID(uuid).then(
                     (response: EsResponseDto) => {
        -                return response.hits.hits[0]._source;
        +                return response.hits.hits[0]?._source;
                     },
                     (error) => {
                         throw error;
        diff --git a/documentation/coverage.html b/documentation/coverage.html
        index b75c0b0..8d27efd 100644
        --- a/documentation/coverage.html
        +++ b/documentation/coverage.html
        @@ -173,7 +173,7 @@
                     
        @@ -221,7 +221,7 @@ @@ -356,6 +356,18 @@ (1/1) + + + + + + - - - - - - + + + + + + - - - - - - diff --git a/documentation/graph/dependencies.svg b/documentation/graph/dependencies.svg index 7a7c66e..7660ee6 100644 --- a/documentation/graph/dependencies.svg +++ b/documentation/graph/dependencies.svg @@ -4,217 +4,217 @@ - - + + dependencies - -dependencies - -cluster_SearchModule - - - -cluster_SearchModule_exports - - - -cluster_SearchModule_providers - - + +dependencies cluster_AppModule - + cluster_AppModule_imports - - - -cluster_CommonModule - - - -cluster_CommonModule_imports - - - -cluster_CommonModule_exports - + cluster_HttpResponseModule - + cluster_HttpResponseModule_exports - + cluster_HttpResponseModule_providers - + + + +cluster_CommonModule + + + +cluster_CommonModule_imports + + + +cluster_CommonModule_exports + cluster_LoggerModule - + cluster_LoggerModule_exports - + cluster_LoggerModule_providers - + + + +cluster_SearchModule + + + +cluster_SearchModule_exports + + + +cluster_SearchModule_providers + CommonModule - -CommonModule + +CommonModule AppModule - -AppModule + +AppModule CommonModule->AppModule - - + + HttpResponseModule - -HttpResponseModule + +HttpResponseModule CommonModule->HttpResponseModule - - + + LoggerModule - -LoggerModule + +LoggerModule CommonModule->LoggerModule - - + + SearchModule - -SearchModule + +SearchModule SearchModule->AppModule - - + + SearchService - -SearchService + +SearchService SearchModule->SearchService - - + + HttpResponseModule - -HttpResponseModule + +HttpResponseModule HttpResponseModule->CommonModule - - + + HttpResponseService - -HttpResponseService + +HttpResponseService HttpResponseModule->HttpResponseService - - + + LoggerModule - -LoggerModule + +LoggerModule LoggerModule->CommonModule - - + + LoggerService - -LoggerService + +LoggerService LoggerModule->LoggerService - - + + HttpResponseService - -HttpResponseService + +HttpResponseService HttpResponseService->HttpResponseModule - - + + LoggerService - -LoggerService + +LoggerService LoggerService->LoggerModule - - + + SearchService - -SearchService + +SearchService SearchService->SearchModule - - + + diff --git a/documentation/injectables/PageInterceptor.html b/documentation/injectables/PageInterceptor.html index 2a1c737..e17aff4 100644 --- a/documentation/injectables/PageInterceptor.html +++ b/documentation/injectables/PageInterceptor.html @@ -95,10 +95,6 @@ Readonly ES_PORT -
      • - Private - prevSearch -
      • @@ -142,12 +138,12 @@ @@ -178,6 +174,18 @@ + + + + + + + +
        - - - page - - -
        - Type : number - -
        - Decorators : -
        - - @IsDefined()
        @IsNotEmpty()
        @IsInt()
        @ApiProperty({description: 'page', example: 3})
        -
        -
        - -
        -

        Page number to display.

        +

        Indicates in which order elements need to be displayed.

        - +
        -

        Given query string to perform the -search on.

        +

        Given query string to perform the search on.

        - +
        - +
        EsQueryDto 100 % - (7/7) + (8/8)
        PageMetaDto 100 % - (7/7) + (3/3)
        + + src/core/domain/enums/page-order.enum.ts + functiontoOrder + 100 % + (1/1) +
        @@ -416,18 +428,6 @@ (6/6)
        - - src/core/domain/interfaces/page-meta.interface.ts - interfacePageMeta - 100 % - (7/7) -
        @@ -452,6 +452,18 @@ (2/2)
        + + src/core/filters/http-exception.filter.ts + classHttpExceptionFilter + 100 % + (2/2) +
        @@ -548,18 +560,6 @@ (4/4)
        - - src/core/interceptors/page.interceptor.ts - classPrevSearch - 100 % - (6/6) -
        @@ -569,7 +569,7 @@ PageInterceptor 100 % - (8/8) + (7/7)
        -constructor(httpService: HttpService) +constructor(httpService: HttpService, cacheManager: Cache)
        - +
        cacheManager + Cache + + No +
      @@ -214,8 +222,8 @@ - + @@ -294,8 +302,8 @@ - + @@ -393,8 +401,8 @@ - + @@ -483,7 +491,7 @@ - + @@ -516,7 +524,7 @@ - + @@ -529,39 +537,6 @@ - - - - - - - - - - - - - - - - - -
      - - - Private - prevSearch - - -
      - Type : PrevSearch - -
      - -
      -

      Info about previously completed search

      -
      -
      @@ -569,72 +544,17 @@
      import { HttpService } from "@nestjs/axios";
      -import { CallHandler, ExecutionContext, Injectable, NestInterceptor } from "@nestjs/common";
      -import { Observable, map, take } from "rxjs";
      +import { CACHE_MANAGER, CallHandler, ExecutionContext, Inject, Injectable, NestInterceptor } from "@nestjs/common";
      +import { Observable, map, take, switchMap, of } from "rxjs";
       import { PageDto } from "../domain/dtos";
       import { EsQueryDto } from "../domain/dtos/elastic/es-query.dto";
       import { RequestDto } from "../domain/dtos/request.dto";
       import { SearchQueryDto } from "../domain/dtos/search-q.dto";
       import { EsTime } from "../domain/enums/es-time.enum";
      -import { Order } from "../domain/enums/page-order.enum";
      -import { PageMeta } from "../domain/interfaces";
      +import { Order, toOrder } from "../domain/enums/page-order.enum";
       import { EsPit } from "../domain/interfaces/elastic/es-pit.interface";
      -
      -/**
      - * Previous search data storage
      - */
      -class PrevSearch {
      -    /**
      -     * Constructs an uninitialized object
      -     */
      -    constructor() {
      -        this.pit = undefined;
      -        this.tiebreaker = undefined;
      -        this.prevPage = -1;
      -    }
      -
      -    /**
      -     * PIT object of the previous search
      -     */
      -    private pit: EsPit;
      -    set _pit(pit: EsPit) {
      -        this.pit = pit;
      -    }
      -    get _pit(): EsPit {
      -        return this.pit;
      -    }
      -
      -    /**
      -     * Tiebreaker and sort parameters
      -     */
      -    private tiebreaker: unknown[];
      -    set _tiebreaker(tiebreaker: unknown[]) {
      -        this.tiebreaker = tiebreaker;
      -    }
      -    get _tiebreaker(): unknown[] {
      -        return this.tiebreaker;
      -    }
      -
      -    /**
      -     * Number of the previous page
      -     */
      -    private prevPage: number;
      -    set _prevPage(page: number) {
      -        this.prevPage = page;
      -    }
      -    get _prevPage(): number {
      -        return this.prevPage;
      -    }
      -
      -    /**
      -     * Checks if there was the search before current one
      -     * @returns true/false, showing whether or not there was another search before
      -     */
      -    public isSet(): boolean {
      -        if (this.pit && this.tiebreaker && this.prevPage !== -1) return true;
      -        return false;
      -    }
      -}
      +import { Cache } from 'cache-manager'
      +import { PageMetaDto } from "../domain/dtos/page-meta.dto";
       
       /**
        * Pagination-implementing interceptor
      @@ -643,95 +563,13 @@ class PrevSearch {
       export class PageInterceptor implements NestInterceptor {
           /**
            * Injects needed dependencies and instantiates the storage object
      -     * @param httpService 
      -     * @param searchService 
      +     * @param httpService
      +     * @param searchService
            */
      -    constructor(private readonly httpService: HttpService) {
      -        this.prevSearch = new PrevSearch;
      -    }
      -
      -    /**
      -     * Override of intercept() method, specified in NestInterceptor interface
      -     * @param context 
      -     * @param next 
      -     * @returns Page with content and metadata
      -     */
      -    async intercept(context: ExecutionContext, next: CallHandler<any>): Promise<Observable<PageDto>> {
      -        let request: RequestDto = context.switchToHttp().getRequest<RequestDto>();
      -        const query: SearchQueryDto = request.query;
      -        let reverse: boolean = false;
      -
      -        request.es_query = new EsQueryDto();
      -
      -        request.es_query.query = {
      -            query_string: {
      -                query: query.query,
      -                default_field: 'content',
      -            }
      -        };
      -        request.es_query.sort = [
      -            { _score: { order: !query?.order ? Order.DESC : query.order } },
      -            { _shard_doc: 'desc' }
      -        ];
      -
      -        if (this.prevSearch.isSet()) {
      -            request.es_query.pit = this.prevSearch._pit;
      -            request.es_query.search_after = this.prevSearch._tiebreaker;
      -
      -            let limit = !query?.limit ? 10 : query.limit;
      -            request.es_query.size = limit * Math.abs(query.page - this.prevSearch._prevPage);
      -            
      -            if (query.page < this.prevSearch._prevPage) {
      -                request.es_query.sort = [{ _score: { order: 'asc' } }];
      -                request.es_query.size += limit - 1;
      -                reverse = true;
      -            } else if (query.page == this.prevSearch._prevPage) {
      -                // Caching should be HERE
      -                request.es_query.sort = [{ _score: { order: 'asc' } }];
      -                reverse = true;
      -            }
      -        } else {
      -            this.prevSearch._pit = request.es_query.pit = await this.getPIT(1);
      -
      -            let limit = !query?.limit ? 10 : query.limit;
      -            request.es_query.size = limit * query.page;
      -        }
      -
      -        return next.handle().pipe(
      -            map((res) => {
      -                // Setting the page meta-data
      -                let meta: PageMeta = {
      -                    total: res.hits.total.value,
      -                    pagenum: !query?.page ? 1 : +query.page,
      -                    order: query?.order?.toUpperCase() === Order.ASC ? Order.ASC : Order.DESC,
      -                    pagesize: !query?.limit ? 10 : query.limit,
      -                    hasNext: undefined,
      -                    hasPrev: undefined,
      -                }; 
      -                meta.hasNext = meta.pagenum * meta.pagesize < meta.total ? true : false;
      -                meta.hasPrev = meta.pagenum != 1 ? true : false;
      -
      -                // Saving the search info
      -                this.prevSearch._pit.id = res.pit_id;
      -                this.prevSearch._tiebreaker = res.hits.hits[res.hits.hits.length - 1]?.sort;
      -                this.prevSearch._prevPage = query.page;
      -
      -                // Check if the performed search is a backwards search
      -                let data = res.hits.hits.slice(-meta.pagesize);
      -                if (reverse) {
      -                    this.prevSearch._tiebreaker = data[0]?.sort;
      -                    data.reverse();
      -                    reverse = false;
      -                }
      -
      -                // Omitting the redundant info and leaving only the document
      -                data = data.map((el) => el._source);
      -
      -                // Return the page
      -                return new PageDto(data, meta);
      -            })
      -        );
      -    }
      +    constructor(
      +        private readonly httpService: HttpService,
      +        @Inject(CACHE_MANAGER) private cacheManager: Cache
      +    ) {}
       
           /**
            * Elastichsearch server port-number
      @@ -744,28 +582,76 @@ export class PageInterceptor implements NestInterceptor {
            private readonly ES_IP = process.env.ES_CONTAINER_NAME;
       
           /**
      -     * Info about previously completed search
      +     * Override of intercept() method, specified in NestInterceptor interface
      +     * @param context
      +     * @param next
      +     * @returns Page with content and metadata
            */
      -    private prevSearch: PrevSearch;
      +    async intercept(context: ExecutionContext, next: CallHandler<any>): Promise<Observable<PageDto>> {
      +        const request: RequestDto = context.switchToHttp().getRequest<RequestDto>();
      +        const query: SearchQueryDto = request.query;
      +
      +        const offset = !query.offset ? 0 : query.offset;
      +        const limit = !query.limit ? 10 : query.limit; 
      +        const order = !query.order ? Order.DESC : query.order;
      +
      +        const prev_page = await this.cacheManager.get('prev_page'); 
      +        if (prev_page) {
      +            if (offset == prev_page[1] && limit == prev_page[2] && order == prev_page[3]) return of(prev_page[0]);
      +        }
      +
      +        // Contruct a body for querying Elasticsearch
      +        request.es_query = new EsQueryDto();
      +        request.es_query.query = {
      +            query_string: {
      +                query: query.query,
      +                default_field: 'content',
      +            }
      +        };
      +        request.es_query.from = offset;
      +        request.es_query.size = limit;
      +
      +        return next.handle().pipe(
      +            switchMap(async (res) => {
      +                // Setting the page meta-data
      +                let meta: PageMetaDto = {
      +                    total: res.hits.total.value,
      +                    order: toOrder(order),
      +                };
      +
      +                // Check if the performed search is a backwards search
      +                let data = res.hits.hits;
      +                // Omitting the redundant info and leaving only the document
      +                data = data.map((el) => el._source);
      +                // Change the order if set
      +                if (order == Order.ASC) data.reverse();
      +
      +                // Cache and return the page
      +                const page: PageDto = new PageDto(data, meta);
      +                await this.cacheManager.set('prev_page', [page, offset, limit, order]);
      +                return page;
      +            })
      +        );
      +    }
       
           /**
            * Acquires a PIT ID from Elasticsearch, needed for a request
            * @param alive, amount of time in minutes (defaults to 1). If time unit is not specified - defaults to minutes.
            * @returns PIT object <EsPit> containing PIT ID and keep_alive value
            */
      -     public async getPIT(alive: number, unit: EsTime = EsTime.min): Promise<EsPit> {
      -        return new Promise((resolve, reject) => {
      -            try {
      -                this.httpService.post<EsPit>(`http://${this.ES_IP}:${this.ES_PORT}/papers/_pit?keep_alive=${alive+unit}`)
      -                    .pipe(take(1), map(axiosRes => axiosRes.data))
      -                    .subscribe((res: EsPit) => {
      -                        res.keep_alive = alive + unit;
      -                        resolve(res);
      -                    });
      -            } catch (error) {
      -                reject(error);
      -            }
      -        });
      +    public async getPIT(alive: number, unit: EsTime = EsTime.min): Promise<EsPit> {
      +       return new Promise((resolve, reject) => {
      +           try {
      +               this.httpService.post<EsPit>(`http://${this.ES_IP}:${this.ES_PORT}/papers/_pit?keep_alive=${alive+unit}`)
      +                   .pipe(take(1), map(axiosRes => axiosRes.data))
      +                   .subscribe((res: EsPit) => {
      +                       res.keep_alive = alive + unit;
      +                       resolve(res);
      +                   });
      +           } catch (error) {
      +               reject(error);
      +           }
      +       });
           }
       
           /**
      @@ -773,21 +659,21 @@ export class PageInterceptor implements NestInterceptor {
            * @param pitID, ID of the PIT, that would be deleted
            * @returns true/false, depending on the result of deletion of the PIT
            */
      -     async deletePIT(pitID: string): Promise<boolean> {
      -        return new Promise((resolve, reject) => {
      -            try {
      -                this.httpService.delete(`http://${this.ES_IP}:${this.ES_PORT}/_pit`, {
      -                    data: { id: pitID },
      -                    headers: { 'Content-Type': 'application/json' },
      -                })
      -                .pipe(take(1), map(axiosRes => axiosRes.data))
      -                .subscribe((res) => {
      -                    resolve(res.succeeded);
      -                });
      -            } catch (error) {
      -                reject(error);
      -            }
      -        })
      +    async deletePIT(pitID: string): Promise<boolean> {
      +       return new Promise((resolve, reject) => {
      +           try {
      +               this.httpService.delete(`http://${this.ES_IP}:${this.ES_PORT}/_pit`, {
      +                   data: { id: pitID },
      +                   headers: { 'Content-Type': 'application/json' },
      +               })
      +               .pipe(take(1), map(axiosRes => axiosRes.data))
      +               .subscribe((res) => {
      +                   resolve(res.succeeded);
      +               });
      +           } catch (error) {
      +               reject(error);
      +           }
      +       })
           }
       }
      diff --git a/documentation/injectables/SearchService.html b/documentation/injectables/SearchService.html index a80d7ea..ab4019c 100644 --- a/documentation/injectables/SearchService.html +++ b/documentation/injectables/SearchService.html @@ -206,8 +206,8 @@ HTTPService instance

      - + @@ -411,7 +411,7 @@ HTTPService instance

      import { HttpService } from "@nestjs/axios";
      -import { GatewayTimeoutException, HttpException, Injectable } from "@nestjs/common";
      +import { BadRequestException, GatewayTimeoutException, HttpException, Injectable, NotFoundException } from "@nestjs/common";
       import { map, take } from "rxjs";
       import { EsResponseDto} from "../../domain/dtos";
       import { EsQueryDto } from "../../domain/dtos/elastic/es-query.dto";
      @@ -446,7 +446,7 @@ export class SearchService {
           async findByID(uuid: string): Promise<EsResponseDto> { // Should I change 'object' to specific DTO?
               let ESQ: EsQueryDto = new EsQueryDto;
       
      -        ESQ.size = 1;
      +        // ESQ.size = 1;
               ESQ.query = {
                   query_string: {
                       query: ('id:' + uuid),
      @@ -461,6 +461,9 @@ export class SearchService {
                       }))
                       ?.pipe(take(1), map(axiosRes => axiosRes.data))
                       .subscribe((res: EsResponseDto) => {
      +                    if (!res.hits.hits.length) {
      +                        reject(new NotFoundException);
      +                    } 
                           if (res.timed_out) {
                               reject(new GatewayTimeoutException('Elasticsearch Timed Out'));
                           }
      @@ -480,6 +483,10 @@ export class SearchService {
           async findByContext(es_query: EsQueryDto): Promise<EsResponseDto> {
               return new Promise((resolve, reject) => {
                   try {
      +                if (!es_query.query.query_string.query) {
      +                    throw new BadRequestException;
      +                }
      +
                       (this.httpService.get<EsResponseDto>(`http://${this.ES_IP}:${this.ES_PORT}/_search`, {
                           data: es_query,
                           headers: {'Content-Type': 'application/json'},
      @@ -489,7 +496,6 @@ export class SearchService {
                           if (res.timed_out) {
                               reject(new GatewayTimeoutException('Elasticsearch Timed Out'));
                           }
      -
                           resolve(res);
                       });
                   } catch (error) {
      diff --git a/documentation/js/menu-wc.js b/documentation/js/menu-wc.js
      index a3eb3db..3e0b561 100644
      --- a/documentation/js/menu-wc.js
      +++ b/documentation/js/menu-wc.js
      @@ -121,13 +121,13 @@ customElements.define('compodoc-menu', class extends HTMLElement {
                                       SearchModule
                                           
    • -
    • +
      + + + + +

      src/core/domain/enums/page-order.enum.ts

      +
      +

      + + + + + + + + + + + + + diff --git a/documentation/modules/HttpResponseModule.html b/documentation/modules/HttpResponseModule.html index a50283e..f298ffb 100644 --- a/documentation/modules/HttpResponseModule.html +++ b/documentation/modules/HttpResponseModule.html @@ -57,14 +57,14 @@ cluster_HttpResponseModule - -cluster_HttpResponseModule_providers - - cluster_HttpResponseModule_exports + +cluster_HttpResponseModule_providers + + HttpResponseService diff --git a/documentation/modules/LoggerModule.html b/documentation/modules/LoggerModule.html index d91a07c..733b70a 100644 --- a/documentation/modules/LoggerModule.html +++ b/documentation/modules/LoggerModule.html @@ -57,14 +57,14 @@ cluster_LoggerModule - -cluster_LoggerModule_providers - - cluster_LoggerModule_exports + +cluster_LoggerModule_providers + + LoggerService diff --git a/documentation/modules/LoggerModule/dependencies.svg b/documentation/modules/LoggerModule/dependencies.svg index 0321adc..5c106c0 100644 --- a/documentation/modules/LoggerModule/dependencies.svg +++ b/documentation/modules/LoggerModule/dependencies.svg @@ -14,14 +14,14 @@ cluster_LoggerModule - -cluster_LoggerModule_exports - - cluster_LoggerModule_providers + +cluster_LoggerModule_exports + + LoggerService diff --git a/documentation/modules/SearchModule.html b/documentation/modules/SearchModule.html index fbcca7c..452318a 100644 --- a/documentation/modules/SearchModule.html +++ b/documentation/modules/SearchModule.html @@ -177,7 +177,7 @@
      import { HttpModule } from "@nestjs/axios";
      -import { Module } from "@nestjs/common";
      +import { CacheModule, Module } from "@nestjs/common";
       import { PapersController } from "../../application";
       import { SearchService } from "../../core/services/common/search.service";
       
      @@ -187,6 +187,7 @@ import { SearchService } from "../../core/services/common/search.service&qu
       @Module({
           imports: [
               HttpModule,
      +        CacheModule.register(),
           ],
           exports: [SearchService],
           providers: [SearchService],
      diff --git a/documentation/modules/SearchModule/dependencies.svg b/documentation/modules/SearchModule/dependencies.svg
      index 3d5fb1d..bfd3490 100644
      --- a/documentation/modules/SearchModule/dependencies.svg
      +++ b/documentation/modules/SearchModule/dependencies.svg
      @@ -14,14 +14,14 @@
       cluster_SearchModule
       
       
      -
      -cluster_SearchModule_providers
      -
      -
       
       cluster_SearchModule_exports
       
       
      +
      +cluster_SearchModule_providers
      +
      +
       
       
       SearchService 
      diff --git a/documentation/overview.html b/documentation/overview.html
      index 91e00cb..7b5e6ed 100644
      --- a/documentation/overview.html
      +++ b/documentation/overview.html
      @@ -317,7 +317,7 @@
                   

      -

      9 Interfaces

      +

      8 Interfaces

      diff --git a/src/application/controller/papers.controller.ts b/src/application/controller/papers.controller.ts index 3151334..50b30e7 100644 --- a/src/application/controller/papers.controller.ts +++ b/src/application/controller/papers.controller.ts @@ -1,13 +1,16 @@ -import { CacheInterceptor, Controller, Get, HttpCode, Inject, Param, ParseUUIDPipe, Req, UseInterceptors } from "@nestjs/common"; +import { Body, Controller, Get, HttpCode, Param, ParseUUIDPipe, Query, Req, UseFilters, UseInterceptors, UsePipes } from "@nestjs/common"; import { SearchService } from "../../core/services/common/search.service"; import { PageInterceptor } from "../../core/interceptors/page.interceptor"; -import { ApiExtraModels, ApiGatewayTimeoutResponse, ApiOperation, ApiResponse, ApiTags, getSchemaPath } from "@nestjs/swagger"; +import { ApiExtraModels, ApiGatewayTimeoutResponse, ApiOperation, ApiResponse, ApiTags } from "@nestjs/swagger"; import { RequestDto } from "../../core/domain/dtos/request.dto"; import { EsHitDto, EsResponseDto, PageDto, PaperDto } from "../../core/domain"; +import { HttpExceptionFilter } from "src/core/filters/http-exception.filter"; +import { QueryStringPipe } from "src/core/pipes/query-str.pipe"; /** * /papers/ route controller */ +@UseFilters(HttpExceptionFilter) @Controller({ version: '1', path: 'papers', @@ -73,7 +76,7 @@ export class PapersController { getByID(@Param('uuid', ParseUUIDPipe) uuid: string): Promise { return this.searchService.findByID(uuid).then( (response: EsResponseDto) => { - return response.hits.hits[0]._source; + return response.hits.hits[0]?._source; }, (error) => { throw error; diff --git a/src/core/domain/enums/page-order.enum.ts b/src/core/domain/enums/page-order.enum.ts index 234f858..61e5176 100644 --- a/src/core/domain/enums/page-order.enum.ts +++ b/src/core/domain/enums/page-order.enum.ts @@ -13,9 +13,15 @@ export enum Order { DESC = 'desc', } +/** + * Converts string value to appropriate enum-member + * @param str String to be converted + * @returns Appropriate enum-member + */ export function toOrder(str: string): Order { switch (str) { case 'asc': return Order.ASC; case 'desc': return Order.DESC; - } + } + return; } \ No newline at end of file diff --git a/src/core/filters/http-exception.filter.ts b/src/core/filters/http-exception.filter.ts new file mode 100644 index 0000000..32db991 --- /dev/null +++ b/src/core/filters/http-exception.filter.ts @@ -0,0 +1,23 @@ +import { ArgumentsHost, Catch, ExceptionFilter, HttpException } from "@nestjs/common"; + +/** + * Basic filter for processing unhabdled HTTP exceptions + */ +@Catch(HttpException) +export class HttpExceptionFilter implements ExceptionFilter { + /** + * Exception handling method + * @param exception Execution object currently being processed + * @param host Arguments host utility object + */ + catch(exception: HttpException, host: ArgumentsHost) { + const ctx = host.switchToHttp(); + const response = ctx.getResponse(); + const status = exception.getStatus(); + + response.status(status).json({ + status: status, + message: exception.message, + }); + } +} \ No newline at end of file diff --git a/src/core/pipes/query-str.pipe.ts b/src/core/pipes/query-str.pipe.ts new file mode 100644 index 0000000..d9bfcca --- /dev/null +++ b/src/core/pipes/query-str.pipe.ts @@ -0,0 +1,13 @@ +import { ArgumentMetadata, BadRequestException, ImATeapotException, Injectable, PipeTransform } from "@nestjs/common"; +import { RequestDto } from "../domain"; + +@Injectable() +export class QueryStringPipe implements PipeTransform { + constructor() {} + + transform(value: RequestDto, metadata: ArgumentMetadata): RequestDto { + console.log(value.query.limit) + + return value; + } +} \ No newline at end of file diff --git a/src/core/services/common/search.service.ts b/src/core/services/common/search.service.ts index 609ce0d..4cd131b 100644 --- a/src/core/services/common/search.service.ts +++ b/src/core/services/common/search.service.ts @@ -1,5 +1,5 @@ import { HttpService } from "@nestjs/axios"; -import { GatewayTimeoutException, HttpException, Injectable } from "@nestjs/common"; +import { GatewayTimeoutException, Injectable, NotFoundException } from "@nestjs/common"; import { map, take } from "rxjs"; import { EsResponseDto} from "../../domain/dtos"; import { EsQueryDto } from "../../domain/dtos/elastic/es-query.dto"; @@ -49,6 +49,9 @@ export class SearchService { })) ?.pipe(take(1), map(axiosRes => axiosRes.data)) .subscribe((res: EsResponseDto) => { + if (!res.hits.hits.length) { + reject(new NotFoundException); + } if (res.timed_out) { reject(new GatewayTimeoutException('Elasticsearch Timed Out')); } @@ -77,7 +80,6 @@ export class SearchService { if (res.timed_out) { reject(new GatewayTimeoutException('Elasticsearch Timed Out')); } - resolve(res); }); } catch (error) { diff --git a/src/infrastructure/modules/app.module.ts b/src/infrastructure/modules/app.module.ts index 308d14f..9b20b70 100644 --- a/src/infrastructure/modules/app.module.ts +++ b/src/infrastructure/modules/app.module.ts @@ -1,5 +1,5 @@ import { CacheInterceptor, CacheModule, Module } from '@nestjs/common'; -import { APP_INTERCEPTOR } from '@nestjs/core'; +import { APP_INTERCEPTOR, APP_PIPE } from '@nestjs/core'; import { ConfigModule } from '@nestjs/config'; import { configuration } from '../config/env.objects'; import { validate } from '../config/env.validation'; @@ -8,6 +8,7 @@ import * as modules from '../../core/modules' import { CommonModule } from './common/common.module'; import { PrometheusModule } from '@willsoto/nestjs-prometheus'; import { SearchModule } from './search.module'; +import { QueryStringPipe } from 'src/core/pipes/query-str.pipe'; /** * application modules list diff --git a/src/main.ts b/src/main.ts index c3efc9c..58e4e19 100644 --- a/src/main.ts +++ b/src/main.ts @@ -4,6 +4,7 @@ import { NestFactory } from '@nestjs/core'; import { AppModule } from './infrastructure/modules/app.module'; import { SwaggerModule, DocumentBuilder, SwaggerDocumentOptions } from '@nestjs/swagger'; import { ConfigService } from '@nestjs/config'; +import { QueryStringPipe } from './core/pipes/query-str.pipe'; /** * Main entry point of the application * @returns Nothing @@ -16,7 +17,7 @@ async function bootstrap() { app.useGlobalPipes( new ValidationPipe({ disableErrorMessages: false, - }) + }), ); /** diff --git a/src/test/page.interceptor.spec.ts b/src/test/page.interceptor.spec.ts index d60c8f1..27837a4 100644 --- a/src/test/page.interceptor.spec.ts +++ b/src/test/page.interceptor.spec.ts @@ -151,11 +151,7 @@ describe('Unit tests for PageInterceptor', () => { res.subscribe((page) => { expect(page.meta).toEqual({ total: 921, - pagenum: 5, order: 'desc', - hasNext: true, - hasPrev: true, - pagesize: 100 }); }); }); diff --git a/src/test/search.service.spec.ts b/src/test/search.service.spec.ts index 864f6c3..d773f4f 100644 --- a/src/test/search.service.spec.ts +++ b/src/test/search.service.spec.ts @@ -42,23 +42,25 @@ describe('Unit tests for SearchService', () => { expect(httpGetSpy).toHaveBeenCalled(); }); - it('Should send correct data via HttpService.get() body parameter', () => { - let httpGetSpy = jest.spyOn(httpService, 'get'); + // it('Should send correct data via HttpService.get() body parameter', () => { + // let httpGetSpy = jest.spyOn(httpService, 'get'); - const uuid = 'thisIsUUID_Provided'; - searchService.findByID(uuid); - expect(httpGetSpy).toHaveBeenCalledWith<[string, object]>(expect.anything(), { - data: { - size: 1, - query: { - query_string: { - query: 'id:' + uuid - } - } - }, - headers: { 'Content-Type': 'application/json' } - }); - }); + // const uuid = 'thisIsUUID_Provided'; + // searchService.findByID(uuid); + // expect(httpGetSpy).toHaveBeenCalledWith<[string, object]>(expect.anything(), { + // data: { + // size: 1, + // query: { + // query_string: { + // query: 'id:' + uuid + // } + // }, + // search_after: undefined, + // sort: undefined, + // }, + // headers: { 'Content-Type': 'application/json' } + // }); + // }); it('Should call HttpService.get() with correct URI and port number', () => { let httpGetSpy = jest.spyOn(httpService, 'get'); -- 2.39.5 From 83b5cc6e5e865415d2e3f2d1401ff61e2ba281e5 Mon Sep 17 00:00:00 2001 From: danny-mhlv Date: Fri, 7 Oct 2022 13:18:50 +0300 Subject: [PATCH 08/13] Fixed validation issues. Cleared unnecessary DTOs --- .../controller/papers.controller.ts | 15 +++--- src/core/domain/dtos/index.ts | 4 +- src/core/domain/dtos/request.dto.ts | 48 ----------------- src/core/domain/dtos/search-q.dto.ts | 33 +++++++----- src/core/domain/dtos/search-result.dto.ts | 53 ------------------- src/core/interceptors/page.interceptor.ts | 28 +++------- src/core/pipes/query-str.pipe.ts | 13 ----- src/core/services/common/search.service.ts | 24 ++++++--- src/infrastructure/modules/app.module.ts | 1 - src/main.ts | 2 +- src/test/search.service.spec.ts | 2 +- 11 files changed, 53 insertions(+), 170 deletions(-) delete mode 100644 src/core/domain/dtos/request.dto.ts delete mode 100644 src/core/domain/dtos/search-result.dto.ts delete mode 100644 src/core/pipes/query-str.pipe.ts diff --git a/src/application/controller/papers.controller.ts b/src/application/controller/papers.controller.ts index 50b30e7..bdf507a 100644 --- a/src/application/controller/papers.controller.ts +++ b/src/application/controller/papers.controller.ts @@ -1,11 +1,9 @@ -import { Body, Controller, Get, HttpCode, Param, ParseUUIDPipe, Query, Req, UseFilters, UseInterceptors, UsePipes } from "@nestjs/common"; +import { Controller, Get, HttpCode, Param, ParseUUIDPipe, Query, Req, UseFilters, UseInterceptors } from "@nestjs/common"; import { SearchService } from "../../core/services/common/search.service"; import { PageInterceptor } from "../../core/interceptors/page.interceptor"; import { ApiExtraModels, ApiGatewayTimeoutResponse, ApiOperation, ApiResponse, ApiTags } from "@nestjs/swagger"; -import { RequestDto } from "../../core/domain/dtos/request.dto"; -import { EsHitDto, EsResponseDto, PageDto, PaperDto } from "../../core/domain"; +import { EsHitDto, EsResponseDto, PageDto, PaperDto, SearchQueryDto } from "../../core/domain"; import { HttpExceptionFilter } from "src/core/filters/http-exception.filter"; -import { QueryStringPipe } from "src/core/pipes/query-str.pipe"; /** * /papers/ route controller @@ -15,8 +13,7 @@ import { QueryStringPipe } from "src/core/pipes/query-str.pipe"; version: '1', path: 'papers', }) -@ApiExtraModels(RequestDto, EsHitDto, EsResponseDto) -// @UseInterceptors(CacheInterceptor) +@ApiExtraModels(EsHitDto, EsResponseDto) export class PapersController { constructor(private searchService: SearchService) {} @@ -39,10 +36,10 @@ export class PapersController { description: 'Elasticsearch request timed out' }) @Get('search') - @UseInterceptors(PageInterceptor) @HttpCode(200) - getByContext(@Req() request: RequestDto): Promise { - return this.searchService.findByContext(request.es_query).then( + @UseInterceptors(PageInterceptor) + getByContext(@Query() request: SearchQueryDto): Promise { + return this.searchService.findByContext(request).then( (response) => { return response; }, diff --git a/src/core/domain/dtos/index.ts b/src/core/domain/dtos/index.ts index d24be14..ac35a7b 100644 --- a/src/core/domain/dtos/index.ts +++ b/src/core/domain/dtos/index.ts @@ -3,6 +3,4 @@ export * from './elastic/es-response.dto'; export * from './elastic/es-hit.dto'; export * from './page.dto'; export * from './search-q.dto'; -export * from './search-result.dto'; -export * from './paper.dto'; -export * from './request.dto'; \ No newline at end of file +export * from './paper.dto'; \ No newline at end of file diff --git a/src/core/domain/dtos/request.dto.ts b/src/core/domain/dtos/request.dto.ts deleted file mode 100644 index 973264d..0000000 --- a/src/core/domain/dtos/request.dto.ts +++ /dev/null @@ -1,48 +0,0 @@ -import { ApiExtraModels, ApiProperty, ApiPropertyOptional } from "@nestjs/swagger"; -import { IsDefined, IsNotEmpty, IsOptional } from "class-validator"; -import { EsQueryDto } from "./elastic/es-query.dto"; -import { SearchQueryDto } from "./search-q.dto"; - -/** - * List of allowed properties in this DTO - */ -const allowedProperties = ['query', 'es_query']; - -/** - * Request object, which contains query parameters and Elasticsearch query object - */ -@ApiExtraModels() -export class RequestDto { - /** - * Query parameters object - */ - @IsDefined() - @IsNotEmpty() - @ApiProperty({ - type: SearchQueryDto, - description: 'Actual query with parameters acquired from the request', - example: {} - }) - query: SearchQueryDto; - - /** - * Elasticsearch query object - */ - @IsOptional() - @ApiPropertyOptional({ - type: EsQueryDto, - description: 'Elasticsearch query body constructed by pagination mechanism', - example: {}, - }) - es_query?: EsQueryDto; - - /** - * Constructs an object with provided parameters - * @param query - * @param es_query - */ - constructor(query: SearchQueryDto, es_query: EsQueryDto) { - this.query = query; - this.es_query = es_query; - } -} \ No newline at end of file diff --git a/src/core/domain/dtos/search-q.dto.ts b/src/core/domain/dtos/search-q.dto.ts index ee4d929..0651a03 100644 --- a/src/core/domain/dtos/search-q.dto.ts +++ b/src/core/domain/dtos/search-q.dto.ts @@ -1,10 +1,11 @@ import { ApiExtraModels, ApiPropertyOptional } from "@nestjs/swagger"; -import { IsDefined, IsInt, IsNotEmpty, IsOptional, IsString } from "class-validator"; +import { Type } from "class-transformer"; +import { IsDefined, IsInt, IsNotEmpty, IsOptional, IsString, Min } from "class-validator"; /** * List of allowed properties in this DTO */ -const allowedProperties = ['query', 'pagen', 'limit', 'order']; +const allowedProperties = ['query', 'limit', 'offset', 'order']; /** * Elasticsearch response DTO @@ -28,9 +29,12 @@ export class SearchQueryDto { */ @IsOptional() @IsInt() + @Type(() => Number) + @Min(1) @ApiPropertyOptional({ description: 'Limits the number of displayed elements', example: 10, + required: false }) limit?: number; @@ -39,26 +43,26 @@ export class SearchQueryDto { */ @IsOptional() @IsInt() + @Type(() => Number) + @Min(0) @ApiPropertyOptional({ description: 'Offset from the start of the list of hits', example: 0, + required: false, }) offset?: number; /** * Indicates in which order elements need to be displayed. */ - @IsOptional() - @IsString() - @ApiPropertyOptional({ - description: 'Indicates in which order elements need to be displayed', - example: 'asc', - }) - order?: string; - - /** - * - */ + @IsOptional() + @IsString() + @ApiPropertyOptional({ + description: 'Indicates in which order elements need to be displayed', + example: 'asc', + required: false, + }) + order?: string; /** * Constructs an object with provided parameters @@ -67,9 +71,10 @@ export class SearchQueryDto { * @param limit * @param order */ - constructor(query: string, page: number, limit: number, order: string) { + constructor(query: string = undefined, limit: number = 10, offset: number = 0, order: string = undefined) { this.query = query; this.limit = limit; + this.offset = offset; this.order = order; } } \ No newline at end of file diff --git a/src/core/domain/dtos/search-result.dto.ts b/src/core/domain/dtos/search-result.dto.ts deleted file mode 100644 index c926308..0000000 --- a/src/core/domain/dtos/search-result.dto.ts +++ /dev/null @@ -1,53 +0,0 @@ -import { ApiExtraModels, ApiProperty } from "@nestjs/swagger"; -import { IsArray, IsDefined, IsInt, IsNotEmpty } from "class-validator"; -import { EsResponseDto } from "./elastic/es-response.dto"; - -/** - * List of allowed properties in this DTO - */ -const allowedProperties = ['data', 'status']; - -/** - * Elasticsearch response DTO - */ -@ApiExtraModels() -export class SearchResultDto { - /** - * Status code - */ - @IsDefined() - @IsNotEmpty() - @IsInt() - @ApiProperty({ - description: 'Status code', - example: 200, - }) - statusCode: number; - - /** - * All the data acquired. - */ - @IsDefined() - @IsNotEmpty() - @IsArray() - @ApiProperty({ - description: 'Data acquired from the Elasticsearch', - example: { - took: 1, - timed_out: false, - _shards: {}, - hits: {} - }, - }) - data: EsResponseDto; - - /** - * Constructs an object with provided parameters - * @param code - * @param data - */ - constructor(code: number, data: EsResponseDto) { - this.statusCode = code; - this.data = data; - } -} \ No newline at end of file diff --git a/src/core/interceptors/page.interceptor.ts b/src/core/interceptors/page.interceptor.ts index 73c2089..8a78ded 100644 --- a/src/core/interceptors/page.interceptor.ts +++ b/src/core/interceptors/page.interceptor.ts @@ -2,9 +2,6 @@ import { HttpService } from "@nestjs/axios"; import { CACHE_MANAGER, CallHandler, ExecutionContext, Inject, Injectable, NestInterceptor } from "@nestjs/common"; import { Observable, map, take, switchMap, of } from "rxjs"; import { PageDto } from "../domain/dtos"; -import { EsQueryDto } from "../domain/dtos/elastic/es-query.dto"; -import { RequestDto } from "../domain/dtos/request.dto"; -import { SearchQueryDto } from "../domain/dtos/search-q.dto"; import { EsTime } from "../domain/enums/es-time.enum"; import { Order, toOrder } from "../domain/enums/page-order.enum"; import { EsPit } from "../domain/interfaces/elastic/es-pit.interface"; @@ -43,29 +40,20 @@ export class PageInterceptor implements NestInterceptor { * @returns Page with content and metadata */ async intercept(context: ExecutionContext, next: CallHandler): Promise> { - const request: RequestDto = context.switchToHttp().getRequest(); - const query: SearchQueryDto = request.query; + const query = context.switchToHttp().getRequest().query; - const offset = !query.offset ? 0 : query.offset; - const limit = !query.limit ? 10 : query.limit; - const order = !query.order ? Order.DESC : query.order; + // const offset = !query.offset ? 0 : query.offset; + const offset = query.offset; + // const limit = !query.limit ? 10 : query.limit; + const limit = query.limit; + // const order = !query.order ? Order.DESC : query.order; + const order = query.order; - const prev_page = await this.cacheManager.get('prev_page'); + const prev_page = await this.cacheManager.get('prev_page'); if (prev_page) { if (offset == prev_page[1] && limit == prev_page[2] && order == prev_page[3]) return of(prev_page[0]); } - // Contruct a body for querying Elasticsearch - request.es_query = new EsQueryDto(); - request.es_query.query = { - query_string: { - query: query.query, - default_field: 'content', - } - }; - request.es_query.from = offset; - request.es_query.size = limit; - return next.handle().pipe( switchMap(async (res) => { // Setting the page meta-data diff --git a/src/core/pipes/query-str.pipe.ts b/src/core/pipes/query-str.pipe.ts deleted file mode 100644 index d9bfcca..0000000 --- a/src/core/pipes/query-str.pipe.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { ArgumentMetadata, BadRequestException, ImATeapotException, Injectable, PipeTransform } from "@nestjs/common"; -import { RequestDto } from "../domain"; - -@Injectable() -export class QueryStringPipe implements PipeTransform { - constructor() {} - - transform(value: RequestDto, metadata: ArgumentMetadata): RequestDto { - console.log(value.query.limit) - - return value; - } -} \ No newline at end of file diff --git a/src/core/services/common/search.service.ts b/src/core/services/common/search.service.ts index 4cd131b..b72685e 100644 --- a/src/core/services/common/search.service.ts +++ b/src/core/services/common/search.service.ts @@ -1,7 +1,7 @@ import { HttpService } from "@nestjs/axios"; import { GatewayTimeoutException, Injectable, NotFoundException } from "@nestjs/common"; import { map, take } from "rxjs"; -import { EsResponseDto} from "../../domain/dtos"; +import { EsResponseDto, SearchQueryDto} from "../../domain/dtos"; import { EsQueryDto } from "../../domain/dtos/elastic/es-query.dto"; /** @@ -32,10 +32,9 @@ export class SearchService { * @returns Elasticsearch hits or an error object */ async findByID(uuid: string): Promise { // Should I change 'object' to specific DTO? - let ESQ: EsQueryDto = new EsQueryDto; - - ESQ.size = 1; - ESQ.query = { + const es_query: EsQueryDto = new EsQueryDto(); + es_query.size = 1; + es_query.query = { query_string: { query: ('id:' + uuid), } @@ -44,7 +43,7 @@ export class SearchService { return new Promise((resolve, reject) => { try { (this.httpService.get(`http://${this.ES_IP}:${this.ES_PORT}/_search`, { - data: ESQ, + data: es_query, headers: {'Content-Type': 'application/json'}, })) ?.pipe(take(1), map(axiosRes => axiosRes.data)) @@ -68,7 +67,18 @@ export class SearchService { * @param query, * @returns Elasticsearch hits or an error object */ - async findByContext(es_query: EsQueryDto): Promise { + async findByContext(query: SearchQueryDto): Promise { + // Contruct a body for querying Elasticsearch + const es_query: EsQueryDto = new EsQueryDto(); + es_query.query = { + query_string: { + query: query.query, + default_field: 'content', + } + }; + es_query.from = query.offset; + es_query.size = query.limit; + return new Promise((resolve, reject) => { try { (this.httpService.get(`http://${this.ES_IP}:${this.ES_PORT}/_search`, { diff --git a/src/infrastructure/modules/app.module.ts b/src/infrastructure/modules/app.module.ts index 9b20b70..1eefb80 100644 --- a/src/infrastructure/modules/app.module.ts +++ b/src/infrastructure/modules/app.module.ts @@ -8,7 +8,6 @@ import * as modules from '../../core/modules' import { CommonModule } from './common/common.module'; import { PrometheusModule } from '@willsoto/nestjs-prometheus'; import { SearchModule } from './search.module'; -import { QueryStringPipe } from 'src/core/pipes/query-str.pipe'; /** * application modules list diff --git a/src/main.ts b/src/main.ts index 58e4e19..f413052 100644 --- a/src/main.ts +++ b/src/main.ts @@ -4,7 +4,7 @@ import { NestFactory } from '@nestjs/core'; import { AppModule } from './infrastructure/modules/app.module'; import { SwaggerModule, DocumentBuilder, SwaggerDocumentOptions } from '@nestjs/swagger'; import { ConfigService } from '@nestjs/config'; -import { QueryStringPipe } from './core/pipes/query-str.pipe'; + /** * Main entry point of the application * @returns Nothing diff --git a/src/test/search.service.spec.ts b/src/test/search.service.spec.ts index d773f4f..689c43c 100644 --- a/src/test/search.service.spec.ts +++ b/src/test/search.service.spec.ts @@ -152,7 +152,7 @@ describe('Unit tests for SearchService', () => { } } - searchService.findByContext(es_query); + // searchService.findByContext(es_query); expect(httpGetSpy).toHaveBeenCalledWith<[string, object]>(expect.anything(), { data: es_query, headers: { 'Content-Type': 'application/json' } -- 2.39.5 From 4c23bce69fc6331e977229ad9ca3c9a8f1aadc8a Mon Sep 17 00:00:00 2001 From: danny-mhlv Date: Fri, 7 Oct 2022 15:10:18 +0300 Subject: [PATCH 09/13] Fixed problems with reading 'undefined's --- .../controller/papers.controller.ts | 2 +- src/core/domain/enums/page-order.enum.ts | 2 + src/core/interceptors/page.interceptor.ts | 4 +- src/core/services/common/search.service.ts | 14 +-- src/test/search.service.spec.ts | 93 ++++--------------- 5 files changed, 30 insertions(+), 85 deletions(-) diff --git a/src/application/controller/papers.controller.ts b/src/application/controller/papers.controller.ts index bdf507a..60d0f3a 100644 --- a/src/application/controller/papers.controller.ts +++ b/src/application/controller/papers.controller.ts @@ -1,4 +1,4 @@ -import { Controller, Get, HttpCode, Param, ParseUUIDPipe, Query, Req, UseFilters, UseInterceptors } from "@nestjs/common"; +import { Controller, Get, HttpCode, Param, ParseUUIDPipe, Query, UseFilters, UseInterceptors } from "@nestjs/common"; import { SearchService } from "../../core/services/common/search.service"; import { PageInterceptor } from "../../core/interceptors/page.interceptor"; import { ApiExtraModels, ApiGatewayTimeoutResponse, ApiOperation, ApiResponse, ApiTags } from "@nestjs/swagger"; diff --git a/src/core/domain/enums/page-order.enum.ts b/src/core/domain/enums/page-order.enum.ts index 61e5176..9e724bd 100644 --- a/src/core/domain/enums/page-order.enum.ts +++ b/src/core/domain/enums/page-order.enum.ts @@ -19,6 +19,8 @@ export enum Order { * @returns Appropriate enum-member */ export function toOrder(str: string): Order { + if (!str) return Order.DESC; + switch (str) { case 'asc': return Order.ASC; case 'desc': return Order.DESC; diff --git a/src/core/interceptors/page.interceptor.ts b/src/core/interceptors/page.interceptor.ts index 8a78ded..0551733 100644 --- a/src/core/interceptors/page.interceptor.ts +++ b/src/core/interceptors/page.interceptor.ts @@ -1,5 +1,5 @@ import { HttpService } from "@nestjs/axios"; -import { CACHE_MANAGER, CallHandler, ExecutionContext, Inject, Injectable, NestInterceptor } from "@nestjs/common"; +import { BadRequestException, CACHE_MANAGER, CallHandler, ExecutionContext, Inject, Injectable, InternalServerErrorException, NestInterceptor } from "@nestjs/common"; import { Observable, map, take, switchMap, of } from "rxjs"; import { PageDto } from "../domain/dtos"; import { EsTime } from "../domain/enums/es-time.enum"; @@ -63,7 +63,7 @@ export class PageInterceptor implements NestInterceptor { }; // Check if the performed search is a backwards search - let data = res.hits.hits; + let data = res?.hits?.hits; // Omitting the redundant info and leaving only the document data = data.map((el) => el._source); // Change the order if set diff --git a/src/core/services/common/search.service.ts b/src/core/services/common/search.service.ts index b72685e..b037ab5 100644 --- a/src/core/services/common/search.service.ts +++ b/src/core/services/common/search.service.ts @@ -1,5 +1,5 @@ import { HttpService } from "@nestjs/axios"; -import { GatewayTimeoutException, Injectable, NotFoundException } from "@nestjs/common"; +import { GatewayTimeoutException, ImATeapotException, Injectable, NotFoundException } from "@nestjs/common"; import { map, take } from "rxjs"; import { EsResponseDto, SearchQueryDto} from "../../domain/dtos"; import { EsQueryDto } from "../../domain/dtos/elastic/es-query.dto"; @@ -28,7 +28,7 @@ export class SearchService { /** * Finds a paper by its own ID - * @param uuid + * @param uuid String, that represents unique identifier of a paper * @returns Elasticsearch hits or an error object */ async findByID(uuid: string): Promise { // Should I change 'object' to specific DTO? @@ -48,12 +48,12 @@ export class SearchService { })) ?.pipe(take(1), map(axiosRes => axiosRes.data)) .subscribe((res: EsResponseDto) => { - if (!res.hits.hits.length) { - reject(new NotFoundException); - } if (res.timed_out) { reject(new GatewayTimeoutException('Elasticsearch Timed Out')); } + if (!res?.hits?.hits?.length) { + reject(new NotFoundException); + } resolve(res); }); } catch (error) { @@ -65,7 +65,7 @@ export class SearchService { /** * Finds relevant documents by context using the given query string * @param query, - * @returns Elasticsearch hits or an error object + * @returns Elasticsearch response */ async findByContext(query: SearchQueryDto): Promise { // Contruct a body for querying Elasticsearch @@ -86,7 +86,7 @@ export class SearchService { headers: {'Content-Type': 'application/json'}, })) ?.pipe(take(1), map(axiosRes => axiosRes.data)) - .subscribe((res: EsResponseDto) => { + ?.subscribe((res: EsResponseDto) => { if (res.timed_out) { reject(new GatewayTimeoutException('Elasticsearch Timed Out')); } diff --git a/src/test/search.service.spec.ts b/src/test/search.service.spec.ts index 689c43c..2fa1197 100644 --- a/src/test/search.service.spec.ts +++ b/src/test/search.service.spec.ts @@ -3,7 +3,7 @@ import { GatewayTimeoutException, HttpException } from "@nestjs/common"; import { ConfigModule } from "@nestjs/config"; import { Test } from "@nestjs/testing"; import { of } from "rxjs"; -import { EsQueryDto, EsResponseDto } from "src/core/domain"; +import { SearchQueryDto } from "src/core/domain"; import { SearchService } from "src/core/services/common/search.service"; describe('Unit tests for SearchService', () => { @@ -42,26 +42,6 @@ describe('Unit tests for SearchService', () => { expect(httpGetSpy).toHaveBeenCalled(); }); - // it('Should send correct data via HttpService.get() body parameter', () => { - // let httpGetSpy = jest.spyOn(httpService, 'get'); - - // const uuid = 'thisIsUUID_Provided'; - // searchService.findByID(uuid); - // expect(httpGetSpy).toHaveBeenCalledWith<[string, object]>(expect.anything(), { - // data: { - // size: 1, - // query: { - // query_string: { - // query: 'id:' + uuid - // } - // }, - // search_after: undefined, - // sort: undefined, - // }, - // headers: { 'Content-Type': 'application/json' } - // }); - // }); - it('Should call HttpService.get() with correct URI and port number', () => { let httpGetSpy = jest.spyOn(httpService, 'get'); @@ -76,28 +56,6 @@ describe('Unit tests for SearchService', () => { expect(searchService.findByID('')).toBeInstanceOf(Promise); }); - // it('Should return a Promise with EsResponseDto', () => { - // // Axios response mock - // httpService.get = jest.fn().mockReturnValueOnce( - // of({ - // status: undefined, - // statusText: undefined, - // headers: undefined, - // config: undefined, - // data: { - // took: 1, - // timed_out: false, - // hits: { - // total: {}, - // hits: [{}] - // } - // }, - // }) - // ); - - // expect(searchService.findByID('')).resolves.toBeInstanceOf(EsResponseDto) - // }); - // Errors it('Should throw 504 | GatewayTimeoutException', () => { // Axios response mock @@ -136,25 +94,27 @@ describe('Unit tests for SearchService', () => { it('Should touch HttpService.get() method', () => { let httpGetSpy = jest.spyOn(httpService, 'get'); - searchService.findByContext(null); + searchService.findByContext({query: ""}); expect(httpGetSpy).toHaveBeenCalled(); }); it('Should send correct data via HttpService.get() body parameter', () => { let httpGetSpy = jest.spyOn(httpService, 'get'); - let es_query = new EsQueryDto(); - es_query = { - query: { - query_string: { - query: 'thisIsTheQuery!' - } - } - } + const query = new SearchQueryDto('keyword', 1, 32, 'desc'); - // searchService.findByContext(es_query); + searchService.findByContext(query); expect(httpGetSpy).toHaveBeenCalledWith<[string, object]>(expect.anything(), { - data: es_query, + data: { + query: { + query_string: { + query: 'keyword', + default_field: 'content', + } + }, + from: 32, + size: 1, + }, headers: { 'Content-Type': 'application/json' } }); }); @@ -162,7 +122,7 @@ describe('Unit tests for SearchService', () => { it('Should call HttpService.get() with correct URI and port number', () => { let httpGetSpy = jest.spyOn(httpService, 'get'); - searchService.findByContext(null); + searchService.findByContext({query: ""}); expect(httpGetSpy).toHaveBeenCalledWith<[string, object]>( `http://${process.env.ES_CONTAINER_NAME}:${process.env.ES_PORT}/_search`, expect.anything() @@ -170,26 +130,9 @@ describe('Unit tests for SearchService', () => { }); it('Should return a Promise', () => { - expect(searchService.findByContext(null)).toBeInstanceOf(Promise); + expect(searchService.findByContext({query: ""})).toBeInstanceOf(Promise); }); - // it('Should return a Promise with EsResponseDto', () => { - // // Axios response mock - // httpService.get = jest.fn().mockReturnValueOnce( - // of({ - // status: undefined, - // statusText: undefined, - // headers: undefined, - // config: undefined, - // data: { - // dummy: 'dum' - // } - // }) - // ); - - // expect(searchService.findByContext(null)).resolves.toMatchObject(null); - // }); - // Errors it('Should throw 504 | GatewayTimeoutException', () => { // Axios response mock @@ -206,7 +149,7 @@ describe('Unit tests for SearchService', () => { }) ); - searchService.findByContext(null).catch((err) => { + searchService.findByContext({query: ""}).catch((err) => { expect(err).toBeInstanceOf(GatewayTimeoutException); }); }); @@ -216,7 +159,7 @@ describe('Unit tests for SearchService', () => { throw new HttpException({ oops: 'sorry' }, 999); }); - searchService.findByContext(null).catch((err) => { + searchService.findByContext({query: ""}).catch((err) => { expect(err).toBeInstanceOf(HttpException); expect(err.response).toEqual({ oops: 'sorry' }); expect(err.status).toEqual(999); -- 2.39.5 From 3707100826307fd89d00307f5d57b14431e6fa34 Mon Sep 17 00:00:00 2001 From: danny-mhlv Date: Fri, 7 Oct 2022 15:28:32 +0300 Subject: [PATCH 10/13] Fixed relative path to HttpExceptionFilter, that failed the e2e test --- src/application/controller/papers.controller.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/application/controller/papers.controller.ts b/src/application/controller/papers.controller.ts index 60d0f3a..04c6531 100644 --- a/src/application/controller/papers.controller.ts +++ b/src/application/controller/papers.controller.ts @@ -3,7 +3,7 @@ import { SearchService } from "../../core/services/common/search.service"; import { PageInterceptor } from "../../core/interceptors/page.interceptor"; import { ApiExtraModels, ApiGatewayTimeoutResponse, ApiOperation, ApiResponse, ApiTags } from "@nestjs/swagger"; import { EsHitDto, EsResponseDto, PageDto, PaperDto, SearchQueryDto } from "../../core/domain"; -import { HttpExceptionFilter } from "src/core/filters/http-exception.filter"; +import { HttpExceptionFilter } from "../../core/filters/http-exception.filter"; /** * /papers/ route controller -- 2.39.5 From a5b175b2922fcdac659ab3de65c34bd78603d866 Mon Sep 17 00:00:00 2001 From: danny-mhlv Date: Wed, 12 Oct 2022 13:47:52 +0300 Subject: [PATCH 11/13] Fixed insufficient caching problem --- src/core/interceptors/page.interceptor.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/core/interceptors/page.interceptor.ts b/src/core/interceptors/page.interceptor.ts index 0551733..3a409b1 100644 --- a/src/core/interceptors/page.interceptor.ts +++ b/src/core/interceptors/page.interceptor.ts @@ -42,16 +42,17 @@ export class PageInterceptor implements NestInterceptor { async intercept(context: ExecutionContext, next: CallHandler): Promise> { const query = context.switchToHttp().getRequest().query; - // const offset = !query.offset ? 0 : query.offset; const offset = query.offset; - // const limit = !query.limit ? 10 : query.limit; const limit = query.limit; - // const order = !query.order ? Order.DESC : query.order; const order = query.order; + const query_string = query.query; const prev_page = await this.cacheManager.get('prev_page'); if (prev_page) { - if (offset == prev_page[1] && limit == prev_page[2] && order == prev_page[3]) return of(prev_page[0]); + if (offset == prev_page[1] && + limit == prev_page[2] && + order == prev_page[3] && + query_string === prev_page[4]) return of(prev_page[0]); } return next.handle().pipe( @@ -71,7 +72,7 @@ export class PageInterceptor implements NestInterceptor { // Cache and return the page const page: PageDto = new PageDto(data, meta); - await this.cacheManager.set('prev_page', [page, offset, limit, order]); + await this.cacheManager.set('prev_page', [page, offset, limit, order, query_string]); return page; }) ); -- 2.39.5 From f5ad2e0ff79fbf003e2363cfcd8630a1f9693a3c Mon Sep 17 00:00:00 2001 From: Danny Mikhaylov Date: Tue, 8 Nov 2022 05:18:17 +0300 Subject: [PATCH 12/13] Changed 'match' query to 'multi-match'. Allows to search for both 'content' and 'title' of the paper --- .../domain/dtos/elastic/es-multimatch.dto.ts | 68 +++++++++++++++++++ src/core/services/common/search.service.ts | 11 ++- 2 files changed, 72 insertions(+), 7 deletions(-) create mode 100644 src/core/domain/dtos/elastic/es-multimatch.dto.ts diff --git a/src/core/domain/dtos/elastic/es-multimatch.dto.ts b/src/core/domain/dtos/elastic/es-multimatch.dto.ts new file mode 100644 index 0000000..6617fea --- /dev/null +++ b/src/core/domain/dtos/elastic/es-multimatch.dto.ts @@ -0,0 +1,68 @@ +import { ApiExtraModels, ApiProperty, ApiPropertyOptional } from "@nestjs/swagger"; +import { IsArray, IsDefined, IsInt, IsObject, IsOptional } from "class-validator"; +import { EsPit } from "../../interfaces/elastic/es-pit.interface"; +import { EsQuery } from "../../interfaces/elastic/es-query.interface" + +/** + * List of allowed properties in this DTO + */ + const allowedProperties = ['query']; + + /** + * Elasticsearch multi-match query DTO + */ + @ApiExtraModels() + export class EsMultimatchQueryDto { + /** + * Offset from the start of the list of hits + */ + @IsOptional() + @IsInt() + @ApiPropertyOptional({ + description: 'Offset from the start of the list of hits', + example: 5, + }) + public from?: number; + + /** + * Maximum number of elements returned by Elasticsearch + */ + @IsOptional() + @IsInt() + @ApiPropertyOptional({ + description: 'Maximum number of elements returned by Elasticsearch', + example: 30 + }) + public size?: number; + + /** + * The search query object passed to Elasticsearch + */ + @IsDefined() + @IsObject() + @ApiProperty({ + description: 'Search query object passed to Elasticsearch', + example: { + multi_match: { + query: 'Maths', + fields: [ + 'title', + 'content' + ] + } + }, + }) + private readonly query: Object; + + /** + * Constructs a multi-match + */ + constructor(query: string = '', fields: Array = ['content']) { + this.query = { + multi_match: { + query: query, + fields: fields + } + } + } + } \ No newline at end of file diff --git a/src/core/services/common/search.service.ts b/src/core/services/common/search.service.ts index b037ab5..7e5a47c 100644 --- a/src/core/services/common/search.service.ts +++ b/src/core/services/common/search.service.ts @@ -1,6 +1,7 @@ import { HttpService } from "@nestjs/axios"; import { GatewayTimeoutException, ImATeapotException, Injectable, NotFoundException } from "@nestjs/common"; import { map, take } from "rxjs"; +import { EsMultimatchQueryDto } from "src/core/domain/dtos/elastic/es-multimatch.dto"; import { EsResponseDto, SearchQueryDto} from "../../domain/dtos"; import { EsQueryDto } from "../../domain/dtos/elastic/es-query.dto"; @@ -69,13 +70,9 @@ export class SearchService { */ async findByContext(query: SearchQueryDto): Promise { // Contruct a body for querying Elasticsearch - const es_query: EsQueryDto = new EsQueryDto(); - es_query.query = { - query_string: { - query: query.query, - default_field: 'content', - } - }; + const es_query: EsMultimatchQueryDto = new EsMultimatchQueryDto(query.query, [ + 'title', 'content' + ]); es_query.from = query.offset; es_query.size = query.limit; -- 2.39.5 From cbf7938ed738cb441822751736dced3a18850995 Mon Sep 17 00:00:00 2001 From: Danny Date: Wed, 9 Nov 2022 13:27:50 +0300 Subject: [PATCH 13/13] Fixed the test issue --- src/core/domain/dtos/elastic/es-multimatch.dto.ts | 4 +--- src/test/search.service.spec.ts | 13 ++++++++----- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/core/domain/dtos/elastic/es-multimatch.dto.ts b/src/core/domain/dtos/elastic/es-multimatch.dto.ts index 6617fea..81295d0 100644 --- a/src/core/domain/dtos/elastic/es-multimatch.dto.ts +++ b/src/core/domain/dtos/elastic/es-multimatch.dto.ts @@ -1,7 +1,5 @@ import { ApiExtraModels, ApiProperty, ApiPropertyOptional } from "@nestjs/swagger"; -import { IsArray, IsDefined, IsInt, IsObject, IsOptional } from "class-validator"; -import { EsPit } from "../../interfaces/elastic/es-pit.interface"; -import { EsQuery } from "../../interfaces/elastic/es-query.interface" +import { IsDefined, IsInt, IsObject, IsOptional } from "class-validator"; /** * List of allowed properties in this DTO diff --git a/src/test/search.service.spec.ts b/src/test/search.service.spec.ts index 2fa1197..76bc66a 100644 --- a/src/test/search.service.spec.ts +++ b/src/test/search.service.spec.ts @@ -107,13 +107,16 @@ describe('Unit tests for SearchService', () => { expect(httpGetSpy).toHaveBeenCalledWith<[string, object]>(expect.anything(), { data: { query: { - query_string: { - query: 'keyword', - default_field: 'content', + multi_match: { + query: query.query, + fields: [ + 'title', + 'content' + ] } }, - from: 32, - size: 1, + from: query.offset, + size: query.limit }, headers: { 'Content-Type': 'application/json' } }); -- 2.39.5
      + + + toOrder + + +
      +toOrder(str: string) +
      +

      Converts string value to appropriate enum-member

      +
      + +
      + Parameters : + + + + + + + + + + + + + + + + + + + + + +
      NameTypeOptionalDescription
      str + string + + No + +

      String to be converted

      + +
      +
      +
      +
      +
      + Returns : Order + +
      +
      +

      Appropriate enum-member

      +