Pagination redone

src/core/domain/dtos/elastic/es-query.dto.ts

@@ -1,5 +1,5 @@
 import { ApiExtraModels, ApiProperty, ApiPropertyOptional } from "@nestjs/swagger";
-import { IsArray, IsDefined, IsInt, IsNotEmpty, IsNumber, IsObject, IsOptional } from "class-validator";
+import { IsArray, IsDefined, IsInt, IsObject, IsOptional } from "class-validator";
 import { EsPit } from "../../interfaces/elastic/es-pit.interface";
 import { EsQuery } from "../../interfaces/elastic/es-query.interface"
 
@@ -13,71 +13,80 @@ import { EsQuery } from "../../interfaces/elastic/es-query.interface"
   */
  @ApiExtraModels()
  export class EsQueryDto {
-     /**
+   /**
-      * Maximum number of elements returned by Elasticsearch
+    * Offset from the start of the list of hits
-      */
+    */
-     @IsOptional()
+   @IsOptional()
-     @IsDefined()
+   @IsInt()
-     @IsNumber()
+   @ApiPropertyOptional({
-     @IsInt()
+      description: 'Offset from the start of the list of hits',
-     @ApiPropertyOptional({
+      example: 5,
-         description: 'Maximum number of elements returned by Elasticsearch',
+   })
-         example: 30
+   from?: number;
-     })
-     size?: number;
 
-     /**
+   /**
-      * The search query object passed to Elasticsearch
+    * Maximum number of elements returned by Elasticsearch
-      */
+    */
-     @IsDefined()
+    @IsOptional()
-     @IsObject()
+    @IsInt()
-     @ApiProperty({
+    @ApiPropertyOptional({
-         description: 'Search query object passed to Elasticsearch',
+        description: 'Maximum number of elements returned by Elasticsearch',
-         example: {},
+        example: 30
-     })
+    })
-     query: EsQuery;
+    size?: number;
     
-     /**
+    /**
-      * Object, that stores PIT ID and time alive
+     * The search query object passed to Elasticsearch
-      */
+     */
-     @IsOptional()
+    @IsDefined()
-     @IsObject()
+    @IsObject()
-     @ApiPropertyOptional({
+    @ApiProperty({
-        description: 'PIT object',
+        description: 'Search query object passed to Elasticsearch',
-        example: {}
+        example: {},
-     })
+    })
-     pit?: EsPit;
+    query: EsQuery;
 
-     /**
+    /**
-      * Sorting info
+     * Object, that stores PIT ID and time alive
-      */
+     */
-     @IsOptional()
+    @IsOptional()
-     @IsArray()
+    @IsObject()
-     @ApiPropertyOptional({
+    @ApiPropertyOptional({
-        description: '',
+       description: 'PIT object',
-        example: []
+       example: {}
-     })
+    })
-     sort?: unknown[];
+    pit?: EsPit;
 
-     /**
+    /**
-      * Pagination info
+     * Sorting info
-      */
+     */
-     @IsOptional()
+    @IsOptional()
-     @IsArray()
+    @IsArray()
-     @ApiPropertyOptional({
+    @ApiPropertyOptional({
-        description: '',
+       description: '',
-        example: []
+       example: []
-     })
+    })
-     search_after?: unknown[];
+    sort?: unknown[];
 
-     /**
+    /**
-      * Constructs an empty object
+     * Pagination info
-      */
+     */
-     constructor() {
+    @IsOptional()
-        this.size = 10;
+    @IsArray()
-        this.query = undefined;
+    @ApiPropertyOptional({
-        this.pit = undefined;
+       description: '',
-        this.sort = undefined;
+       example: []
-        this.search_after = undefined;
+    })
-     }
+    search_after?: unknown[];
+
+    /**
+     * Constructs an empty object
+     */
+    constructor() {
+       this.size = 10;
+       this.query = undefined;
+       this.pit = undefined;
+       this.sort = undefined;
+       this.search_after = undefined;
+    }
  }

src/core/domain/dtos/page-meta.dto.ts

@@ -1,8 +1,6 @@
-import { ApiExtraModels, ApiProperty, PartialType } from "@nestjs/swagger";
+import { ApiExtraModels, ApiProperty } from "@nestjs/swagger";
 import { IsArray } from "class-validator";
 import { Order } from "../enums";
-import { PageMeta } from "../interfaces/page-meta.interface";
-import { PaperDto } from "./paper.dto";
 
 /**
  * List of allowed properties in this DTO
@@ -13,7 +11,7 @@ const allowedProperties = ['total', 'pagenum', 'order', 'hasNext', 'hasPrev', 'p
  * Page model for pagination
  */
 @ApiExtraModels()
-export class PageMetaDto implements PageMeta {
+export class PageMetaDto {
     /**
      * Total number of hits (results) acquired from the search
      */
@@ -24,16 +22,6 @@ export class PageMetaDto implements PageMeta {
     })
     total: number;
 
-    /**
-     * Current page number
-     */
-    @ApiProperty({
-        description: 'Current page number',
-        minimum: 1,
-        example: 3
-    })
-    pagenum: number;
-
     /**
      * Order of the elements on the page
      */
@@ -42,32 +30,4 @@ export class PageMetaDto implements PageMeta {
         example: Order.DESC
     })
     order: Order;
-
-    /**
-     * Flag, that shows if there's a page following the current one
-     */
-    @ApiProperty({
-        description: 'Flag, that shows if there\'s a page following the current one',
-        example: true
-    })
-    hasNext: boolean;
-
-    /**
-     * Flag, that shows if there's a page preceding the current one
-     */
-    @ApiProperty({
-        description: 'Flag, that shows if there\'s a page preceding the current one',
-        example: true
-    })
-    hasPrev: boolean;
-
-    /**
-     * Maximum number of elements on the page
-     */
-    @ApiProperty({
-        description: 'Maximum number of elements on the page',
-        minimum: 1,
-        example: 20
-    })
-    pagesize: number;
 }

src/core/domain/dtos/page.dto.ts

@@ -1,7 +1,5 @@
 import { ApiExtraModels, ApiProperty, PartialType } from "@nestjs/swagger";
 import { IsArray } from "class-validator";
-import { Order } from "../enums";
-import { PageMeta } from "../interfaces/page-meta.interface";
 import { PageMetaDto } from "./page-meta.dto";
 import { PaperDto } from "./paper.dto";
 
@@ -22,7 +20,7 @@ export class PageDto {
     @ApiProperty({
         description: 'All data (papers) the page contains',
         isArray: true,
-        type: PaperDto
+        type: PaperDto,
     })
     readonly data: PaperDto[];
 
@@ -31,7 +29,7 @@ export class PageDto {
      */
     @ApiProperty({
         description: 'Metadata for the page',
-        // example: {},
+        type: PageMetaDto,
         
     })
     readonly meta: PageMetaDto;
@@ -41,7 +39,7 @@ export class PageDto {
      * @param data 
      * @param meta 
      */
-    constructor(data: PaperDto[], meta: PageMeta) {
+    constructor(data: PaperDto[], meta: PageMetaDto) {
         this.data = data;
         this.meta = meta;
     }

src/core/domain/dtos/search-q.dto.ts

@@ -1,4 +1,4 @@
-import { ApiExtraModels, ApiProperty } from "@nestjs/swagger";
+import { ApiExtraModels, ApiPropertyOptional } from "@nestjs/swagger";
 import { IsDefined, IsInt, IsNotEmpty, IsOptional, IsString } from "class-validator";
 
 /**
@@ -12,63 +12,64 @@ const allowedProperties = ['query', 'pagen', 'limit', 'order'];
 @ApiExtraModels()
 export class SearchQueryDto {
     /**
-     * Given query string to perform the
+     * Given query string to perform the search on.
-     * search on.
      */
     @IsDefined()
     @IsNotEmpty()
     @IsString()
-    @ApiProperty({
+    @ApiPropertyOptional({
-        description: 'query',
+        description: 'Given query string to perform the search on',
-        example: 'Particle Accelerator'
+        example: 'Particle Accelerator',
     })
     query: string;
 
-    /**
-     * Page number to display.
-     */
-    @IsDefined()
-    @IsNotEmpty()
-    @IsInt()
-    @ApiProperty({
-        description: 'page',
-        example: 3,
-    })
-    page: number;
-
     /**
      * Limits the number of displayed elements.
      */
     @IsOptional()
     @IsInt()
-    @ApiProperty({
+    @ApiPropertyOptional({
-        description: 'limit',
+        description: 'Limits the number of displayed elements',
         example: 10,
     })
-    limit: number;
+    limit?: number;
 
     /**
-     * Limits the number of displayed elements.
+     * Offset from the start of the list of hits.
+     */
+    @IsOptional()
+    @IsInt()
+    @ApiPropertyOptional({
+        description: 'Offset from the start of the list of hits',
+        example: 0,
+    })
+    offset?: number;
+
+    /**
+     * Indicates in which order elements need to be displayed.
      */
      @IsOptional()
      @IsString()
-     @ApiProperty({
+     @ApiPropertyOptional({
-         description: 'order',
+         description: 'Indicates in which order elements need to be displayed',
          example: 'asc',
      })
-     order: string;
+     order?: string;
 
-     /**
+    /**
-      * Constructs an object with provided parameters
+     * 
-      * @param query 
+     */
-      * @param page 
+
-      * @param limit 
+    /**
-      * @param order 
+     * Constructs an object with provided parameters
-      */
+     * @param query 
-     constructor(query: string, page: number, limit: number, order: string) {
+     * @param page 
+     * @param limit 
+     * @param order 
+     */
+    constructor(query: string, page: number, limit: number, order: string) {
         this.query = query;
-        this.page = page;
         this.limit = limit;
         this.order = order;
-     }
+    }
 }

src/core/domain/enums/page-order.enum.ts

@@ -12,3 +12,10 @@ export enum Order {
      */
     DESC = 'desc',
 }
+
+export function toOrder(str: string): Order {
+    switch (str) {
+        case 'asc': return Order.ASC;
+        case 'desc': return Order.DESC;
+    } 
+}

src/core/domain/interfaces/index.ts

@@ -1,5 +1,4 @@
 export * from './http-response.interface'
-export * from './page-meta.interface'
 export * from './search-info.interface'
 export * from './elastic/es-query.interface'
 export * from './elastic/es-query-string.interface'

src/core/domain/interfaces/page-meta.interface.ts

@@ -1,36 +0,0 @@
-import { Order } from "../enums/page-order.enum";
-
-/**
- * Structure of page metadata
- */
-export interface PageMeta {
-    /**
-     * Total search results
-     */
-    total: number;
-
-    /**
-     * Number of the page
-     */
-    pagenum: number;
-
-    /**
-     * Order of the elements on the page
-     */
-    order: Order;
-
-    /**
-     * Flag that indicates presence of the next page
-     */
-    hasNext: boolean;
-
-    /**
-     * Flag that indicates presence of the previous page
-     */ 
-    hasPrev: boolean;
-
-    /**
-     *  Number of elements on the page
-     */
-    pagesize: number;
-}

src/core/interceptors/page.interceptor.ts

@@ -6,10 +6,10 @@ import { EsQueryDto } from "../domain/dtos/elastic/es-query.dto";
 import { RequestDto } from "../domain/dtos/request.dto";
 import { SearchQueryDto } from "../domain/dtos/search-q.dto";
 import { EsTime } from "../domain/enums/es-time.enum";
-import { Order } from "../domain/enums/page-order.enum";
+import { Order, toOrder } from "../domain/enums/page-order.enum";
-import { PageMeta } from "../domain/interfaces";
 import { EsPit } from "../domain/interfaces/elastic/es-pit.interface";
 import { Cache } from 'cache-manager'
+import { PageMetaDto } from "../domain/dtos/page-meta.dto";
 
 /**
  * Pagination-implementing interceptor
@@ -45,8 +45,12 @@ export class PageInterceptor implements NestInterceptor {
     async intercept(context: ExecutionContext, next: CallHandler<any>): Promise<Observable<PageDto>> {
         const request: RequestDto = context.switchToHttp().getRequest<RequestDto>();
         const query: SearchQueryDto = request.query;
-        let reverse: boolean = false;
 
+        const offset = !query.offset ? 0 : query.offset;
+        const limit = !query.limit ? 10 : query.limit; 
+        const order = !query.order ? Order.DESC : query.order;
+
+        // Contruct a body for querying Elasticsearch
         request.es_query = new EsQueryDto();
         request.es_query.query = {
             query_string: {
@@ -54,63 +58,33 @@ export class PageInterceptor implements NestInterceptor {
                 default_field: 'content',
             }
         };
+        request.es_query.from = offset;
+        request.es_query.size = limit;
         request.es_query.sort = [
-            { _score: { order: !query?.order ? Order.DESC : query.order } },
+            { "_score": { "order": order } },
-            { _shard_doc: 'desc' }
         ];        
 
-        const limit = !query?.limit ? 10 : query.limit;
+        const prev_page = await this.cacheManager.get('prev_page'); 
-
+        if (prev_page) {
-        if (await this.cacheManager.get('prevPage')) {
+            if (offset == prev_page[1] && limit == prev_page[2]) return prev_page[0];
-            if (query.page == (await this.cacheManager.get('_pagenum'))) return await this.cacheManager.get('prevPage');
-
-            request.es_query.pit = await this.cacheManager.get('_pit');
-            request.es_query.search_after = await this.cacheManager.get('_sa');
-            request.es_query.size = limit * Math.abs(query.page - (await this.cacheManager.get('_pagenum')));
-
-            if (query.page < (await this.cacheManager.get('_pagenum'))) {
-                request.es_query.sort = [{ _score: { order: Order.ASC } }];
-                request.es_query.size += limit - 1;
-                reverse = true;
-            }
-        } else {
-            request.es_query.pit = await this.getPIT(1);
-            request.es_query.size = limit * query.page;
         }
 
         return next.handle().pipe(
             switchMap(async (res) => {
                 // Setting the page meta-data
-                let meta: PageMeta = {
+                let meta: PageMetaDto = {
                     total: res.hits.total.value,
-                    pagenum: !query?.page ? 1 : +query.page,
+                    order: toOrder(order),
-                    order: query?.order?.toUpperCase() === Order.ASC ? Order.ASC : Order.DESC,
-                    pagesize: !query?.limit ? 10 : query.limit,
-                    hasNext: undefined,
-                    hasPrev: undefined,
                 };
-                meta.hasNext = meta.pagenum * meta.pagesize < meta.total ? true : false;
-                meta.hasPrev = meta.pagenum != 1 ? true : false;
-
-                // Saving the search info
-                await this.cacheManager.set('_pit', { id: res.pit_id, keep_alive: `1${EsTime.min}` })
-                await this.cacheManager.set('_sa', res.hits.hits[res.hits.hits.length - 1]?.sort);
-                await this.cacheManager.set('_pagenum', query.page);
 
                 // Check if the performed search is a backwards search
-                let data = res.hits.hits.slice(-meta.pagesize);
+                let data = res.hits.hits;
-                if (reverse) {
-                    this.cacheManager.set('_sa', data[0]?.sort);
-                    data.reverse();
-                    reverse = false;
-                }
-
                 // Omitting the redundant info and leaving only the document
                 data = data.map((el) => el._source);
 
                 // Cache and return the page
                 const page: PageDto = new PageDto(data, meta);
-                await this.cacheManager.set('prevPage', page);
+                await this.cacheManager.set('prev_page', [page, offset, limit]);
                 return page;
             })
         );
@@ -121,19 +95,19 @@ export class PageInterceptor implements NestInterceptor {
      * @param alive, amount of time in minutes (defaults to 1). If time unit is not specified - defaults to minutes.
      * @returns PIT object <EsPit> containing PIT ID and keep_alive value
      */
-     public async getPIT(alive: number, unit: EsTime = EsTime.min): Promise<EsPit> {
+    public async getPIT(alive: number, unit: EsTime = EsTime.min): Promise<EsPit> {
-        return new Promise((resolve, reject) => {
+       return new Promise((resolve, reject) => {
-            try {
+           try {
-                this.httpService.post<EsPit>(`http://${this.ES_IP}:${this.ES_PORT}/papers/_pit?keep_alive=${alive+unit}`)
+               this.httpService.post<EsPit>(`http://${this.ES_IP}:${this.ES_PORT}/papers/_pit?keep_alive=${alive+unit}`)
-                    .pipe(take(1), map(axiosRes => axiosRes.data))
+                   .pipe(take(1), map(axiosRes => axiosRes.data))
-                    .subscribe((res: EsPit) => {
+                   .subscribe((res: EsPit) => {
-                        res.keep_alive = alive + unit;
+                       res.keep_alive = alive + unit;
-                        resolve(res);
+                       resolve(res);
-                    });
+                   });
-            } catch (error) {
+           } catch (error) {
-                reject(error);
+               reject(error);
-            }
+           }
-        });
+       });
     }
 
     /**
@@ -141,20 +115,20 @@ export class PageInterceptor implements NestInterceptor {
      * @param pitID, ID of the PIT, that would be deleted
      * @returns true/false, depending on the result of deletion of the PIT
      */
-     async deletePIT(pitID: string): Promise<boolean> {
+    async deletePIT(pitID: string): Promise<boolean> {
-        return new Promise((resolve, reject) => {
+       return new Promise((resolve, reject) => {
-            try {
+           try {
-                this.httpService.delete(`http://${this.ES_IP}:${this.ES_PORT}/_pit`, {
+               this.httpService.delete(`http://${this.ES_IP}:${this.ES_PORT}/_pit`, {
-                    data: { id: pitID },
+                   data: { id: pitID },
-                    headers: { 'Content-Type': 'application/json' },
+                   headers: { 'Content-Type': 'application/json' },
-                })
+               })
-                .pipe(take(1), map(axiosRes => axiosRes.data))
+               .pipe(take(1), map(axiosRes => axiosRes.data))
-                .subscribe((res) => {
+               .subscribe((res) => {
-                    resolve(res.succeeded);
+                   resolve(res.succeeded);
-                });
+               });
-            } catch (error) {
+           } catch (error) {
-                reject(error);
+               reject(error);
-            }
+           }
-        })
+       })
     }
 }

src/test/e2e/papers.controller.e2e.spec.ts

@@ -205,20 +205,8 @@ describe('E2E Testing of /papers', () => {
         expect(test.body.meta.total).toBeDefined();
         expect(test.body.meta.total).toBe(2);
 
-        expect(test.body.meta.pagenum).toBeDefined();
-        expect(test.body.meta.pagenum).toBe(1);
-
         expect(test.body.meta.order).toBeDefined();
         expect(test.body.meta.order).toBe(Order.DESC);
-        
-        expect(test.body.meta.pagesize).toBeDefined();
-        expect(test.body.meta.pagesize).toBe(10);
-
-        expect(test.body.meta.hasNext).toBeDefined();
-        expect(test.body.meta.hasNext).toBe(false);
-
-        expect(test.body.meta.hasPrev).toBeDefined();
-        expect(test.body.meta.hasPrev).toBe(false);
     });
 
     afterAll(async () => {

src/test/page.interceptor.spec.ts

@@ -160,39 +160,6 @@ describe('Unit tests for PageInterceptor', () => {
                 });
             });
         });     
-
-        it('Should reverse the search results', async () => {
-            execCtxMock.getRequest.mockReturnValueOnce({
-                query: {
-                    page: 1,
-                    order: 'desc',
-                    limit: 3
-                }
-            });
-
-            await pageInter['cacheManager'].set('_pagenum', 3);
-            await pageInter['cacheManager'].set('prevPage', { set: "yes" });
-
-            callHandlerMock.handle.mockReturnValueOnce(
-                of({
-                    hits: {
-                        total: { value: 1 },
-                        hits: [
-                            { sort: ['1', 'less relevant'], _source: '1' },
-                            { sort: ['2', 'average'], _source: '2' },
-                            { sort: ['3', 'most relevant'], _source: '3' }
-                        ]
-                    }
-                })
-            );
-
-            pageInter.intercept(execCtxMock, callHandlerMock).then((res) => {
-                res.subscribe(async (page) => {
-                    expect(await pageInter['cacheManager'].get('_sa')).toEqual(['1', 'less relevant']);
-                    expect(page.data).toEqual(['3', '2', '1']);
-                });
-            });
-        });        
     });
 
     describe('getPIT()', () => {