POC: add complete endpoint in instruments that returns paginated data with totalCount

src/app.module.ts

@@ -144,6 +144,7 @@ import { MetricsModule } from "./metrics/metrics.module";
     MongooseModule.forRootAsync({
       useFactory: async (configService: ConfigService) => ({
         uri: configService.get<string>("mongodbUri"),
+        sanitizeFilter: true,
       }),
       inject: [ConfigService],
     }),

src/common/decorators/api-data-count-response.decorator.ts

@@ -0,0 +1,35 @@
+import { applyDecorators, Type } from "@nestjs/common";
+import { HttpStatus } from "@nestjs/common/enums";
+import { ApiExtraModels, ApiOkResponse, getSchemaPath } from "@nestjs/swagger";
+import { DataCountOutputDto } from "../types";
+
+export const ApiDataCountResponse = <DataDto extends Type<unknown>>(
+  dataDto: DataDto,
+  description = "List of items with total count. The list is limited by the limit and offset parameters.",
+  statusCode = HttpStatus.OK,
+) =>
+  applyDecorators(
+    ApiExtraModels(DataCountOutputDto, dataDto),
+    ApiOkResponse({
+      schema: {
+        allOf: [
+          { $ref: getSchemaPath(DataCountOutputDto) },
+          {
+            properties: {
+              data: {
+                type: "array",
+                items: { $ref: getSchemaPath(dataDto) },
+              },
+              totalCount: {
+                type: "number",
+                default: 0,
+              },
+            },
+          },
+        ],
+        required: ["data", "totalCount"],
+      },
+      status: statusCode,
+      description: description,
+    }),
+  );

src/common/interfaces/common.interface.ts

@@ -47,6 +47,19 @@ export interface IFilters<T, Y = null> {
   limits?: ILimitsFilter;
 }
 
+export interface IFiltersV4<T> {
+  where?: FilterQuery<T>;
+  include?: string[];
+  fields?: string[];
+  limits?: ILimitsFilterV4;
+}
+
+export interface ILimitsFilterV4 {
+  limit: number;
+  skip: number;
+  sort: Record<string, "asc" | "desc">;
+}
+
 export interface IFacets<T> {
   fields?: T;
   facets?: string[];

src/datasets/types/dataset-filter-content.ts → src/common/swagger-filter-content.ts

@@ -8,7 +8,7 @@ const FILTERS: Record<"limits" | "fields" | "where" | "include", object> = {
   where: {
     type: "object",
     example: {
-      datasetName: { $regex: "Dataset", $options: "i" },
+      field1: { $regex: "Test", $options: "i" },
     },
   },
   include: {
@@ -22,24 +22,24 @@ const FILTERS: Record<"limits" | "fields" | "where" | "include", object> = {
     type: "array",
     items: {
       type: "string",
-      example: "datasetName",
+      example: "field1",
     },
   },
   limits: {
     type: "object",
     properties: {
-      limit: {
-        type: "number",
-        example: 10,
-      },
       skip: {
         type: "number",
         example: 0,
       },
+      limit: {
+        type: "number",
+        example: 10,
+      },
       sort: {
         type: "object",
         properties: {
-          datasetName: {
+          field1: {
             type: "string",
             example: "asc | desc",
           },
@@ -54,7 +54,7 @@ const FILTERS: Record<"limits" | "fields" | "where" | "include", object> = {
  * But we want to have it when we run the application as it improves swagger documentation and usage a lot.
  * We use "content" property as it is described in the swagger specification: https://swagger.io/docs/specification/v3_0/describing-parameters/#schema-vs-content:~:text=explode%3A%20false-,content,-is%20used%20in
  */
-export const getSwaggerDatasetFilterContent = (
+export const getSwaggerFilterContent = (
   filtersToInclude: Record<keyof typeof FILTERS, boolean> = {
     where: true,
     include: true,

src/common/types.ts

@@ -51,3 +51,9 @@ export class IsValidResponse {
   @ApiProperty({ type: Boolean })
   isvalid: boolean;
 }
+
+export class DataCountOutputDto<T> {
+  data: T[];
+
+  totalCount: number;
+}

src/common/utils.ts

@@ -7,10 +7,12 @@ import {
   IAxiosError,
   IFilters,
   ILimitsFilter,
+  ILimitsFilterV4,
   IScientificFilter,
 } from "./interfaces/common.interface";
 import { ScientificRelation } from "./scientific-relation.enum";
 import { DatasetType } from "src/datasets/types/dataset-type.enum";
+import { isEmpty } from "lodash";
 
 // add Å to mathjs accepted units as equivalent to angstrom
 const isAlphaOriginal = Unit.isValidAlpha;
@@ -364,6 +366,46 @@ export const parsePipelineProjection = (fieldsProjection: string[]) => {
   return pipelineProjection;
 };
 
+export const buildDataCountFacetPipeline = (
+  limits: ILimitsFilterV4 | undefined,
+  fields: string[] | undefined,
+) => {
+  const skip = limits?.skip || 0;
+  const limit = limits?.limit || 10;
+
+  const facet: PipelineStage.Facet = {
+    $facet: {
+      data: [{ $skip: skip }, { $limit: limit }],
+      count: [{ $count: "totalCount" }],
+    },
+  };
+
+  if (limits?.sort && !isEmpty(limits?.sort)) {
+    const sortParsed = parsePipelineSort(limits.sort);
+
+    facet.$facet.data.push({ $sort: sortParsed });
+  }
+
+  if (fields) {
+    const project: PipelineStage.Project["$project"] =
+      parsePipelineProjection(fields);
+
+    facet.$facet.data.push({ $project: project });
+  }
+
+  return facet;
+};
+
+// NOTE: This returns default projection of the data and totalCount fields for findAll complete endpoints
+export const getDefaultDataCountProject = (): PipelineStage.Project => {
+  return {
+    $project: {
+      data: 1,
+      totalCount: { $arrayElemAt: ["$count.totalCount", 0] },
+    },
+  };
+};
+
 export const parseLimitFiltersForPipeline = (
   limits: ILimitsFilter | undefined,
 ): PipelineStage[] => {

src/datasets/datasets-public.v4.controller.ts

@@ -27,8 +27,8 @@ import {
 import { DatasetLookupKeysEnum } from "./types/dataset-lookup";
 import { IncludeValidationPipe } from "./pipes/include-validation.pipe";
 import { FilterValidationPipe } from "./pipes/filter-validation.pipe";
-import { getSwaggerDatasetFilterContent } from "./types/dataset-filter-content";
 import { AllowAny } from "src/auth/decorators/allow-any.decorator";
+import { getSwaggerFilterContent } from "src/common/swagger-filter-content";
 
 @ApiExtraModels(HistoryClass, TechniqueClass, RelationshipClass)
 @ApiTags("datasets public v4")
@@ -58,7 +58,7 @@ export class DatasetsPublicV4Controller {
       "Database filters to apply when retrieving the public datasets",
     required: false,
     type: String,
-    content: getSwaggerDatasetFilterContent(),
+    content: getSwaggerFilterContent(),
   })
   @ApiResponse({
     status: HttpStatus.OK,
@@ -171,12 +171,7 @@ export class DatasetsPublicV4Controller {
     description: "Database filters to apply when retrieving public dataset",
     required: true,
     type: String,
-    content: getSwaggerDatasetFilterContent({
-      where: true,
-      include: true,
-      fields: true,
-      limits: true,
-    }),
+    content: getSwaggerFilterContent(),
   })
   @ApiResponse({
     status: HttpStatus.OK,
@@ -208,7 +203,7 @@ export class DatasetsPublicV4Controller {
       "Database filters to apply when retrieving count for public datasets",
     required: false,
     type: String,
-    content: getSwaggerDatasetFilterContent({
+    content: getSwaggerFilterContent({
       where: true,
       include: false,
       fields: false,

src/datasets/datasets.v4.controller.ts

@@ -69,8 +69,8 @@ import { DatasetLookupKeysEnum } from "./types/dataset-lookup";
 import { IncludeValidationPipe } from "./pipes/include-validation.pipe";
 import { PidValidationPipe } from "./pipes/pid-validation.pipe";
 import { FilterValidationPipe } from "./pipes/filter-validation.pipe";
-import { getSwaggerDatasetFilterContent } from "./types/dataset-filter-content";
 import { plainToInstance } from "class-transformer";
+import { getSwaggerFilterContent } from "src/common/swagger-filter-content";
 
 @ApiBearerAuth()
 @ApiExtraModels(
@@ -355,7 +355,7 @@ export class DatasetsV4Controller {
     description: "Database filters to apply when retrieving datasets",
     required: false,
     type: String,
-    content: getSwaggerDatasetFilterContent(),
+    content: getSwaggerFilterContent(),
   })
   @ApiResponse({
     status: HttpStatus.OK,
@@ -523,12 +523,7 @@ export class DatasetsV4Controller {
     description: "Database filters to apply when retrieving dataset",
     required: true,
     type: String,
-    content: getSwaggerDatasetFilterContent({
-      where: true,
-      include: true,
-      fields: true,
-      limits: true,
-    }),
+    content: getSwaggerFilterContent(),
   })
   @ApiResponse({
     status: HttpStatus.OK,
@@ -566,7 +561,7 @@ export class DatasetsV4Controller {
     description: "Database filters to apply when retrieving count for datasets",
     required: false,
     type: String,
-    content: getSwaggerDatasetFilterContent({
+    content: getSwaggerFilterContent({
       where: true,
       include: false,
       fields: false,

src/instruments/instruments.controller.ts

@@ -29,13 +29,15 @@ import { AppAbility } from "src/casl/casl-ability.factory";
 import { Action } from "src/casl/action.enum";
 import { Instrument, InstrumentDocument } from "./schemas/instrument.schema";
 import { FormatPhysicalQuantitiesInterceptor } from "src/common/interceptors/format-physical-quantities.interceptor";
-import { IFilters } from "src/common/interfaces/common.interface";
+import { IFilters, IFiltersV4 } from "src/common/interfaces/common.interface";
 import {
   filterDescription,
   filterExample,
   replaceLikeOperator,
 } from "src/common/utils";
-import { CountApiResponse } from "src/common/types";
+import { CountApiResponse, DataCountOutputDto } from "src/common/types";
+import { getSwaggerFilterContent } from "src/common/swagger-filter-content";
+import { ApiDataCountResponse } from "src/common/decorators/api-data-count-response.decorator";
 
 @ApiBearerAuth()
 @ApiTags("instruments")
@@ -89,6 +91,34 @@ export class InstrumentsController {
     return this.instrumentsService.findAll(instrumentFilter);
   }
 
+  @UseGuards(PoliciesGuard)
+  @CheckPolicies("instruments", (ability: AppAbility) =>
+    ability.can(Action.InstrumentRead, Instrument),
+  )
+  @Get("/complete")
+  @ApiQuery({
+    name: "filters",
+    description: "Database filters to apply when retrieve all instruments",
+    required: false,
+    type: String,
+    content: getSwaggerFilterContent({
+      where: true,
+      include: false,
+      fields: true,
+      limits: true,
+    }),
+  })
+  @ApiDataCountResponse(Instrument, "Return the instruments requested")
+  async findAllComplete(
+    @Query("filters") filters?: string,
+  ): Promise<DataCountOutputDto<Instrument>> {
+    const parsedFilters: IFiltersV4<InstrumentDocument> = JSON.parse(
+      filters ?? "{}",
+    );
+
+    return this.instrumentsService.findAllComplete(parsedFilters);
+  }
+
   @UseGuards(PoliciesGuard)
   @CheckPolicies("instruments", (ability: AppAbility) =>
     ability.can(Action.InstrumentRead, Instrument),

src/instruments/instruments.service.ts

@@ -1,9 +1,13 @@
 import { Injectable } from "@nestjs/common";
 import { InjectModel } from "@nestjs/mongoose";
-import { FilterQuery, Model } from "mongoose";
-import { IFilters } from "src/common/interfaces/common.interface";
-import { CountApiResponse } from "src/common/types";
-import { parseLimitFilters } from "src/common/utils";
+import { FilterQuery, Model, PipelineStage } from "mongoose";
+import { IFilters, IFiltersV4 } from "src/common/interfaces/common.interface";
+import { CountApiResponse, DataCountOutputDto } from "src/common/types";
+import {
+  buildDataCountFacetPipeline,
+  getDefaultDataCountProject,
+  parseLimitFilters,
+} from "src/common/utils";
 import { CreateInstrumentDto } from "./dto/create-instrument.dto";
 import { PartialUpdateInstrumentDto } from "./dto/update-instrument.dto";
 import { Instrument, InstrumentDocument } from "./schemas/instrument.schema";
@@ -37,6 +41,23 @@ export class InstrumentsService {
     return instruments;
   }
 
+  async findAllComplete(
+    filter: IFiltersV4<InstrumentDocument>,
+  ): Promise<DataCountOutputDto<Instrument>> {
+    const $match: PipelineStage.Match = { $match: filter.where ?? {} };
+
+    const $facet = buildDataCountFacetPipeline(filter.limits, filter.fields);
+
+    const $project = getDefaultDataCountProject();
+
+    const pipeline: PipelineStage[] = [$match, $facet, $project];
+
+    const [result] = await this.instrumentModel
+      .aggregate<DataCountOutputDto<Instrument>>(pipeline)
+      .exec();
+
+    return { data: result.data, totalCount: result.totalCount || 0 };
+  }
   async count(filter: IFilters<InstrumentDocument>): Promise<CountApiResponse> {
     const whereFilter: FilterQuery<InstrumentDocument> = filter.where ?? {};