fix: accept v3 fields in filters when using v3 jobs

src/common/utils.ts

@@ -1040,6 +1040,9 @@ export const datasetsFullQueryDescriptionFields =
 export const jobsFullQueryExampleFields =
   '{"ownerGroup": "group1", "statusCode": "jobCreated"}';
 
+export const jobsFullQueryExampleFieldsV3 =
+  '{"emailJobInitiator": "[email protected]", "jobStatusMessage": "jobCreated"}';
+
 export const jobsFullQueryDescriptionFields =
   '<pre>\n  \
 {\n \
@@ -1059,6 +1062,20 @@ export const jobsFullQueryDescriptionFields =
 }\n \
   </pre>';
 
+export const jobsFullQueryDescriptionFieldsV3 =
+  '<pre>\n  \
+{\n \
+  "creationTime": { <optional>\n \
+    "begin": string,\n \
+    "end": string,\n \
+  },\n \
+  "type": string, <optional>\n \
+  "id": string, <optional>\n \
+  "jobStatusMessage": string, <optional>\n \
+  ... <optional>\n \
+}\n \
+  </pre>';
+
 export const proposalsFullQueryExampleFields =
   '{"text": "some text", "proposalId": "proposal_id"}';
 

src/jobs/jobs.controller.ts

@@ -41,8 +41,8 @@ import { FullFacetResponse } from "src/common/types";
 import {
   fullQueryDescriptionLimits,
   fullQueryExampleLimits,
-  jobsFullQueryExampleFields,
-  jobsFullQueryDescriptionFields,
+  jobsFullQueryDescriptionFieldsV3,
+  jobsFullQueryExampleFieldsV3,
 } from "src/common/utils";
 import { CreateJobV3MappingInterceptor } from "./interceptors/create-job-v3-mapping.interceptor";
 import { UpdateJobV3MappingInterceptor } from "./interceptors/update-job-v3-mapping.interceptor";
@@ -53,6 +53,12 @@ import { IncludeValidationPipe } from "./pipes/include-validation.pipe";
 import { DatasetLookupKeysEnum } from "src/datasets/types/dataset-lookup";
 import { PartialOutputDatasetDto } from "src/datasets/dto/output-dataset.dto";
 import { ALLOWED_JOB_KEYS, ALLOWED_JOB_FILTER_KEYS } from "./types/job-lookup";
+import {
+  V3ConditionToV4Pipe,
+  V3FieldsToV4Pipe,
+  V3FilterToV4Pipe,
+  V3LimitsToV4Pipe,
+} from "./pipes/v3-filter.pipe";
 
 @ApiBearerAuth()
 @ApiTags("jobs")
@@ -155,10 +161,10 @@ export class JobsController {
     name: "fields",
     description:
       "Filters to apply when retrieving jobs.\n" +
-      jobsFullQueryDescriptionFields,
+      jobsFullQueryDescriptionFieldsV3,
     required: false,
     type: String,
-    example: jobsFullQueryExampleFields,
+    example: jobsFullQueryExampleFieldsV3,
   })
   @ApiQuery({
     name: "limits",
@@ -177,12 +183,13 @@ export class JobsController {
   @SerializeOptions({ type: OutputJobV3Dto, excludeExtraneousValues: true })
   async fullQuery(
     @Req() request: Request,
-    @Query() filters: { fields?: string; limits?: string },
+    @Query("fields", new V3ConditionToV4Pipe()) fields?: string,
+    @Query("limits", new V3LimitsToV4Pipe()) limits?: string,
   ): Promise<OutputJobV3Dto[] | null> {
-    const jobs = (await this.jobsControllerUtils.fullQueryJobs(
-      request,
-      filters,
-    )) as JobClass[] | null;
+    const jobs = (await this.jobsControllerUtils.fullQueryJobs(request, {
+      fields,
+      limits,
+    })) as JobClass[] | null;
     return jobs as OutputJobV3Dto[] | null;
   }
 
@@ -203,18 +210,18 @@ export class JobsController {
     name: "fields",
     description:
       "Define the filter conditions by specifying the values of fields requested.\n" +
-      jobsFullQueryDescriptionFields,
+      jobsFullQueryDescriptionFieldsV3,
     required: false,
     type: String,
-    example: jobsFullQueryExampleFields,
+    example: jobsFullQueryExampleFieldsV3,
   })
   @ApiQuery({
     name: "facets",
     description:
       "Define a list of field names, for which facet counts should be calculated.",
     required: false,
     type: String,
-    example: '["type","ownerGroup","statusCode"]',
+    example: '["type","ownerGroup","jobStatusMessage"]',
   })
   @ApiResponse({
     status: HttpStatus.OK,
@@ -224,9 +231,10 @@ export class JobsController {
   })
   async fullFacet(
     @Req() request: Request,
-    @Query() filters: { fields?: string; facets?: string },
+    @Query("facets", new V3FieldsToV4Pipe()) facets?: string,
+    @Query("fields", new V3ConditionToV4Pipe()) fields?: string,
   ): Promise<Record<string, unknown>[]> {
-    return this.jobsControllerUtils.fullFacetJobs(request, filters);
+    return this.jobsControllerUtils.fullFacetJobs(request, { facets, fields });
   }
 
   /**
@@ -440,6 +448,7 @@ export class JobsController {
 
     @Query(
       "filter",
+      new V3FilterToV4Pipe(),
       new FilterValidationPipe(ALLOWED_JOB_KEYS, ALLOWED_JOB_FILTER_KEYS, {
         where: false,
         include: true,

src/jobs/pipes/v3-filter.pipe.ts

@@ -0,0 +1,197 @@
+import { PipeTransform, Injectable, ArgumentMetadata } from "@nestjs/common";
+import { jobV3toV4FieldMap } from "../types/jobs-filter-content";
+import _ from "lodash";
+
+type KeyMap = Record<string, string>;
+
+type Func = (value: unknown) => unknown;
+
+type FuncMap = Record<string, Func>;
+
+interface TransformDeepOptions {
+  keyMap?: KeyMap;
+  funcMap?: FuncMap;
+  arrayFn?: Func;
+  valueFn?: Func;
+}
+
+const transformDeep = (
+  obj: unknown,
+  opts: TransformDeepOptions = {},
+): unknown => {
+  const { keyMap = {}, funcMap = {}, arrayFn, valueFn } = opts;
+
+  if (Array.isArray(obj)) {
+    return obj.map((item) =>
+      arrayFn ? arrayFn(transformDeep(item, opts)) : transformDeep(item, opts),
+    );
+  }
+
+  if (obj && typeof obj === "object") {
+    const newObj: Record<string, unknown> = {};
+    for (const [key, value] of Object.entries(obj)) {
+      const mappedKey = keyMap[key] ?? key;
+      let transformed: unknown;
+      if (funcMap[key]) {
+        transformed = funcMap[key](value);
+      } else {
+        transformed = transformDeep(value, opts);
+      }
+      newObj[mappedKey] = valueFn ? valueFn(transformed) : transformed;
+    }
+    return newObj;
+  }
+
+  return obj;
+};
+
+class ParseJsonPipe implements PipeTransform<string, string> {
+  transform(value: string): string {
+    if (!value || typeof value !== "string") return value;
+    try {
+      return JSON.parse(value);
+    } catch {
+      return value;
+    }
+  }
+}
+
+class ParseDeepJsonPipe implements PipeTransform<string, string | object> {
+  private jsonPipe = new ParseJsonPipe();
+
+  transform(value: string): string | object {
+    const parsed = this.jsonPipe.transform(value);
+    return transformDeep(parsed, {
+      valueFn: (value) => this.jsonPipe.transform(value as string),
+    }) as object;
+  }
+}
+
+class ReplaceObjKeysPipe implements PipeTransform<unknown, unknown> {
+  constructor(private keyMap: KeyMap) {}
+
+  transform(value: unknown): unknown {
+    return transformDeep(value, { keyMap: this.keyMap });
+  }
+}
+
+class TransformObjValuesPipe implements PipeTransform<unknown, unknown> {
+  constructor(private funcMap: FuncMap) {}
+
+  transform(value: unknown): unknown {
+    return transformDeep(value, { funcMap: this.funcMap });
+  }
+}
+
+class TransformArrayValuesPipe implements PipeTransform<unknown, unknown> {
+  constructor(private arrayFn: (item: unknown) => unknown) {}
+
+  transform(value: unknown): unknown {
+    return transformDeep(value, { arrayFn: this.arrayFn });
+  }
+}
+
+class ComposePipe<T = unknown> implements PipeTransform<T, T> {
+  private readonly pipes: PipeTransform[];
+  private readonly jsonToString = new JsonToStringPipe();
+  private readonly parseDeepJson = new ParseDeepJsonPipe();
+
+  constructor(
+    pipes: PipeTransform[],
+    private readonly jsonTransform = true,
+  ) {
+    this.pipes = [...pipes];
+    if (this.jsonTransform) {
+      this.pipes.unshift(this.parseDeepJson);
+      this.pipes.push(this.jsonToString);
+    }
+  }
+
+  transform(value: T, metadata: ArgumentMetadata = {} as ArgumentMetadata): T {
+    return this.pipes.reduce(
+      (val, pipe) => pipe.transform(val, metadata),
+      value,
+    );
+  }
+}
+
+class JsonToStringPipe implements PipeTransform<object, string | object> {
+  transform(value: object): string | object {
+    try {
+      return JSON.stringify(value);
+    } catch {
+      return value;
+    }
+  }
+}
+
+@Injectable()
+export class V3ConditionToV4Pipe extends ComposePipe<object> {
+  // it replaces object keys following the keyMappings object
+  // for example, it replaces keys from the v3 DTO (user-facing)
+  // to database fields later used in the aggregation pipeline
+  // for example from {where: {user-facing-1: 'abc'} to {where: {db-field1: 'abc'}
+
+  constructor(keyMappings = jobV3toV4FieldMap, jsonTransform = true) {
+    super([new ReplaceObjKeysPipe(keyMappings)], jsonTransform);
+  }
+}
+
+@Injectable()
+export class V3LimitsToV4Pipe extends ComposePipe<object> {
+  // it replaces list elements following the <keyMappings object>:asc|desc
+  // for example, it replaces {order: ['user-facing1:asc', 'user-facing2:asc']}
+  // with {order: ['db-field1:asc', 'db-field2:asc']}
+
+  constructor(keyMappings = jobV3toV4FieldMap, jsonTransform = true) {
+    const sortToOrderPipe = new TransformObjValuesPipe({
+      order: (value: unknown) => {
+        const isArray = _.isArray(value);
+        const order = (isArray ? value : [value]).reduce((acc, orderValue) => {
+          const [field, direction] = (orderValue as string).split(":");
+          return acc.concat(`${keyMappings[field]}:${direction ?? "asc"}`);
+        }, [] as string[]);
+        return isArray ? order : order[0];
+      },
+    });
+    super([sortToOrderPipe], jsonTransform);
+  }
+}
+
+@Injectable()
+export class V3FieldsToV4Pipe extends ComposePipe<object> {
+  // it replaces list elements following the keyMappings object
+  // for example, it replaces the fields: [user-facing1, user-facing2]
+  // with [db-field1, db-field2]
+
+  constructor(keyMappings = jobV3toV4FieldMap, jsonTransform = true) {
+    super(
+      [
+        new TransformArrayValuesPipe((item) => {
+          if (_.isString(item) && keyMappings[item]) return keyMappings[item];
+          return item;
+        }),
+      ],
+      jsonTransform,
+    );
+  }
+}
+
+@Injectable()
+export class V3FilterToV4Pipe extends ComposePipe<string> {
+  // it combines the 3 pipes together
+  // for example
+  // from {where: {user-facing1: 'abc'}, limits: {order: ['user-facing1:asc']}, fields: ['user-facing1']}
+  // to {where: {db-field1: 'abc'}, limits: {order: ['db-field1:asc']}, fields: ['db-field1']}
+
+  constructor(keyMappings = jobV3toV4FieldMap, jsonTransform = true) {
+    super(
+      [
+        new V3LimitsToV4Pipe(keyMappings, false),
+        new V3ConditionToV4Pipe(keyMappings, false),
+        new V3FieldsToV4Pipe(keyMappings, false),
+      ],
+      jsonTransform,
+    );
+  }
+}

src/jobs/types/jobs-filter-content.ts

@@ -23,7 +23,7 @@ const FILTERS: Record<"limits" | "fields" | "where" | "include", object> = {
     items: {
       type: "string",
     },
-    example: ["ownerUser", "datasets.keywords"],
+    example: ["type", "datasets.keywords"],
   },
   limits: {
     type: "object",
@@ -39,7 +39,7 @@ const FILTERS: Record<"limits" | "fields" | "where" | "include", object> = {
       sort: {
         type: "object",
         properties: {
-          ownerUser: {
+          type: {
             type: "string",
             example: "asc | desc",
           },