From 06844150eca8438643ca037f1f72b0b2b6e65a20 Mon Sep 17 00:00:00 2001 From: minottic Date: Tue, 2 Dec 2025 10:44:26 +0000 Subject: [PATCH 1/3] feat: include history in gets only when explicit --- src/datasets/datasets.controller.ts | 10 ++++++---- test/TestData.js | 1 - test/jest-e2e-tests/datasetHistory.e2e-spec.ts | 17 +++++++++++++++-- 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/src/datasets/datasets.controller.ts b/src/datasets/datasets.controller.ts index 8fc938889..7cdd17c88 100644 --- a/src/datasets/datasets.controller.ts +++ b/src/datasets/datasets.controller.ts @@ -594,10 +594,10 @@ export class DatasetsController { } } - const excludeHistory = - Array.isArray(fields) && !fields.includes("history"); + const includeHistory = + Array.isArray(fields) && fields.includes("history"); - if (!excludeHistory) + if (includeHistory) propertiesModifier.history = convertGenericHistoriesToObsoleteHistories( await this.historyService.find({ documentId: inputDataset._id, @@ -991,7 +991,9 @@ export class DatasetsController { if (datasets && datasets.length > 0) { outputDatasets = await Promise.all( - datasets.map((dataset) => this.convertCurrentToObsoleteSchema(dataset)), + datasets.map((dataset) => + this.convertCurrentToObsoleteSchema(dataset, parsedFilters.fields), + ), ); } diff --git a/test/TestData.js b/test/TestData.js index 8117b684f..36280b8bd 100644 --- a/test/TestData.js +++ b/test/TestData.js @@ -449,7 +449,6 @@ const TestData = { } }, sharedWith: [], - history: "JEST_ANY", size: 0, sourceFolder: "/iramjet/tif", sourceFolderHost: "nfs://file.your.site", diff --git a/test/jest-e2e-tests/datasetHistory.e2e-spec.ts b/test/jest-e2e-tests/datasetHistory.e2e-spec.ts index c2d586525..41901d413 100644 --- a/test/jest-e2e-tests/datasetHistory.e2e-spec.ts +++ b/test/jest-e2e-tests/datasetHistory.e2e-spec.ts @@ -55,9 +55,11 @@ describe("Test v3 history in datasetLifecycle", () => { await app.close(); }); - it("Should check v3 built history", async () => { + it("Should check v3 built history with field including history", async () => { + const filter = { fields: ["history"] }; await request(app.getHttpServer()) .get(`/api/v3/datasets/${encodeURIComponent(dsId)}`) + .query({ filter: JSON.stringify(filter) }) .auth(token, { type: "bearer" }) .expect(TestData.SuccessfulGetStatusCode) .then((res) => { @@ -84,7 +86,18 @@ describe("Test v3 history in datasetLifecycle", () => { }); }); - it("Should check v3 built history with field including history", async () => { + it("Should check v3 built history without fields", async () => { + await request(app.getHttpServer()) + .get(`/api/v3/datasets/${encodeURIComponent(dsId)}`) + .auth(token, { type: "bearer" }) + .expect(TestData.SuccessfulGetStatusCode) + .then((res) => { + expect(res.body.datasetName).toBeDefined(); + expect(res.body.history).toBeUndefined(); + }); + }); + + it("Should check v3 built history with field including history and datasetName", async () => { const filter = { fields: ["datasetName", "history"] }; await request(app.getHttpServer()) .get(`/api/v3/datasets/${encodeURIComponent(dsId)}`) From c3a5db97c0245e582a0dc168feceb72fde1ad831 Mon Sep 17 00:00:00 2001 From: minottic Date: Tue, 2 Dec 2025 12:58:34 +0000 Subject: [PATCH 2/3] Fetch full ds from DB for replay start --- src/datasets/datasets.controller.ts | 8 ++++++-- src/datasets/utils/history.util.ts | 8 ++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/datasets/datasets.controller.ts b/src/datasets/datasets.controller.ts index 7cdd17c88..0e78cf3cd 100644 --- a/src/datasets/datasets.controller.ts +++ b/src/datasets/datasets.controller.ts @@ -597,14 +597,18 @@ export class DatasetsController { const includeHistory = Array.isArray(fields) && fields.includes("history"); - if (includeHistory) + if (includeHistory) { + const currentDataset = (await this.datasetsService.findOne({ + where: { pid: inputDataset._id }, + })) as DatasetDocument; propertiesModifier.history = convertGenericHistoriesToObsoleteHistories( await this.historyService.find({ documentId: inputDataset._id, subsystem: "Dataset", }), - inputDataset, + currentDataset, ); + } } const outputDataset = { diff --git a/src/datasets/utils/history.util.ts b/src/datasets/utils/history.util.ts index 828b0ebb4..606753d07 100644 --- a/src/datasets/utils/history.util.ts +++ b/src/datasets/utils/history.util.ts @@ -5,7 +5,6 @@ import { DatasetDocument, } from "src/datasets/schemas/dataset.schema"; import { computeDeltaWithOriginals } from "src/common/utils/delta.util"; -import { cloneDeep } from "lodash"; const IGNORE_FIELDS = ["updatedAt", "updatedBy", "_id"]; @@ -93,17 +92,14 @@ export function convertGenericHistoriesToObsoleteHistories( histories: GenericHistory[], currentDataset: DatasetDocument | DatasetClass, ): HistoryClass[] { - let currentDatasetCopy; - if ("$clone" in currentDataset) currentDatasetCopy = currentDataset.$clone(); - else currentDatasetCopy = cloneDeep(currentDataset); const result: HistoryClass[] = []; for (const history of histories) { const obsoleteHistory = convertGenericHistoryToObsoleteHistory( history, - currentDatasetCopy, + currentDataset, ); for (const key of Object.keys(history.before || {})) { - (currentDatasetCopy as unknown as Record)[key] = + (currentDataset as unknown as Record)[key] = history.before?.[key]; } result.push(obsoleteHistory); From 72c9b432cdf841fc6a92020035355968e8cf5f3f Mon Sep 17 00:00:00 2001 From: minottic Date: Tue, 9 Dec 2025 11:44:29 +0000 Subject: [PATCH 3/3] Append history to getById --- src/datasets/datasets.controller.ts | 35 ++++++++------ .../jest-e2e-tests/datasetHistory.e2e-spec.ts | 48 +++++++++++++------ 2 files changed, 53 insertions(+), 30 deletions(-) diff --git a/src/datasets/datasets.controller.ts b/src/datasets/datasets.controller.ts index 0e78cf3cd..9cbc9a046 100644 --- a/src/datasets/datasets.controller.ts +++ b/src/datasets/datasets.controller.ts @@ -594,21 +594,10 @@ export class DatasetsController { } } - const includeHistory = - Array.isArray(fields) && fields.includes("history"); - - if (includeHistory) { - const currentDataset = (await this.datasetsService.findOne({ - where: { pid: inputDataset._id }, - })) as DatasetDocument; - propertiesModifier.history = convertGenericHistoriesToObsoleteHistories( - await this.historyService.find({ - documentId: inputDataset._id, - subsystem: "Dataset", - }), - currentDataset, + if (Array.isArray(fields) && fields.includes("history")) + propertiesModifier.history = await this.convertToObsoleteHistory( + inputDataset._id, ); - } } const outputDataset = { @@ -619,6 +608,19 @@ export class DatasetsController { return plainToInstance(OutputDatasetObsoleteDto, outputDataset); } + private async convertToObsoleteHistory(datasetId: string) { + const currentDataset = (await this.datasetsService.findOne({ + where: { pid: datasetId }, + })) as DatasetDocument; + return convertGenericHistoriesToObsoleteHistories( + await this.historyService.find({ + documentId: datasetId, + subsystem: "Dataset", + }), + currentDataset, + ); + } + // POST https://scicat.ess.eu/api/v3/datasets @UseGuards(PoliciesGuard) @CheckPolicies("datasets", (ability: AppAbility) => @@ -1287,7 +1289,10 @@ export class DatasetsController { }); if (dataset.length == 0) throw new ForbiddenException("Unauthorized access"); - return dataset[0] as OutputDatasetObsoleteDto; + return { + ...dataset[0], + history: await this.convertToObsoleteHistory(dataset[0].pid), + }; } // PATCH /datasets/:id diff --git a/test/jest-e2e-tests/datasetHistory.e2e-spec.ts b/test/jest-e2e-tests/datasetHistory.e2e-spec.ts index 41901d413..d9001415e 100644 --- a/test/jest-e2e-tests/datasetHistory.e2e-spec.ts +++ b/test/jest-e2e-tests/datasetHistory.e2e-spec.ts @@ -58,13 +58,13 @@ describe("Test v3 history in datasetLifecycle", () => { it("Should check v3 built history with field including history", async () => { const filter = { fields: ["history"] }; await request(app.getHttpServer()) - .get(`/api/v3/datasets/${encodeURIComponent(dsId)}`) + .get("/api/v3/datasets") .query({ filter: JSON.stringify(filter) }) .auth(token, { type: "bearer" }) .expect(TestData.SuccessfulGetStatusCode) .then((res) => { - expect(res.body.history.length).toBe(1); - const lifecycle = res.body.history[0].datasetlifecycle; + expect(res.body[0].history.length).toBe(1); + const lifecycle = res.body[0].history[0].datasetlifecycle; expect(lifecycle.previousValue.archivable).toBe(true); expect(lifecycle.currentValue.archivable).toBe(false); expect(lifecycle.previousValue.retrievable).toBeDefined(); @@ -76,37 +76,55 @@ describe("Test v3 history in datasetLifecycle", () => { it("Should check v3 built history with field not including history", async () => { const filter = { fields: ["datasetName"] }; await request(app.getHttpServer()) - .get(`/api/v3/datasets/${encodeURIComponent(dsId)}`) + .get("/api/v3/datasets") .query({ filter: JSON.stringify(filter) }) .auth(token, { type: "bearer" }) .expect(TestData.SuccessfulGetStatusCode) .then((res) => { - expect(res.body.datasetName).toBeDefined(); - expect(res.body.history).toBeUndefined(); + expect(res.body[0].datasetName).toBeDefined(); + expect(res.body[0].history).toBeUndefined(); }); }); - it("Should check v3 built history without fields", async () => { + ["", "/fullquery"].forEach((t) => { + it(`Should check v3 built history without fields ${t}`, async () => { + await request(app.getHttpServer()) + .get(`/api/v3/datasets${t}`) + .auth(token, { type: "bearer" }) + .expect(TestData.SuccessfulGetStatusCode) + .then((res) => { + expect(res.body[0].datasetName).toBeDefined(); + expect(res.body[0].history).toBeUndefined(); + }); + }); + }); + + it("Should check v3 built history with field including history and datasetName", async () => { + const filter = { fields: ["datasetName", "history"] }; await request(app.getHttpServer()) - .get(`/api/v3/datasets/${encodeURIComponent(dsId)}`) + .get("/api/v3/datasets") + .query({ filter: JSON.stringify(filter) }) .auth(token, { type: "bearer" }) .expect(TestData.SuccessfulGetStatusCode) .then((res) => { - expect(res.body.datasetName).toBeDefined(); - expect(res.body.history).toBeUndefined(); + expect(res.body[0].datasetName).toBeDefined(); + expect(res.body[0].history).toBeDefined(); }); }); - it("Should check v3 built history with field including history and datasetName", async () => { - const filter = { fields: ["datasetName", "history"] }; + it("Should check v3 built history by ID", async () => { await request(app.getHttpServer()) .get(`/api/v3/datasets/${encodeURIComponent(dsId)}`) - .query({ filter: JSON.stringify(filter) }) .auth(token, { type: "bearer" }) .expect(TestData.SuccessfulGetStatusCode) .then((res) => { - expect(res.body.datasetName).toBeDefined(); - expect(res.body.history).toBeDefined(); + expect(res.body.history.length).toBe(1); + const lifecycle = res.body.history[0].datasetlifecycle; + expect(lifecycle.previousValue.archivable).toBe(true); + expect(lifecycle.currentValue.archivable).toBe(false); + expect(lifecycle.previousValue.retrievable).toBeDefined(); + expect(lifecycle.currentValue.retrievable).toBeUndefined(); + expect(typeof lifecycle.previousValue._id).toBe("string"); }); }); });