From b6601d9117e2e2e91afbc17e172b1f400980407d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cyrill=20P=C3=BCntener?= Date: Mon, 15 Dec 2025 14:43:41 +0100 Subject: [PATCH 01/49] bump version to 0.57.0 --- backend/package.json | 2 +- cli/setup.cfg | 2 +- docs/package.json | 2 +- frontend/package.json | 2 +- package.json | 2 +- packages/api-dto/package.json | 2 +- packages/backend-common/package.json | 2 +- packages/shared/package.json | 2 +- packages/validation/package.json | 2 +- queueConsumer/package.json | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/backend/package.json b/backend/package.json index ec78fccb5..84bb5dc2e 100644 --- a/backend/package.json +++ b/backend/package.json @@ -1,6 +1,6 @@ { "name": "kleinkram-backend", - "version": "0.56.0", + "version": "0.57.0", "description": "", "author": "", "license": "MIT", diff --git a/cli/setup.cfg b/cli/setup.cfg index c1051814c..5ace8acf5 100644 --- a/cli/setup.cfg +++ b/cli/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = kleinkram -version = 0.56.0 +version = 0.57.0 description = give me your bags long_description = file: README.md long_description_content_type = text/markdown diff --git a/docs/package.json b/docs/package.json index 2be307028..49c77b5d2 100644 --- a/docs/package.json +++ b/docs/package.json @@ -1,6 +1,6 @@ { "name": "kleinkram-docs", - "version": "0.56.0", + "version": "0.57.0", "license": "MIT", "target": "es2022", "scripts": { diff --git a/frontend/package.json b/frontend/package.json index b68fcfb55..743f2ed41 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "kleinkram-frontend", - "version": "0.56.0", + "version": "0.57.0", "description": "Data storage of ROS bags", "productName": "Kleinkram", "author": "Johann Schwabe ", diff --git a/package.json b/package.json index 575529dd0..f8be28bf1 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "kleinkram", - "version": "0.56.0", + "version": "0.57.0", "main": "index.js", "repository": "git@github.com:leggedrobotics/GrandTourDatasets.git", "author": "Johann Schwabe ", diff --git a/packages/api-dto/package.json b/packages/api-dto/package.json index bd1e0a3d6..4931c4b68 100644 --- a/packages/api-dto/package.json +++ b/packages/api-dto/package.json @@ -1,6 +1,6 @@ { "name": "@kleinkram/api-dto", - "version": "0.56.0", + "version": "0.57.0", "main": "dist/index.js", "types": "dist/index.d.ts", "license": "MIT", diff --git a/packages/backend-common/package.json b/packages/backend-common/package.json index 1c07882ba..1f226c895 100644 --- a/packages/backend-common/package.json +++ b/packages/backend-common/package.json @@ -1,6 +1,6 @@ { "name": "@kleinkram/backend-common", - "version": "0.56.0", + "version": "0.57.0", "main": "dist/index.js", "types": "dist/index.d.ts", "license": "MIT", diff --git a/packages/shared/package.json b/packages/shared/package.json index 8990853e1..7dbb6ecf1 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -1,6 +1,6 @@ { "name": "@kleinkram/shared", - "version": "0.56.0", + "version": "0.57.0", "main": "dist/index.js", "types": "dist/index.d.ts", "license": "MIT", diff --git a/packages/validation/package.json b/packages/validation/package.json index 5aacb0227..91a9f93c9 100644 --- a/packages/validation/package.json +++ b/packages/validation/package.json @@ -1,6 +1,6 @@ { "name": "@kleinkram/validation", - "version": "0.56.0", + "version": "0.57.0", "main": "dist/index.js", "types": "dist/index.d.ts", "license": "MIT", diff --git a/queueConsumer/package.json b/queueConsumer/package.json index 95e569195..34c59c9d5 100644 --- a/queueConsumer/package.json +++ b/queueConsumer/package.json @@ -1,6 +1,6 @@ { "name": "kleinkram-queue-consumer", - "version": "0.56.0", + "version": "0.57.0", "license": "MIT", "scripts": { "build": "nest build --webpack", From ae052ea3f6306a43687b3a4dc8bf3bf2c1f080cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cyrill=20P=C3=BCntener?= Date: Mon, 15 Dec 2025 14:44:08 +0100 Subject: [PATCH 02/49] fix typo in URL path --- frontend/src/components/user-profile/admin-settings.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/components/user-profile/admin-settings.vue b/frontend/src/components/user-profile/admin-settings.vue index eb3710fe0..cc0cc3831 100644 --- a/frontend/src/components/user-profile/admin-settings.vue +++ b/frontend/src/components/user-profile/admin-settings.vue @@ -103,7 +103,7 @@ async function recalculateHashes(): Promise { async function reextractTopics(): Promise { // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment - const { data } = await axios.post('file/reextractTopics'); + const { data } = await axios.post('files/reextractTopics'); $q.notify({ // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/restrict-template-expressions From e9e0748a4503faad5e1a4bf98a9230d0a80366d2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Dec 2025 13:46:08 +0000 Subject: [PATCH 03/49] chore(deps): bump winston from 3.18.3 to 3.19.0 in /queueConsumer Bumps [winston](https://github.com/winstonjs/winston) from 3.18.3 to 3.19.0. - [Release notes](https://github.com/winstonjs/winston/releases) - [Changelog](https://github.com/winstonjs/winston/blob/master/CHANGELOG.md) - [Commits](https://github.com/winstonjs/winston/compare/v3.18.3...v3.19.0) --- updated-dependencies: - dependency-name: winston dependency-version: 3.19.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- queueConsumer/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/queueConsumer/package.json b/queueConsumer/package.json index 34c59c9d5..136796fb0 100644 --- a/queueConsumer/package.json +++ b/queueConsumer/package.json @@ -63,7 +63,7 @@ "systeminformation": "^5.27.11", "typeorm": "^0.3.27", "util": "^0.12.5", - "winston": "3.18.3", + "winston": "3.19.0", "winston-loki": "^6.1.3" }, "devDependencies": { From 710a55ae8459552b787933c6e238bdefc2943ab8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Dec 2025 13:51:25 +0000 Subject: [PATCH 04/49] chore(deps-dev): bump jest and @types/jest in /backend Bumps [jest](https://github.com/jestjs/jest/tree/HEAD/packages/jest) and [@types/jest](https://github.com/DefinitelyTyped/DefinitelyTyped/tree/HEAD/types/jest). These dependencies needed to be updated together. Updates `jest` from 29.7.0 to 30.2.0 - [Release notes](https://github.com/jestjs/jest/releases) - [Changelog](https://github.com/jestjs/jest/blob/main/CHANGELOG.md) - [Commits](https://github.com/jestjs/jest/commits/v30.2.0/packages/jest) Updates `@types/jest` from 29.5.14 to 30.0.0 - [Release notes](https://github.com/DefinitelyTyped/DefinitelyTyped/releases) - [Commits](https://github.com/DefinitelyTyped/DefinitelyTyped/commits/HEAD/types/jest) --- updated-dependencies: - dependency-name: jest dependency-version: 30.2.0 dependency-type: direct:development update-type: version-update:semver-major - dependency-name: "@types/jest" dependency-version: 30.0.0 dependency-type: direct:development update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- backend/package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/package.json b/backend/package.json index 84bb5dc2e..d6f87e5d1 100644 --- a/backend/package.json +++ b/backend/package.json @@ -88,7 +88,7 @@ "@types/cookie-parser": "^1.4.9", "@types/dotenv": "^8.2.3", "@types/express": "^5.0.3", - "@types/jest": "^29.5.2", + "@types/jest": "^30.0.0", "@types/jsonwebtoken": "^9.0.10", "@types/multer": "^2.0.0", "@types/node": "^24.10.1", @@ -104,7 +104,7 @@ "eslint": "^9.37.0", "eslint-config-prettier": "^10.1.8", "eslint-plugin-prettier": "^5.5.4", - "jest": "^29.7.0", + "jest": "^30.2.0", "jest-junit": "^16.0.0", "node-loader": "^2.0.0", "prettier": "^3.6.2", From 523ba3e6a1b19337a2805ebe640f86a95b65613d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cyrill=20P=C3=BCntener?= Date: Mon, 15 Dec 2025 20:09:52 +0100 Subject: [PATCH 05/49] feat: use log streaming (fixes #1995) --- .../services/action-manager.service.ts | 21 +++++++++++++++---- .../actions/services/docker-daemon.service.ts | 4 ++-- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/queueConsumer/src/actions/services/action-manager.service.ts b/queueConsumer/src/actions/services/action-manager.service.ts index 3d5cdbf9c..39e78f8b6 100644 --- a/queueConsumer/src/actions/services/action-manager.service.ts +++ b/queueConsumer/src/actions/services/action-manager.service.ts @@ -350,16 +350,29 @@ export class ActionManagerService { // new transaction for each batch await this.actionRepository.manager.transaction( async (manager) => { - const _action = await manager.findOneOrFail( + const _action = await manager.findOne( ActionEntity, { where: { uuid: actionUuid }, + select: ['uuid', 'logs'], + lock: { mode: 'pessimistic_write' }, }, ); - _action.logs ??= []; - _action.logs.push(...nextLogBatch); - await manager.save(_action); + if (!_action) { + return; + } + + const newLogs = [ + ...(_action.logs ?? []), + ...nextLogBatch, + ]; + + await manager.update( + ActionEntity, + { uuid: actionUuid }, + { logs: newLogs }, + ); }, ); }), diff --git a/queueConsumer/src/actions/services/docker-daemon.service.ts b/queueConsumer/src/actions/services/docker-daemon.service.ts index edc754f5e..57bed133c 100644 --- a/queueConsumer/src/actions/services/docker-daemon.service.ts +++ b/queueConsumer/src/actions/services/docker-daemon.service.ts @@ -222,10 +222,10 @@ export class DockerDaemon { logger.info(`Container started wit id: ${container.id}`); // stop the container after max_runtime seconds - await this.killContainerAfterMaxRuntime( + this.killContainerAfterMaxRuntime( container, containerOptions.limits?.max_runtime ?? 0, - ); + ).catch((error: unknown) => logger.error(error)); return { container, From 427e6044e72919f311dff3511ba7592680c4337f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cyrill=20P=C3=BCntener?= Date: Mon, 15 Dec 2025 20:15:01 +0100 Subject: [PATCH 06/49] fix: #1994 --- .github/workflows/build-sample-actions.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-sample-actions.yml b/.github/workflows/build-sample-actions.yml index 1f753493f..c293cd9c8 100644 --- a/.github/workflows/build-sample-actions.yml +++ b/.github/workflows/build-sample-actions.yml @@ -47,7 +47,11 @@ jobs: id: tag run: | if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then - echo "tag=dev" >> $GITHUB_OUTPUT + if [ "${{ matrix.type }}" == "prod" ] && [ "${{ github.ref }}" == "refs/heads/main" ]; then + echo "tag=latest" >> $GITHUB_OUTPUT + else + echo "tag=dev" >> $GITHUB_OUTPUT + fi echo "kleinkram_version=${{ inputs.kleinkram_version }}" >> $GITHUB_OUTPUT elif [ "${{ matrix.type }}" == "dev" ]; then echo "tag=dev" >> $GITHUB_OUTPUT From eba9f839b0fc0140e3cafd642213c5757cd57ec6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cyrill=20P=C3=BCntener?= Date: Mon, 15 Dec 2025 20:22:49 +0100 Subject: [PATCH 07/49] feat: configure vitepress to use clean URLs (fixes #1997) --- backend/scripts/generate-openapi.ts | 2 +- cli/README.md | 2 +- cli/kleinkram/cli/app.py | 2 +- docs/.vitepress/config.mts | 1 + docs/nginx.conf | 2 +- frontend/src/components/documentation-icon.vue | 2 +- 6 files changed, 6 insertions(+), 5 deletions(-) diff --git a/backend/scripts/generate-openapi.ts b/backend/scripts/generate-openapi.ts index bfc368531..f01bd8931 100644 --- a/backend/scripts/generate-openapi.ts +++ b/backend/scripts/generate-openapi.ts @@ -211,7 +211,7 @@ aside: false modules.push({ name: name, path: `/${String(controllerPath)}`, - link: `generated/${name}.html`, + link: `generated/${name}`, description: `Docs for ${name} module`, }); } diff --git a/cli/README.md b/cli/README.md index 96f048daf..68d865bbb 100644 --- a/cli/README.md +++ b/cli/README.md @@ -50,7 +50,7 @@ Instead of downloading files from a specified mission you can download arbitrary klein download --dest out *id1* *id2* *id3* ``` -For more information consult the [documentation](https://docs.datasets.leggedrobotics.com/usage/python/getting-started.html). +For more information consult the [documentation](https://docs.datasets.leggedrobotics.com//usage/python/setup). ## Development diff --git a/cli/kleinkram/cli/app.py b/cli/kleinkram/cli/app.py index c467df6fe..d36d4aa8b 100644 --- a/cli/kleinkram/cli/app.py +++ b/cli/kleinkram/cli/app.py @@ -61,7 +61,7 @@ The Kleinkram CLI is a command line interface for Kleinkram. For a list of available commands, run `klein --help` or visit \ -https://docs.datasets.leggedrobotics.com/usage/python/getting-started.html \ +https://docs.datasets.leggedrobotics.com/usage/python/setup \ for more information. """ diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index 9b980bfda..40998a0c7 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -5,6 +5,7 @@ const require = createRequire(import.meta.url); export default withMermaid({ lang: 'en-US', + cleanUrls: true, title: 'Kleinkram', description: 'A structured bag and mcap dataset storage.', diff --git a/docs/nginx.conf b/docs/nginx.conf index a1fee2828..7995a90a2 100644 --- a/docs/nginx.conf +++ b/docs/nginx.conf @@ -6,6 +6,6 @@ server { # Serve VitePress from the root URL location / { - try_files $uri $uri/ /index.html; + try_files $uri $uri.html $uri/ /index.html; } } diff --git a/frontend/src/components/documentation-icon.vue b/frontend/src/components/documentation-icon.vue index d620f9fa5..df8b3e857 100644 --- a/frontend/src/components/documentation-icon.vue +++ b/frontend/src/components/documentation-icon.vue @@ -14,7 +14,7 @@ const { link } = defineProps<{ link?: string }>(); const documentationBasePath = 'https://docs.datasets.leggedrobotics.com'; -const documentationDefaultPath = '/usage/getting-started.html'; +const documentationDefaultPath = '/usage/getting-started'; const documentationLink = link ?? documentationBasePath + documentationDefaultPath; From 2d5f4ad7dd4cd308de4f94b3dde61b1084335d19 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Dec 2025 20:19:35 +0000 Subject: [PATCH 08/49] chore(deps): bump docker/login-action from 2 to 3 Bumps [docker/login-action](https://github.com/docker/login-action) from 2 to 3. - [Release notes](https://github.com/docker/login-action/releases) - [Commits](https://github.com/docker/login-action/compare/v2...v3) --- updated-dependencies: - dependency-name: docker/login-action dependency-version: '3' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build-sample-actions.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-sample-actions.yml b/.github/workflows/build-sample-actions.yml index 1f753493f..5c5cffda9 100644 --- a/.github/workflows/build-sample-actions.yml +++ b/.github/workflows/build-sample-actions.yml @@ -38,7 +38,7 @@ jobs: uses: docker/setup-buildx-action@v2 - name: Login to Docker Hub - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} From 9765146c7775255e61f4304211a839ffc9663f72 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Dec 2025 20:19:38 +0000 Subject: [PATCH 09/49] chore(deps): bump docker/build-push-action from 4 to 6 Bumps [docker/build-push-action](https://github.com/docker/build-push-action) from 4 to 6. - [Release notes](https://github.com/docker/build-push-action/releases) - [Commits](https://github.com/docker/build-push-action/compare/v4...v6) --- updated-dependencies: - dependency-name: docker/build-push-action dependency-version: '6' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build-sample-actions.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-sample-actions.yml b/.github/workflows/build-sample-actions.yml index 1f753493f..80851dc71 100644 --- a/.github/workflows/build-sample-actions.yml +++ b/.github/workflows/build-sample-actions.yml @@ -58,7 +58,7 @@ jobs: fi - name: Build and push - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v6 with: context: examples/kleinkram-actions/${{ matrix.action }} push: true From 4da5a235a92293a40c4aa8ff60c57da6a93ed2e0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Dec 2025 20:19:42 +0000 Subject: [PATCH 10/49] chore(deps): bump actions/upload-artifact from 5 to 6 Bumps [actions/upload-artifact](https://github.com/actions/upload-artifact) from 5 to 6. - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/v5...v6) --- updated-dependencies: - dependency-name: actions/upload-artifact dependency-version: '6' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/draft-pdf.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/draft-pdf.yml b/.github/workflows/draft-pdf.yml index 0591e8009..28a16d6e7 100644 --- a/.github/workflows/draft-pdf.yml +++ b/.github/workflows/draft-pdf.yml @@ -18,7 +18,7 @@ jobs: # This should be the path to the paper within your repo. paper-path: openjournals/paper.md - name: Upload - uses: actions/upload-artifact@v5 + uses: actions/upload-artifact@v6 with: name: paper # This is the output path where Pandoc will write the compiled From c27c3e40872e8f8919e5ba7cf404edcf7df67700 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Dec 2025 20:19:45 +0000 Subject: [PATCH 11/49] chore(deps): bump docker/setup-buildx-action from 2 to 3 Bumps [docker/setup-buildx-action](https://github.com/docker/setup-buildx-action) from 2 to 3. - [Release notes](https://github.com/docker/setup-buildx-action/releases) - [Commits](https://github.com/docker/setup-buildx-action/compare/v2...v3) --- updated-dependencies: - dependency-name: docker/setup-buildx-action dependency-version: '3' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build-sample-actions.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-sample-actions.yml b/.github/workflows/build-sample-actions.yml index 1f753493f..cb2f2ccf8 100644 --- a/.github/workflows/build-sample-actions.yml +++ b/.github/workflows/build-sample-actions.yml @@ -35,7 +35,7 @@ jobs: uses: actions/checkout@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Login to Docker Hub uses: docker/login-action@v2 From c55c50187e70be5a273354329167a67a649880ec Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Dec 2025 20:19:50 +0000 Subject: [PATCH 12/49] chore(deps): bump pnpm/action-setup from 2 to 4 Bumps [pnpm/action-setup](https://github.com/pnpm/action-setup) from 2 to 4. - [Release notes](https://github.com/pnpm/action-setup/releases) - [Commits](https://github.com/pnpm/action-setup/compare/v2.0.0...v4) --- updated-dependencies: - dependency-name: pnpm/action-setup dependency-version: '4' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/check-migrations.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check-migrations.yml b/.github/workflows/check-migrations.yml index eb9008639..cb7d7ca6c 100644 --- a/.github/workflows/check-migrations.yml +++ b/.github/workflows/check-migrations.yml @@ -13,7 +13,7 @@ jobs: uses: actions/checkout@v4 - name: Install pnpm - uses: pnpm/action-setup@v2 + uses: pnpm/action-setup@v4 with: version: 10 From bf9f386afdc4b46c0afb3ff6d80dcbff4915308d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Dec 2025 20:19:57 +0000 Subject: [PATCH 13/49] chore(deps): bump @nestjs/core from 10.4.6 to 11.1.9 in /queueConsumer Bumps [@nestjs/core](https://github.com/nestjs/nest/tree/HEAD/packages/core) from 10.4.6 to 11.1.9. - [Release notes](https://github.com/nestjs/nest/releases) - [Commits](https://github.com/nestjs/nest/commits/v11.1.9/packages/core) --- updated-dependencies: - dependency-name: "@nestjs/core" dependency-version: 11.1.9 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- queueConsumer/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/queueConsumer/package.json b/queueConsumer/package.json index 34c59c9d5..34d116758 100644 --- a/queueConsumer/package.json +++ b/queueConsumer/package.json @@ -26,7 +26,7 @@ "@nestjs/bull": "^11.0.4", "@nestjs/common": "10.4.7", "@nestjs/config": "^4.0.2", - "@nestjs/core": "10.4.6", + "@nestjs/core": "11.1.9", "@nestjs/platform-express": "^10.4.20", "@nestjs/schedule": "^6.0.1", "@nestjs/typeorm": "^11.0.0", From 0a3ceb1053a89d1331d9f7078072d384c7ab7282 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Dec 2025 20:19:58 +0000 Subject: [PATCH 14/49] chore(deps): bump @nestjs/swagger from 8.1.1 to 11.2.3 in /backend Bumps [@nestjs/swagger](https://github.com/nestjs/swagger) from 8.1.1 to 11.2.3. - [Release notes](https://github.com/nestjs/swagger/releases) - [Commits](https://github.com/nestjs/swagger/compare/8.1.1...11.2.3) --- updated-dependencies: - dependency-name: "@nestjs/swagger" dependency-version: 11.2.3 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- backend/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/package.json b/backend/package.json index 84bb5dc2e..7e785268f 100644 --- a/backend/package.json +++ b/backend/package.json @@ -34,7 +34,7 @@ "@nestjs/passport": "^11.0.5", "@nestjs/platform-express": "^10.4.20", "@nestjs/schedule": "^6.0.1", - "@nestjs/swagger": "^8.0.1", + "@nestjs/swagger": "^11.2.3", "@nestjs/typeorm": "^10.0.2", "@opentelemetry/api": "^1.9.0", "@opentelemetry/exporter-prometheus": "^0.208.0", From dfe7fb933ab62ba9cca52777fea9fb36cd3292fa Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Dec 2025 20:20:05 +0000 Subject: [PATCH 15/49] chore(deps): bump @aws-sdk/client-s3 in /frontend Bumps [@aws-sdk/client-s3](https://github.com/aws/aws-sdk-js-v3/tree/HEAD/clients/client-s3) from 3.726.1 to 3.952.0. - [Release notes](https://github.com/aws/aws-sdk-js-v3/releases) - [Changelog](https://github.com/aws/aws-sdk-js-v3/blob/main/clients/client-s3/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-js-v3/commits/v3.952.0/clients/client-s3) --- updated-dependencies: - dependency-name: "@aws-sdk/client-s3" dependency-version: 3.952.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- frontend/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/package.json b/frontend/package.json index 743f2ed41..fbdb379ec 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -12,7 +12,7 @@ "build": "sh ./create_build_info.sh && quasar build" }, "dependencies": { - "@aws-sdk/client-s3": "3.726.1", + "@aws-sdk/client-s3": "3.952.0", "@foxglove/rosbag": "^0.4.1", "@foxglove/rosmsg": "^5.0.5", "@foxglove/rosmsg-serialization": "^2.0.4", From 5f9f96beefc2b518b4abdd3fbbc989ed1293137d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Dec 2025 20:20:06 +0000 Subject: [PATCH 16/49] chore(deps-dev): bump @nestjs/testing from 10.4.20 to 11.1.9 in /backend Bumps [@nestjs/testing](https://github.com/nestjs/nest/tree/HEAD/packages/testing) from 10.4.20 to 11.1.9. - [Release notes](https://github.com/nestjs/nest/releases) - [Commits](https://github.com/nestjs/nest/commits/v11.1.9/packages/testing) --- updated-dependencies: - dependency-name: "@nestjs/testing" dependency-version: 11.1.9 dependency-type: direct:development update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- backend/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/package.json b/backend/package.json index 84bb5dc2e..67fea6849 100644 --- a/backend/package.json +++ b/backend/package.json @@ -82,7 +82,7 @@ "@jest/globals": "^30.2.0", "@nestjs/cli": "^11.0.10", "@nestjs/schematics": "^11.0.9", - "@nestjs/testing": "^10.0.0", + "@nestjs/testing": "^11.1.9", "@swc/core": "^1.13.5", "@swc/jest": "^0.2.39", "@types/cookie-parser": "^1.4.9", From 0b2513fa0304179e86b098de3203320ed015705f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Dec 2025 20:20:09 +0000 Subject: [PATCH 17/49] chore(deps): bump vue-router from 4.6.3 to 4.6.4 in /frontend Bumps [vue-router](https://github.com/vuejs/router) from 4.6.3 to 4.6.4. - [Release notes](https://github.com/vuejs/router/releases) - [Commits](https://github.com/vuejs/router/compare/v4.6.3...v4.6.4) --- updated-dependencies: - dependency-name: vue-router dependency-version: 4.6.4 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- frontend/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/package.json b/frontend/package.json index 743f2ed41..bb683677f 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -45,7 +45,7 @@ "vue": "^3.5.25", "vue-echarts": "^7.0.3", "vue-json-pretty": "^2.6.0", - "vue-router": "^4.6.3" + "vue-router": "^4.6.4" }, "devDependencies": { "@quasar/app-vite": "^2.4.0", From f39ce8ed8878b2ef42b79bd5f73caf9c6d4ffaef Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Dec 2025 20:20:10 +0000 Subject: [PATCH 18/49] chore(deps-dev): bump eslint from 9.39.1 to 9.39.2 in /backend Bumps [eslint](https://github.com/eslint/eslint) from 9.39.1 to 9.39.2. - [Release notes](https://github.com/eslint/eslint/releases) - [Commits](https://github.com/eslint/eslint/compare/v9.39.1...v9.39.2) --- updated-dependencies: - dependency-name: eslint dependency-version: 9.39.2 dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- backend/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/package.json b/backend/package.json index 84bb5dc2e..cbeec2a22 100644 --- a/backend/package.json +++ b/backend/package.json @@ -101,7 +101,7 @@ "@typescript-eslint/eslint-plugin": "^8.32.1", "@typescript-eslint/parser": "^8.25.0", "concurrently": "^8.2.2", - "eslint": "^9.37.0", + "eslint": "^9.39.2", "eslint-config-prettier": "^10.1.8", "eslint-plugin-prettier": "^5.5.4", "jest": "^29.7.0", From 895d3b4f767d1dad9522c4894f4fa6af2d404515 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Dec 2025 20:20:13 +0000 Subject: [PATCH 19/49] chore(deps-dev): bump autoprefixer from 10.4.22 to 10.4.23 in /frontend Bumps [autoprefixer](https://github.com/postcss/autoprefixer) from 10.4.22 to 10.4.23. - [Release notes](https://github.com/postcss/autoprefixer/releases) - [Changelog](https://github.com/postcss/autoprefixer/blob/main/CHANGELOG.md) - [Commits](https://github.com/postcss/autoprefixer/compare/10.4.22...10.4.23) --- updated-dependencies: - dependency-name: autoprefixer dependency-version: 10.4.23 dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- frontend/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/package.json b/frontend/package.json index 743f2ed41..643651015 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -53,7 +53,7 @@ "@types/qs": "^6.14.0", "@types/spark-md5": "^3.0.5", "@types/sql.js": "^1.4.9", - "autoprefixer": "^10.4.22", + "autoprefixer": "^10.4.23", "jiti": "^2.6.1", "tsx": "^4.20.6", "typescript": "^5.9.3", From 61d9894c26447301ba6c703ff0060e4fbb573348 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Dec 2025 20:20:17 +0000 Subject: [PATCH 20/49] chore(deps): bump googleapis from 166.0.0 to 168.0.0 in /queueConsumer Bumps [googleapis](https://github.com/googleapis/google-api-nodejs-client) from 166.0.0 to 168.0.0. - [Release notes](https://github.com/googleapis/google-api-nodejs-client/releases) - [Changelog](https://github.com/googleapis/google-api-nodejs-client/blob/main/release-please-config.json) - [Commits](https://github.com/googleapis/google-api-nodejs-client/compare/googleapis-v166.0.0...googleapis-v168.0.0) --- updated-dependencies: - dependency-name: googleapis dependency-version: 168.0.0 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- queueConsumer/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/queueConsumer/package.json b/queueConsumer/package.json index 34c59c9d5..1101537fe 100644 --- a/queueConsumer/package.json +++ b/queueConsumer/package.json @@ -49,7 +49,7 @@ "bull": "^4.16.4", "dockerode": "^4.0.9", "express": "^4.21.2", - "googleapis": "^166.0.0", + "googleapis": "^168.0.0", "ioredis": "^5.8.2", "jsonwebtoken": "^9.0.2", "logform": "^2.7.0", From ad35e6ff4b189daf09090e7f533217d30a05027b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Dec 2025 20:20:18 +0000 Subject: [PATCH 21/49] chore(deps-dev): bump @types/node from 24.10.1 to 25.0.2 in /frontend Bumps [@types/node](https://github.com/DefinitelyTyped/DefinitelyTyped/tree/HEAD/types/node) from 24.10.1 to 25.0.2. - [Release notes](https://github.com/DefinitelyTyped/DefinitelyTyped/releases) - [Commits](https://github.com/DefinitelyTyped/DefinitelyTyped/commits/HEAD/types/node) --- updated-dependencies: - dependency-name: "@types/node" dependency-version: 25.0.2 dependency-type: direct:development update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- frontend/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/package.json b/frontend/package.json index 743f2ed41..ade00c0fa 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -49,7 +49,7 @@ }, "devDependencies": { "@quasar/app-vite": "^2.4.0", - "@types/node": "^24.10.1", + "@types/node": "^25.0.2", "@types/qs": "^6.14.0", "@types/spark-md5": "^3.0.5", "@types/sql.js": "^1.4.9", From e81a4321517bd35f7c0df4ba3560663dc15f85cf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Dec 2025 20:20:36 +0000 Subject: [PATCH 22/49] chore(deps-dev): bump @types/node from 24.10.1 to 25.0.2 in /docs Bumps [@types/node](https://github.com/DefinitelyTyped/DefinitelyTyped/tree/HEAD/types/node) from 24.10.1 to 25.0.2. - [Release notes](https://github.com/DefinitelyTyped/DefinitelyTyped/releases) - [Commits](https://github.com/DefinitelyTyped/DefinitelyTyped/commits/HEAD/types/node) --- updated-dependencies: - dependency-name: "@types/node" dependency-version: 25.0.2 dependency-type: direct:development update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- docs/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/package.json b/docs/package.json index 49c77b5d2..1ef250bad 100644 --- a/docs/package.json +++ b/docs/package.json @@ -18,7 +18,7 @@ "vue-json-pretty": "^2.6.0" }, "devDependencies": { - "@types/node": "^24.10.1", + "@types/node": "^25.0.2", "mermaid": "^11.12.1", "ts-morph": "^27.0.2", "tsx": "^4.21.0", From 56077b5f1771dfb5f0334c24b48592d23c25e0b7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Dec 2025 20:20:50 +0000 Subject: [PATCH 23/49] chore(deps): bump systeminformation in /queueConsumer Bumps [systeminformation](https://github.com/sebhildebrandt/systeminformation) from 5.27.11 to 5.27.13. - [Changelog](https://github.com/sebhildebrandt/systeminformation/blob/master/CHANGELOG.md) - [Commits](https://github.com/sebhildebrandt/systeminformation/compare/v5.27.11...v5.27.13) --- updated-dependencies: - dependency-name: systeminformation dependency-version: 5.27.13 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- queueConsumer/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/queueConsumer/package.json b/queueConsumer/package.json index 34c59c9d5..d5f1075e4 100644 --- a/queueConsumer/package.json +++ b/queueConsumer/package.json @@ -60,7 +60,7 @@ "redlock": "^5.0.0-beta.2", "reflect-metadata": "0.2.2", "rxjs": "7.8.2", - "systeminformation": "^5.27.11", + "systeminformation": "^5.27.13", "typeorm": "^0.3.27", "util": "^0.12.5", "winston": "3.18.3", From a8e8e8b8103b4e12a8e47a37bdc09fc2df42ac91 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Dec 2025 20:20:56 +0000 Subject: [PATCH 24/49] chore(deps-dev): bump vue-tsc from 2.2.10 to 3.1.8 in /frontend Bumps [vue-tsc](https://github.com/vuejs/language-tools/tree/HEAD/packages/tsc) from 2.2.10 to 3.1.8. - [Release notes](https://github.com/vuejs/language-tools/releases) - [Changelog](https://github.com/vuejs/language-tools/blob/master/CHANGELOG.md) - [Commits](https://github.com/vuejs/language-tools/commits/v3.1.8/packages/tsc) --- updated-dependencies: - dependency-name: vue-tsc dependency-version: 3.1.8 dependency-type: direct:development update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- frontend/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/package.json b/frontend/package.json index 743f2ed41..2865baf79 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -58,7 +58,7 @@ "tsx": "^4.20.6", "typescript": "^5.9.3", "vite": "^7.2.6", - "vue-tsc": "^3.1.5" + "vue-tsc": "^3.1.8" }, "engines": { "node": "^22 || ^24", From 868696cf13e30ea5654e6c6e18f6113a3d5e05f3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Dec 2025 20:21:08 +0000 Subject: [PATCH 25/49] chore(deps-dev): bump vite from 7.2.6 to 7.3.0 in /docs Bumps [vite](https://github.com/vitejs/vite/tree/HEAD/packages/vite) from 7.2.6 to 7.3.0. - [Release notes](https://github.com/vitejs/vite/releases) - [Changelog](https://github.com/vitejs/vite/blob/v7.3.0/packages/vite/CHANGELOG.md) - [Commits](https://github.com/vitejs/vite/commits/v7.3.0/packages/vite) --- updated-dependencies: - dependency-name: vite dependency-version: 7.3.0 dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- docs/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/package.json b/docs/package.json index 49c77b5d2..f4896837e 100644 --- a/docs/package.json +++ b/docs/package.json @@ -22,7 +22,7 @@ "mermaid": "^11.12.1", "ts-morph": "^27.0.2", "tsx": "^4.21.0", - "vite": "^7.2.6", + "vite": "^7.3.0", "vitepress-plugin-mermaid": "2.0.17", "vue": "^3.5.25" }, From 74d627d3b5c1af6871fbfa797ca715a99c0ec27d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Dec 2025 20:21:17 +0000 Subject: [PATCH 26/49] chore(deps-dev): bump @aws-sdk/client-s3 in /backend Bumps [@aws-sdk/client-s3](https://github.com/aws/aws-sdk-js-v3/tree/HEAD/clients/client-s3) from 3.726.1 to 3.952.0. - [Release notes](https://github.com/aws/aws-sdk-js-v3/releases) - [Changelog](https://github.com/aws/aws-sdk-js-v3/blob/main/clients/client-s3/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-js-v3/commits/v3.952.0/clients/client-s3) --- updated-dependencies: - dependency-name: "@aws-sdk/client-s3" dependency-version: 3.952.0 dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- backend/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/package.json b/backend/package.json index 84bb5dc2e..c851717aa 100644 --- a/backend/package.json +++ b/backend/package.json @@ -78,7 +78,7 @@ "winston-loki": "^6.0.7" }, "devDependencies": { - "@aws-sdk/client-s3": "3.726.1", + "@aws-sdk/client-s3": "3.952.0", "@jest/globals": "^30.2.0", "@nestjs/cli": "^11.0.10", "@nestjs/schematics": "^11.0.9", From 3e7b770c36e0efb21018516b45602ca576febd9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cyrill=20P=C3=BCntener?= Date: Mon, 15 Dec 2025 21:17:38 +0100 Subject: [PATCH 27/49] feat: add support for floats as resource constraints --- .../1765827049888-action_template_floats.ts | 80 +++++++++++++++++++ .../entities/action/action-template.entity.ts | 8 +- .../src/entities/worker/worker.entity.ts | 4 +- .../services/action-manager.service.ts | 26 +++++- 4 files changed, 108 insertions(+), 10 deletions(-) create mode 100644 backend/migration/migrations/1765827049888-action_template_floats.ts diff --git a/backend/migration/migrations/1765827049888-action_template_floats.ts b/backend/migration/migrations/1765827049888-action_template_floats.ts new file mode 100644 index 000000000..494b1f316 --- /dev/null +++ b/backend/migration/migrations/1765827049888-action_template_floats.ts @@ -0,0 +1,80 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class ActionTemplateFloats1765827049888 implements MigrationInterface { + name = 'ActionTemplateFloats1765827049888'; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `ALTER TABLE "action_template" ALTER COLUMN "cpuCores" TYPE double precision`, + ); + await queryRunner.query( + `ALTER TABLE "action_template" ALTER COLUMN "cpuMemory" TYPE double precision`, + ); + await queryRunner.query( + `ALTER TABLE "action_template" ALTER COLUMN "gpuMemory" TYPE double precision`, + ); + await queryRunner.query( + `ALTER TABLE "action_template" ALTER COLUMN "maxRuntime" TYPE double precision`, + ); + + // Worker memory floats + // cpuMemory + await queryRunner.query(` + DO $$ + BEGIN + IF EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name='worker' AND column_name='cpuMemory') THEN + ALTER TABLE "worker" ALTER COLUMN "cpuMemory" TYPE double precision; + ELSE + ALTER TABLE "worker" ADD COLUMN "cpuMemory" double precision NOT NULL DEFAULT 512; + END IF; + END $$; + `); + + // gpuMemory + await queryRunner.query(` + DO $$ + BEGIN + IF EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name='worker' AND column_name='gpuMemory') THEN + ALTER TABLE "worker" ALTER COLUMN "gpuMemory" TYPE double precision; + ELSE + ALTER TABLE "worker" ADD COLUMN "gpuMemory" double precision NOT NULL DEFAULT -1; + END IF; + END $$; + `); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `ALTER TABLE "action_template" DROP COLUMN "maxRuntime"`, + ); + await queryRunner.query( + `ALTER TABLE "action_template" ADD "maxRuntime" integer NOT NULL`, + ); + await queryRunner.query( + `ALTER TABLE "action_template" DROP COLUMN "gpuMemory"`, + ); + await queryRunner.query( + `ALTER TABLE "action_template" ADD "gpuMemory" integer NOT NULL`, + ); + await queryRunner.query( + `ALTER TABLE "action_template" DROP COLUMN "cpuMemory"`, + ); + await queryRunner.query( + `ALTER TABLE "action_template" ADD "cpuMemory" integer NOT NULL`, + ); + await queryRunner.query( + `ALTER TABLE "action_template" DROP COLUMN "cpuCores"`, + ); + await queryRunner.query( + `ALTER TABLE "action_template" ADD "cpuCores" integer NOT NULL`, + ); + + // Revert worker memory to integer (might fail if data is not integer, but this is best effort reversion) + await queryRunner.query( + `ALTER TABLE "worker" ALTER COLUMN "cpuMemory" TYPE integer`, + ); + await queryRunner.query( + `ALTER TABLE "worker" ALTER COLUMN "gpuMemory" TYPE integer`, + ); + } +} diff --git a/packages/backend-common/src/entities/action/action-template.entity.ts b/packages/backend-common/src/entities/action/action-template.entity.ts index f3ca7d435..8fbef46eb 100644 --- a/packages/backend-common/src/entities/action/action-template.entity.ts +++ b/packages/backend-common/src/entities/action/action-template.entity.ts @@ -33,16 +33,16 @@ export class ActionTemplateEntity extends BaseEntity { @Column({ default: false }) isArchived!: boolean; - @Column() + @Column({ type: 'float' }) cpuCores!: number; - @Column() + @Column({ type: 'float' }) cpuMemory!: number; - @Column() + @Column({ type: 'float' }) gpuMemory!: number; - @Column() + @Column({ type: 'float' }) maxRuntime!: number; // in hours @Column({ nullable: true }) diff --git a/packages/backend-common/src/entities/worker/worker.entity.ts b/packages/backend-common/src/entities/worker/worker.entity.ts index a0b6aafbc..28e7f9acd 100644 --- a/packages/backend-common/src/entities/worker/worker.entity.ts +++ b/packages/backend-common/src/entities/worker/worker.entity.ts @@ -10,13 +10,13 @@ export class WorkerEntity extends BaseEntity { @Column() hostname!: string; - @Column() + @Column({ type: 'float' }) cpuMemory!: number; @Column({ nullable: true }) gpuModel?: string; - @Column({ default: -1 }) + @Column({ default: -1, type: 'float' }) gpuMemory!: number; @Column() diff --git a/queueConsumer/src/actions/services/action-manager.service.ts b/queueConsumer/src/actions/services/action-manager.service.ts index 39e78f8b6..f6db3067e 100644 --- a/queueConsumer/src/actions/services/action-manager.service.ts +++ b/queueConsumer/src/actions/services/action-manager.service.ts @@ -143,11 +143,12 @@ export class ActionManagerService { n_cpu: action.template.cpuCores || 1, // eslint-disable-next-line @typescript-eslint/naming-convention - memory_limit: + memory_limit: Math.ceil( (action.template.cpuMemory || 2) * - 1024 * - 1024 * - 1024, // min 2 GB + 1024 * + 1024 * + 1024, + ), // min 2 GB }, // eslint-disable-next-line @typescript-eslint/naming-convention needs_gpu: needsGpu, @@ -191,6 +192,23 @@ export class ActionManagerService { }); if (!logsObservable) { + const containerInfo = await container + .inspect() + .catch(() => null); + + if (containerInfo) { + if (containerInfo.State.OOMKilled) { + throw new Error( + 'Container was killed due to memory constraints (OOMKilled). Container logs are not available.', + ); + } + if (containerInfo.State.ExitCode === 137) { + throw new Error( + 'Container was killed (Exit Code 137). Likely due to memory constraints or manual termination. Logs are not available.', + ); + } + } + throw new Error( 'Container logs are not available. Container might never have been started correctly.', ); From 810f2f009a0f05aa5c44e7d0d92cdabd7cffe102 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cyrill=20P=C3=BCntener?= Date: Mon, 15 Dec 2025 21:42:38 +0100 Subject: [PATCH 28/49] feat: improve error msg for container exit of action containers --- .../services/action-manager.service.ts | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/queueConsumer/src/actions/services/action-manager.service.ts b/queueConsumer/src/actions/services/action-manager.service.ts index f6db3067e..316f11cdc 100644 --- a/queueConsumer/src/actions/services/action-manager.service.ts +++ b/queueConsumer/src/actions/services/action-manager.service.ts @@ -430,40 +430,40 @@ export class ActionManagerService { state = ActionState.FAILED; exit_code = exitCode; state_cause = - 'Container failed to run. The docker run command did ' + - 'not execute successfully. Please open an issue ' + - 'problem persists.'; - + 'Container failed to run. Docker run command failed.'; + break; + } + case 126: { + state = ActionState.FAILED; + exit_code = exitCode; + state_cause = 'Command cannot be invoked (Permission denied?).'; + break; + } + case 127: { + state = ActionState.FAILED; + exit_code = exitCode; + state_cause = 'Command not found.'; break; } case 139: { state = ActionState.FAILED; exit_code = exitCode; state_cause = - 'Container was terminated by the operating system via SIGSEGV signal. ' + - 'This usually happens when the container tries to access memory ' + - 'it is not allowed to access.'; - + 'Container crashed (SIGSEGV). Invalid memory access.'; break; } case 143: { state = ActionState.FAILED; exit_code = exitCode; state_cause = - 'Container was terminated by the operating system via SIGTERM signal. ' + - 'This usually happens when the container is stopped due to approaching ' + - 'time limit.'; - + 'Container stopped (SIGTERM). Time limit approached.'; break; } case 137: { state = ActionState.FAILED; exit_code = exitCode; state_cause = - 'Container was immediately terminated by the operating ' + - 'system via SIGKILL signal. This usually happens when the ' + - 'container exceeds the memory limit or reaches the time CPU limit.'; - + 'Container killed (SIGKILL). Exceeded memory or CPU limit.'; break; } default: { From ddce65e6bd7f5780f7b0a25f0ace7df08399dd5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cyrill=20P=C3=BCntener?= Date: Mon, 15 Dec 2025 22:00:24 +0100 Subject: [PATCH 29/49] feat: add actions tab to files page --- .../src/components/actions/actions-table.vue | 10 +- .../explorer-page/mission-actions.vue | 41 ++ .../explorer-page/mission-files.vue | 494 ++++++++++++++++ frontend/src/pages/files-explorer-page.vue | 535 ++---------------- frontend/src/router/routes.ts | 2 +- 5 files changed, 601 insertions(+), 481 deletions(-) create mode 100644 frontend/src/components/explorer-page/mission-actions.vue create mode 100644 frontend/src/components/explorer-page/mission-files.vue diff --git a/frontend/src/components/actions/actions-table.vue b/frontend/src/components/actions/actions-table.vue index fbd0c2a39..464987f3d 100644 --- a/frontend/src/components/actions/actions-table.vue +++ b/frontend/src/components/actions/actions-table.vue @@ -128,8 +128,14 @@ properties.handler.setSort('createdAt'); properties.handler.setDescending(true); const queryFilters = computed(() => ({ - projectUuid: (route.query.projectUuid as string) || undefined, - missionUuid: (route.query.missionUuid as string) || undefined, + projectUuid: + (route.query.projectUuid as string) || + (route.params.projectUuid as string) || + undefined, + missionUuid: + (route.query.missionUuid as string) || + (route.params.missionUuid as string) || + undefined, take: route.query.rowsPerPage ? Number(route.query.rowsPerPage) : 100, skip: route.query.page ? (Number(route.query.page) - 1) * diff --git a/frontend/src/components/explorer-page/mission-actions.vue b/frontend/src/components/explorer-page/mission-actions.vue new file mode 100644 index 000000000..273c4ee44 --- /dev/null +++ b/frontend/src/components/explorer-page/mission-actions.vue @@ -0,0 +1,41 @@ + + + diff --git a/frontend/src/components/explorer-page/mission-files.vue b/frontend/src/components/explorer-page/mission-files.vue new file mode 100644 index 000000000..b9cb840da --- /dev/null +++ b/frontend/src/components/explorer-page/mission-files.vue @@ -0,0 +1,494 @@ + + + + + diff --git a/frontend/src/pages/files-explorer-page.vue b/frontend/src/pages/files-explorer-page.vue index a23bacbc6..b372ef55a 100644 --- a/frontend/src/pages/files-explorer-page.vue +++ b/frontend/src/pages/files-explorer-page.vue @@ -33,19 +33,7 @@ - - -
-
- - - - - - - - - - - -
- -
- - + + + + + + - - - - - - -
-
- - - - -
-
- - - - - -
-
+ + + + + + + + + diff --git a/frontend/src/pages/data-table-page.vue b/frontend/src/pages/data-table-page.vue index 3040d7218..c05d11dad 100644 --- a/frontend/src/pages/data-table-page.vue +++ b/frontend/src/pages/data-table-page.vue @@ -381,7 +381,7 @@ const tagFilterQuery = computed(() => { // eslint-disable-next-line @typescript-eslint/no-explicit-any const query: Record = {}; for (const key of Object.keys(tagFilter.value)) { - query[key] = tagFilter.value[key] ?? ''; + query[key] = tagFilter.value[key]?.value ?? ''; } return query; }); From 3d80d19fd3ac20a6cfd0fa6000fcad97d65ea2a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cyrill=20P=C3=BCntener?= Date: Tue, 16 Dec 2025 09:39:14 +0100 Subject: [PATCH 36/49] feat: implement #1957 --- backend/src/endpoints/file/file.controller.ts | 9 +- .../src/endpoints/topic/topic.controller.ts | 12 ++- backend/src/services/file.service.ts | 66 ++++++++++---- backend/src/services/topic.service.ts | 34 +++++++- frontend/src/pages/data-table-page.vue | 86 ++++++++++++++----- frontend/src/services/queries/file.ts | 69 +++++++++------ frontend/src/services/queries/topic.ts | 9 +- packages/api-dto/src/types/topic.dto.ts | 22 +++++ 8 files changed, 241 insertions(+), 66 deletions(-) diff --git a/backend/src/endpoints/file/file.controller.ts b/backend/src/endpoints/file/file.controller.ts index 176a4758b..67b0e5314 100644 --- a/backend/src/endpoints/file/file.controller.ts +++ b/backend/src/endpoints/file/file.controller.ts @@ -31,8 +31,8 @@ import { FileQueryDto, FileQueueEntriesDto, FileQueueEntryDto, - FileWithTopicDto, FilesDto, + FileWithTopicDto, FoxgloveLinkResponseDto, IsUploadingDto, MoveFilesResponseDto, @@ -158,6 +158,12 @@ export class FileController { endDate: Date | undefined, @QueryOptionalString('topics', 'Name of Topics (coma separated)') topics: string, + @QueryOptionalString( + 'messageDatatypes', + 'Message datatypes to filter by (coma separated). If multiple are given, ' + + 'files containing any of the datatypes are returned (OR).', + ) + messageDatatypes: string, @QueryOptionalString( 'fileTypes', 'File types to filter by (coma separated)', @@ -193,6 +199,7 @@ export class FileController { startDate, endDate, topics, + messageDatatypes, categories, matchAllTopics, fileTypes, diff --git a/backend/src/endpoints/topic/topic.controller.ts b/backend/src/endpoints/topic/topic.controller.ts index b05906dc2..3670df22e 100644 --- a/backend/src/endpoints/topic/topic.controller.ts +++ b/backend/src/endpoints/topic/topic.controller.ts @@ -1,7 +1,7 @@ import { ApiOkResponse } from '@/decorators'; import { TopicService } from '@/services/topic.service'; import { QuerySkip, QueryTake } from '@/validation/query-decorators'; -import { TopicNamesDto, TopicsDto } from '@kleinkram/api-dto'; +import { TopicNamesDto, TopicsDto, TopicTypesDto } from '@kleinkram/api-dto'; import { Controller, Get } from '@nestjs/common'; import { AddUser, AuthHeader } from '../auth/parameter-decorator'; import { LoggedIn } from '../auth/roles.decorator'; @@ -33,4 +33,14 @@ export class TopicController { async allNames(@AddUser() user: AuthHeader): Promise { return await this.topicService.findAllNames(user.user.uuid); } + + @Get('types') + @LoggedIn() + @ApiOkResponse({ + description: 'Get all topic types', + type: TopicTypesDto, + }) + async allTypes(@AddUser() user: AuthHeader): Promise { + return await this.topicService.findAllTypes(user.user.uuid); + } } diff --git a/backend/src/services/file.service.ts b/backend/src/services/file.service.ts index e60f7dc97..40908d9fa 100644 --- a/backend/src/services/file.service.ts +++ b/backend/src/services/file.service.ts @@ -1,5 +1,19 @@ -import { SortOrder, UpdateFile } from '@kleinkram/api-dto'; +import { fileEntityToDto, fileEntityToDtoWithTopic } from '@/serialization'; +import { + FileEventsDto, + FileExistsResponseDto, + FilesDto, + FileWithTopicDto, + SortOrder, + StorageOverviewDto, + TemporaryFileAccessesDto, + UpdateFile, +} from '@kleinkram/api-dto'; +import { FileAuditService } from '@kleinkram/backend-common/audit/file-audit.service'; +import { redis } from '@kleinkram/backend-common/consts'; import { ActionEntity } from '@kleinkram/backend-common/entities/action/action.entity'; +import { CategoryEntity } from '@kleinkram/backend-common/entities/category/category.entity'; +import { FileEventEntity } from '@kleinkram/backend-common/entities/file/file-event.entity'; import { FileEntity } from '@kleinkram/backend-common/entities/file/file.entity'; import { IngestionJobEntity } from '@kleinkram/backend-common/entities/file/ingestion-job.entity'; import { MissionEntity } from '@kleinkram/backend-common/entities/mission/mission.entity'; @@ -31,7 +45,6 @@ import { Repository, SelectQueryBuilder, } from 'typeorm'; -import { fileEntityToDto, fileEntityToDtoWithTopic } from '../serialization'; import { addFileFilters, addMissionFilters, @@ -40,19 +53,6 @@ import { convertGlobToLikePattern, } from './utilities'; -import { - FileEventsDto, - FileExistsResponseDto, - FilesDto, - FileWithTopicDto, - StorageOverviewDto, - TemporaryFileAccessesDto, -} from '@kleinkram/api-dto'; -import { FileAuditService } from '@kleinkram/backend-common/audit/file-audit.service'; -import { redis } from '@kleinkram/backend-common/consts'; -import { CategoryEntity } from '@kleinkram/backend-common/entities/category/category.entity'; -import { FileEventEntity } from '@kleinkram/backend-common/entities/file/file-event.entity'; - import { TagTypeEntity } from '@kleinkram/backend-common/entities/tagType/tag-type.entity'; import { UserEntity } from '@kleinkram/backend-common/entities/user/user.entity'; import { StorageService } from '@kleinkram/backend-common/modules/storage/storage.service'; @@ -60,13 +60,13 @@ import Queue from 'bull'; // @ts-ignore import Credentials from 'minio/dist/main/Credentials'; // @ts-ignore -import { BucketItem } from 'minio/dist/main/internal/type'; import { addAccessConstraints, addAccessConstraintsToFileQuery, addAccessConstraintsToMissionQuery, addAccessConstraintsToProjectQuery, -} from '../endpoints/auth/auth-helper'; +} from '@/endpoints/auth/auth-helper'; +import { BucketItem } from 'minio/dist/main/internal/type'; import logger from '../logger'; const FIND_MANY_SORT_KEYS = { @@ -371,6 +371,7 @@ export class FileService implements OnModuleInit { startDate: Date | undefined, endDate: Date | undefined, topics: string, + messageDatatype: string, categories: string, matchAllTopics: boolean, fileTypes: string, @@ -445,6 +446,7 @@ export class FileService implements OnModuleInit { // Apply complex filters via helper methods this._applyFileTypeFilter(idQuery, fileTypes); this._applyTopicFilter(idQuery, topics, matchAllTopics); + this._applyMessageDatatypeFilter(idQuery, messageDatatype); // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition if (health) { @@ -623,6 +625,30 @@ export class FileService implements OnModuleInit { } } + /** + * Applies message datatype filtering to the query. + */ + private _applyMessageDatatypeFilter( + query: SelectQueryBuilder, + messageDatatype: string, + ): void { + if (!messageDatatype) { + return; + } + + const splitMessageDatatype = messageDatatype + .split(',') + .filter((t) => t.length > 0); + if (splitMessageDatatype.length === 0) { + return; + } + + // Filter files that have *at least one* of the message datatypes + query.andWhere('topic.type IN (:...splitMessageDatatype)', { + splitMessageDatatype, + }); + } + /** * Applies tag filtering to the query. * This is the most complex filter, requiring a 'relational division' query. @@ -996,6 +1022,9 @@ export class FileService implements OnModuleInit { // eslint-disable-next-line @typescript-eslint/naming-convention * @param uuid The unique identifier of the file * @param expires Whether the download link should expire + * @param preview_only + * @param actor + * @param action */ async generateDownload( uuid: string, @@ -1130,6 +1159,7 @@ export class FileService implements OnModuleInit { * * @param uuid The unique identifier of the file * @param actor + * @param action */ async deleteFile( uuid: string, @@ -1227,6 +1257,8 @@ export class FileService implements OnModuleInit { * @param filenames list of filenames to upload * @param missionUUID the mission to upload the files to * @param userUUID the user that is uploading the files + * @param action + * @param uploadSource */ async getTemporaryAccess( filenames: string[], diff --git a/backend/src/services/topic.service.ts b/backend/src/services/topic.service.ts index 4dfe3fc27..412172cb2 100644 --- a/backend/src/services/topic.service.ts +++ b/backend/src/services/topic.service.ts @@ -1,6 +1,6 @@ import { addAccessConstraints } from '@/endpoints/auth/auth-helper'; import { topicEntityToDto } from '@/serialization'; -import { TopicNamesDto, TopicsDto } from '@kleinkram/api-dto'; +import { TopicNamesDto, TopicsDto, TopicTypesDto } from '@kleinkram/api-dto'; import { TopicEntity } from '@kleinkram/backend-common/entities/topic/topic.entity'; import { UserEntity } from '@kleinkram/backend-common/entities/user/user.entity'; import { UserRole } from '@kleinkram/shared'; @@ -51,6 +51,38 @@ export class TopicService { }; } + async findAllTypes(userUuid: string): Promise { + const baseQuery = this.topicRepository + .createQueryBuilder('topic') + .select('DISTINCT topic.type', 'type') + .orderBy('type'); + + const user = await this.userRepository.findOneOrFail({ + where: { uuid: userUuid }, + }); + + const topicsQuery = + user.role === UserRole.ADMIN + ? baseQuery + : addAccessConstraints( + baseQuery + .leftJoin('topic.file', 'file') + .leftJoin('file.mission', 'mission') + .leftJoin('mission.project', 'project'), + userUuid, + ); + + const topics = await topicsQuery.clone().getRawMany(); + const count = await topicsQuery.getCount(); + + return { + count, + data: topics.map((topic: TopicEntity) => topic.type), + take: count, + skip: 0, + }; + } + async findAll( userUUID: string, skip: number, diff --git a/frontend/src/pages/data-table-page.vue b/frontend/src/pages/data-table-page.vue index c05d11dad..cc937a433 100644 --- a/frontend/src/pages/data-table-page.vue +++ b/frontend/src/pages/data-table-page.vue @@ -2,7 +2,7 @@
@@ -140,6 +140,27 @@
+
+
+
+ +
+
+
+
({ const displayedTopics = ref(allTopics.value); const selectedTopics = ref([]); const matchAllTopics = ref(false); +const { data: allDatatypes } = useQuery({ + queryKey: ['topicTypes'], + queryFn: allTopicTypes, +}); +const displayedDatatypes = ref(allDatatypes.value); +const selectedDatatypes = ref([]); const tagFilter: Ref> = ref({}); end.setHours(23, 59, 59, 999); @@ -371,6 +398,7 @@ const queryKeyFiles = computed(() => [ startDate, endDate, selectedTopics, + selectedDatatypes, matchAllTopics, tagFilter, selectedFileTypesFilter, @@ -390,24 +418,22 @@ const { data: _data, isLoading }: UseQueryReturnType = useQuery({ queryKey: queryKeyFiles, queryFn: () => - fetchFilteredFiles( - filter.value, - handler.value.projectUuid, - handler.value.missionUuid, - startDate.value, - endDate.value, - // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition - selectedTopics.value ?? [], - [], - matchAllTopics.value, - // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition - selectedFileTypesFilter.value ?? ([] as FileType[]), - tagFilterQuery.value, - handler.value.take, - handler.value.skip, - handler.value.sortBy, - handler.value.descending, - ), + fetchFilteredFiles({ + filename: filter.value, + projectUUID: handler.value.projectUuid, + missionUUID: handler.value.missionUuid, + startDate: startDate.value, + endDate: endDate.value, + topics: selectedTopics.value, + messageDatatypes: selectedDatatypes.value, + matchAllTopics: matchAllTopics.value, + fileTypes: selectedFileTypesFilter.value, + tag: tagFilterQuery.value, + take: handler.value.take, + skip: handler.value.skip, + sort: handler.value.sortBy, + desc: handler.value.descending, + }), placeholderData: keepPreviousData, }); @@ -538,6 +564,25 @@ function filterFunction(value: string, update: any): void { ); }); } + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +function filterDatatypeFunction(value: string, update: any): void { + if (value === '') { + // eslint-disable-next-line @typescript-eslint/no-unsafe-call + update(() => { + displayedDatatypes.value = allDatatypes.value; + }); + return; + } + // eslint-disable-next-line @typescript-eslint/no-unsafe-call + update(() => { + if (!allDatatypes.value) return; + const needle = value.toLowerCase(); + displayedDatatypes.value = allDatatypes.value.filter((v) => + v.toLowerCase().includes(needle), + ); + }); +} function useAndTopicFilter(): void { matchAllTopics.value = true; } @@ -575,6 +620,7 @@ function resetFilter(): void { handler.value.setSearch({ name: '' }); filter.value = ''; selectedTopics.value = []; + selectedDatatypes.value = []; matchAllTopics.value = false; if ( fileTypeSelectorReference.value && diff --git a/frontend/src/services/queries/file.ts b/frontend/src/services/queries/file.ts index 622a07c0a..e3cd63b39 100644 --- a/frontend/src/services/queries/file.ts +++ b/frontend/src/services/queries/file.ts @@ -10,23 +10,46 @@ import { FileType, HealthStatus } from '@kleinkram/shared'; import { AxiosResponse } from 'axios'; import axios from 'src/api/axios'; +export interface FilteredFilesConfig { + filename?: string | undefined; + projectUUID?: string | undefined; + missionUUID?: string | undefined; + startDate?: Date | undefined; + endDate?: Date | undefined; + topics?: string[] | undefined; + messageDatatypes?: string[] | undefined; + categories?: string[] | undefined; + matchAllTopics?: boolean | undefined; + fileTypes?: FileType[] | undefined; + tag?: Record | undefined; + take?: number | undefined; + skip?: number | undefined; + sort?: string | undefined; + desc?: boolean | undefined; + health?: HealthStatus | undefined; +} + export const fetchFilteredFiles = async ( - filename: string, - projectUUID?: string, - missionUUID?: string, - startDate?: Date, - endDate?: Date, - topics?: string[], - categories?: string[], - matchAllTopics?: boolean, - fileTypes?: FileType[], - tag?: Record, - take?: number, - skip?: number, - sort?: string, - desc?: boolean, - health?: HealthStatus, + config: FilteredFilesConfig, ): Promise => { + const { + filename, + projectUUID, + missionUUID, + startDate, + endDate, + topics, + messageDatatypes, + categories, + matchAllTopics, + fileTypes, + tag, + take, + skip, + sort, + desc, + health, + } = config; try { const parameters: Record = {}; if (filename) parameters.fileName = filename; @@ -35,6 +58,8 @@ export const fetchFilteredFiles = async ( if (startDate) parameters.startDate = startDate.toISOString(); if (endDate) parameters.endDate = endDate.toISOString(); if (topics && topics.length > 0) parameters.topics = topics.join(','); + if (messageDatatypes && messageDatatypes.length > 0) + parameters.messageDatatypes = messageDatatypes.join(','); if (categories && categories.length > 0) { parameters.categories = categories.join(','); } @@ -104,23 +129,19 @@ export const filesOfMission = async ( ): Promise => { const tag: Record = {}; - return fetchFilteredFiles( - filename ?? '', - undefined, + return fetchFilteredFiles({ + filename: filename ?? '', missionUUID, - undefined, - undefined, - undefined, categories, - true, + matchAllTopics: true, fileTypes, - Object.keys(tag).length > 0 ? tag : undefined, + tag: Object.keys(tag).length > 0 ? tag : undefined, take, skip, sort, desc, health, - ); + }); }; export const findOneByNameAndMission = async ( diff --git a/frontend/src/services/queries/topic.ts b/frontend/src/services/queries/topic.ts index fb9f23dd9..c65779c16 100644 --- a/frontend/src/services/queries/topic.ts +++ b/frontend/src/services/queries/topic.ts @@ -1,7 +1,12 @@ +import { TopicNamesDto, TopicTypesDto } from '@kleinkram/api-dto'; import axios from 'src/api/axios'; export const allTopicsNames = async (): Promise => { - const response = await axios.get('/topic/names'); - // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-return + const response = await axios.get('/topic/names'); + return response.data.data; +}; + +export const allTopicTypes = async (): Promise => { + const response = await axios.get('/topic/types'); return response.data.data; }; diff --git a/packages/api-dto/src/types/topic.dto.ts b/packages/api-dto/src/types/topic.dto.ts index 167d9063e..fa425e3e4 100644 --- a/packages/api-dto/src/types/topic.dto.ts +++ b/packages/api-dto/src/types/topic.dto.ts @@ -70,3 +70,25 @@ export class TopicNamesDto implements Paginated { @IsTake() take!: number; } + +export class TopicTypesDto implements Paginated { + @ApiProperty() + @IsNumber() + count!: number; + + @ApiProperty({ + type: () => [String], + description: 'List of topic types', + }) + @IsArray() + @IsString({ each: true }) + data!: string[]; + + @ApiProperty() + @IsSkip() + skip!: number; + + @ApiProperty() + @IsTake() + take!: number; +} From 5dcb7b36a8af37db687025d622f89ec4ec428988 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cyrill=20P=C3=BCntener?= Date: Tue, 16 Dec 2025 10:20:58 +0100 Subject: [PATCH 37/49] feat: add XML support for `std_msgs/msg/String` preview --- .../inspect-file/viewers/string-viewer.vue | 117 +++++++++++-- frontend/src/utils/xml-formatter.ts | 155 ++++++++++++++++++ 2 files changed, 257 insertions(+), 15 deletions(-) create mode 100644 frontend/src/utils/xml-formatter.ts diff --git a/frontend/src/components/inspect-file/viewers/string-viewer.vue b/frontend/src/components/inspect-file/viewers/string-viewer.vue index 70757717c..db4eb58a6 100644 --- a/frontend/src/components/inspect-file/viewers/string-viewer.vue +++ b/frontend/src/components/inspect-file/viewers/string-viewer.vue @@ -6,6 +6,26 @@ {{ messages.length }} messages + + XML + + + JSON +
- Copy JSON + Copy Messages (JSON) @@ -32,7 +52,12 @@ {{ formatTime(msg.logTime) }} -
+
- 
{{
-                                        parseContent(msg.data.data).json
+                                    
{{
+                                        parseContent(msg.data.data).formatted
                                     }}

                                     
                                                 copyText(
                                                     parseContent(msg.data.data)
-                                                        .json,
+                                                        .formatted,
                                                 )
                                         "
                                     >
-                                        Copy JSON
+                                        {{
+                                            parseContent(msg.data.data).isJson
+                                                ? 'Copy JSON'
+                                                : 'Copy XML'
+                                        }}
@@ -112,6 +141,7 @@ diff --git a/frontend/src/composables/use-file-filter.ts b/frontend/src/composables/use-file-filter.ts new file mode 100644 index 000000000..43ec77b81 --- /dev/null +++ b/frontend/src/composables/use-file-filter.ts @@ -0,0 +1,137 @@ +import { FileType } from '@kleinkram/shared'; +import { useQuery } from '@tanstack/vue-query'; +import { useHandler } from 'src/hooks/query-hooks'; +import { formatDate, parseDate } from 'src/services/date-formating'; +import { allTopicsNames, allTopicTypes } from 'src/services/queries/topic'; +import { FileTypeOption } from 'src/types/file-type-option'; +import { computed, reactive, ref, watch } from 'vue'; + +export interface FilterState { + filter: string; + startDates: string; + endDates: string; + selectedTopics: string[]; + selectedDatatypes: string[]; + matchAllTopics: boolean; + fileTypeFilter: FileTypeOption[] | undefined; + tagFilter: Record; +} + +const DEFAULT_STATE = (): FilterState => { + const start = new Date(0); + const end = new Date(); + end.setHours(23, 59, 59, 999); + + return { + filter: '', + startDates: formatDate(start), + endDates: formatDate(end), + selectedTopics: [], + selectedDatatypes: [], + matchAllTopics: false, + fileTypeFilter: undefined, + tagFilter: {}, + }; +}; + +export function useFileFilter() { + const handler = useHandler(); + const state = reactive(DEFAULT_STATE()); + + // -- Queries for Options -- + const { data: allTopics } = useQuery({ + queryKey: ['topics'], + queryFn: allTopicsNames, + }); + + const { data: allDatatypes } = useQuery({ + queryKey: ['topicTypes'], + queryFn: allTopicTypes, + }); + + // -- Computed -- + const startDate = computed(() => parseDate(state.startDates)); + const endDate = computed(() => parseDate(state.endDates)); + + const selectedFileTypesFilter = computed(() => { + const list = state.fileTypeFilter ?? []; + return list + .filter((option) => option.value) + .map((option) => option.name) as FileType[]; + }); + + const tagFilterQuery = computed(() => { + const query: Record = {}; + for (const key of Object.keys(state.tagFilter)) { + query[key] = state.tagFilter[key]?.value ?? ''; + } + return query; + }); + + // -- Actions -- + function resetStartDate(): void { + const defaultS = DEFAULT_STATE(); + state.startDates = defaultS.startDates; + } + + function resetEndDate(): void { + const defaultS = DEFAULT_STATE(); + state.endDates = defaultS.endDates; + } + + function resetFilter(): void { + handler.value.setProjectUUID(undefined); + handler.value.setMissionUUID(undefined); + handler.value.setSearch({ name: '' }); + + // Preserve fileTypeFilter structure if it exists, just selecting all + const currentFileTypes = state.fileTypeFilter; + + Object.assign(state, DEFAULT_STATE()); + + if (currentFileTypes) { + state.fileTypeFilter = currentFileTypes.map((it) => ({ + ...it, + value: true, + })); + } + } + + function useAndTopicFilter(): void { + state.matchAllTopics = true; + } + + function useOrTopicFilter(): void { + state.matchAllTopics = false; + } + + // -- Debounce Logic -- + const debouncedFilter = ref(state.filter); + let timeout: ReturnType; + + watch( + () => state.filter, + (value: string) => { + clearTimeout(timeout); + timeout = setTimeout(() => { + debouncedFilter.value = value; + }, 300); + }, + ); + + return { + state, + startDate, + endDate, + selectedFileTypesFilter, + tagFilterQuery, + debouncedFilter, + allTopics, + allDatatypes, + resetStartDate, + resetEndDate, + resetFilter, + useAndTopicFilter, + useOrTopicFilter, + }; +} diff --git a/frontend/src/pages/data-table-page.vue b/frontend/src/pages/data-table-page.vue index cc937a433..fba7a4f43 100644 --- a/frontend/src/pages/data-table-page.vue +++ b/frontend/src/pages/data-table-page.vue @@ -1,218 +1,7 @@ @@ -188,4 +179,28 @@ const missionRules = computed(() => { ] : []; }); + +// Dynamic placeholders showing example based on first available option +const dynamicProjectPlaceholder = computed(() => { + const first = projects.value[0]; + return first ? `e.g. ${first.name}` : props.projectPlaceholder; +}); + +const dynamicMissionPlaceholder = computed(() => { + const first = missions.value[0]; + return first ? `e.g. ${first.name}` : props.missionPlaceholder; +}); + +// Check if mission is disabled to show tooltip +const isMissionDisabled = computed(() => { + return ( + props.disabled || !selectedProjectUuid.value || isMissionsLoading.value + ); +}); + +const missionDisabledReason = computed(() => { + if (!selectedProjectUuid.value) return 'Select a project first'; + if (isMissionsLoading.value) return 'Loading missions...'; + return ''; +}); diff --git a/frontend/src/components/common/selection-button-group.vue b/frontend/src/components/common/selection-button-group.vue new file mode 100644 index 000000000..165138ac4 --- /dev/null +++ b/frontend/src/components/common/selection-button-group.vue @@ -0,0 +1,108 @@ + + + + + diff --git a/frontend/src/components/common/smart-search-input.vue b/frontend/src/components/common/smart-search-input.vue new file mode 100644 index 000000000..06a16ae8e --- /dev/null +++ b/frontend/src/components/common/smart-search-input.vue @@ -0,0 +1,434 @@ + + + + + diff --git a/frontend/src/components/file-type-selector.vue b/frontend/src/components/file-type-selector.vue index d52c3941f..dd3e15e22 100644 --- a/frontend/src/components/file-type-selector.vue +++ b/frontend/src/components/file-type-selector.vue @@ -1,52 +1,17 @@ - - diff --git a/frontend/src/components/files/files-filter.vue b/frontend/src/components/files/files-filter.vue index a52a3f818..e4a919863 100644 --- a/frontend/src/components/files/files-filter.vue +++ b/frontend/src/components/files/files-filter.vue @@ -3,305 +3,322 @@ class="q-pa-md q-mt-md bg-grey-1 rounded-borders q-mb-md border-grey-3" style="border: 1px solid #e0e0e0" > -
-
- - - -
- -
- - - +
+
+
- -
- + - - Please select a project first -
-
-
- +
+ +
- -
-
-
- - - - - And - - - Or - - - -
-
- -
-
-
- -
-
-
- -
-
-
- -
- - - -
- -
-
- - Advanced Tag Filter - - {{ Object.values(state.tagFilter).length }} - - - - -
-
-
+
diff --git a/frontend/src/components/files/filter/filter-popup.vue b/frontend/src/components/files/filter/filter-popup.vue new file mode 100644 index 000000000..4161efa9a --- /dev/null +++ b/frontend/src/components/files/filter/filter-popup.vue @@ -0,0 +1,485 @@ + + + + + diff --git a/frontend/src/components/files/filter/metadata-filter-builder.vue b/frontend/src/components/files/filter/metadata-filter-builder.vue new file mode 100644 index 000000000..7f898bf40 --- /dev/null +++ b/frontend/src/components/files/filter/metadata-filter-builder.vue @@ -0,0 +1,166 @@ + + + diff --git a/frontend/src/components/metadata-filter-input.vue b/frontend/src/components/metadata-filter-input.vue index afb314643..5e1a486b3 100644 --- a/frontend/src/components/metadata-filter-input.vue +++ b/frontend/src/components/metadata-filter-input.vue @@ -1,63 +1,134 @@ + + diff --git a/frontend/src/composables/use-file-filter.ts b/frontend/src/composables/use-file-filter.ts index 43ec77b81..f606bd61b 100644 --- a/frontend/src/composables/use-file-filter.ts +++ b/frontend/src/composables/use-file-filter.ts @@ -17,11 +17,18 @@ export interface FilterState { tagFilter: Record; } -const DEFAULT_STATE = (): FilterState => { +export const DEFAULT_STATE = (): FilterState => { const start = new Date(0); const end = new Date(); end.setHours(23, 59, 59, 999); + const allFileTypes = Object.values(FileType) + .filter((t) => t !== FileType.ALL) + .map((name) => ({ + name, + value: true, + })); + return { filter: '', startDates: formatDate(start), @@ -29,14 +36,38 @@ const DEFAULT_STATE = (): FilterState => { selectedTopics: [], selectedDatatypes: [], matchAllTopics: false, - fileTypeFilter: undefined, + fileTypeFilter: allFileTypes, tagFilter: {}, }; }; export function useFileFilter() { const handler = useHandler(); - const state = reactive(DEFAULT_STATE()); + + const defaultState = DEFAULT_STATE(); + + // Sync from URL (handler) if present + if (handler.value.searchParams.name) { + defaultState.filter = handler.value.searchParams.name; + } + if (handler.value.searchParams.startDate) { + defaultState.startDates = handler.value.searchParams.startDate; + } + if (handler.value.searchParams.endDate) { + defaultState.endDates = handler.value.searchParams.endDate; + } + + // Sync FileTypes + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition + if (handler.value.fileTypes && defaultState.fileTypeFilter) { + const activeTypes = new Set(handler.value.fileTypes); + defaultState.fileTypeFilter = defaultState.fileTypeFilter.map((ft) => ({ + ...ft, + value: activeTypes.has(ft.name as FileType), + })); + } + + const state = reactive(defaultState); // -- Queries for Options -- const { data: allTopics } = useQuery({ @@ -63,7 +94,18 @@ export function useFileFilter() { const tagFilterQuery = computed(() => { const query: Record = {}; for (const key of Object.keys(state.tagFilter)) { - query[key] = state.tagFilter[key]?.value ?? ''; + const value = state.tagFilter[key]?.value; + + if (value === '') continue; + + if (typeof value === 'string') { + if (value.trim() !== '') { + query[key] = value; + } + } else { + // Handle numbers, booleans, dates + query[key] = String(value); + } } return query; }); @@ -105,6 +147,67 @@ export function useFileFilter() { state.matchAllTopics = false; } + function applyDateShortcut( + type: 'today' | '7days' | 'lastmonth' | 'ytd' | 'all', + ): void { + if (type === 'all') { + resetStartDate(); + resetEndDate(); + // Clear from Search params + const newParameters = { ...handler.value.searchParams }; + delete newParameters.startDate; + delete newParameters.endDate; + + handler.value.setSearch({ + ...newParameters, + startDate: undefined as unknown as string, + endDate: undefined as unknown as string, + }); + return; + } + + const end = new Date(); + // End of day today + end.setHours(23, 59, 59, 999); + + const start = new Date(); + start.setHours(0, 0, 0, 0); + + switch (type) { + case 'today': { + // start is already beginning of today + break; + } + case '7days': { + start.setDate(start.getDate() - 7); + break; + } + case 'lastmonth': { + // 1 month ago + start.setMonth(start.getMonth() - 1); + break; + } + case 'ytd': { + // Jan 1st + start.setMonth(0, 1); + break; + } + // No default + } + + const s = formatDate(start); + const dateEnd = formatDate(end); + state.startDates = s; + state.endDates = dateEnd; + + // Immediately apply to handler (Search) + handler.value.setSearch({ + ...handler.value.searchParams, + startDate: s, + endDate: dateEnd, + }); + } + // -- Debounce Logic -- const debouncedFilter = ref(state.filter); let timeout: ReturnType; @@ -133,5 +236,6 @@ export function useFileFilter() { resetFilter, useAndTopicFilter, useOrTopicFilter, + applyDateShortcut, }; } diff --git a/frontend/src/composables/use-file-search.ts b/frontend/src/composables/use-file-search.ts new file mode 100644 index 000000000..5287f62c1 --- /dev/null +++ b/frontend/src/composables/use-file-search.ts @@ -0,0 +1,172 @@ +import { FileType } from '@kleinkram/shared'; +import { useQuery } from '@tanstack/vue-query'; +import { KEYWORDS } from 'src/composables/use-filter-parser'; +import { + useAllTags, + useFilteredProjects, + useMissionsOfProjectMinimal, + useProjectQuery, +} from 'src/hooks/query-hooks'; +import { allTopicsNames, allTopicTypes } from 'src/services/queries/topic'; +import { + CompositeFilterProvider, + GenericMetadataStrategy, + KeywordStrategy, + MetadataTag, + ValueStrategy, +} from 'src/services/suggestions/filter-strategies'; +import { computed, Ref } from 'vue'; + +export interface FileSearchContextData { + projects: { name: string; uuid: string }[]; + missions: { name: string; uuid: string }[]; + topics: string[]; + datatypes: string[]; + fileTypes: string[]; + availableTags: MetadataTag[]; + hasProjectSelected: boolean; +} + +export function useFileSearch(currentProjectUuid: Ref) { + // --- Data Fetching --- + + // Projects + const { data: projectsData } = useFilteredProjects(100, 0, 'name', false); + const { data: selectedProject } = useProjectQuery(currentProjectUuid); + + const projects = computed(() => { + const list = + projectsData.value?.data.map((p) => ({ + name: p.name, + uuid: p.uuid, + })) ?? []; + + if ( + selectedProject.value && + !list.some((p) => p.uuid === selectedProject.value.uuid) + ) { + list.push({ + name: selectedProject.value.name, + uuid: selectedProject.value.uuid, + }); + } + return list; + }); + + // Missions (dependent on project) + // Note: This relies on currentProjectUuid being reactive + const { data: missionsData } = useMissionsOfProjectMinimal( + currentProjectUuid, + 100, + 0, + ); + const missions = computed( + () => + missionsData.value?.data.map((m) => ({ + name: m.name, + uuid: m.uuid, + })) ?? [], + ); + + // Topics + const { data: allTopics } = useQuery({ + queryKey: ['topics'], + queryFn: allTopicsNames, + }); + + // Datatypes + const { data: allDatatypes } = useQuery({ + queryKey: ['topicTypes'], + queryFn: allTopicTypes, + }); + + // Tags + const { data: allTags } = useAllTags(); + + // FileTypes (Static) + const allFileTypes = Object.values(FileType).filter( + (f) => f !== FileType.ALL, + ); + + // --- Context Construction --- + + const contextData = computed(() => ({ + projects: projects.value, + missions: missions.value, + topics: allTopics.value ?? [], + datatypes: allDatatypes.value ?? [], + fileTypes: allFileTypes, + availableTags: + allTags.value?.map((t) => ({ + name: t.name, + datatype: t.datatype, + })) ?? [], + hasProjectSelected: !!currentProjectUuid.value, + })); + + // --- Strategies --- + + const provider = new CompositeFilterProvider([ + new KeywordStrategy(KEYWORDS, { + [KEYWORDS.MISSION]: (context) => ({ + enabled: context.data.hasProjectSelected, + reason: 'Select a project first', + }), + [KEYWORDS.TOPIC]: (context) => ({ + enabled: true, + hidden: context.input.includes(KEYWORDS.TOPIC_AND), + }), + [KEYWORDS.TOPIC_AND]: (context) => ({ + enabled: true, + hidden: /(?:^|\s)topic:/.test(context.input), + }), + }), + new ValueStrategy( + KEYWORDS.PROJECT, + 'sym_o_folder', + 'Project', + (context) => context.data.projects, + ), + new ValueStrategy( + KEYWORDS.MISSION, + 'sym_o_flag', + 'Mission', + (context) => context.data.missions, + (context) => context.data.hasProjectSelected, + ), + new ValueStrategy( + KEYWORDS.TOPIC, + 'sym_o_topic', + 'Topic', + (context) => context.data.topics.map((t) => ({ name: t })), + ), + new ValueStrategy( + KEYWORDS.DATATYPE, + 'sym_o_data_object', + 'Datatype', + (context) => context.data.datatypes.map((d) => ({ name: d })), + ), + new ValueStrategy( + KEYWORDS.FILETYPE, + 'sym_o_description', + 'File Type', + (context) => context.data.fileTypes.map((f) => ({ name: f })), + ), + new GenericMetadataStrategy( + KEYWORDS.METADATA, + (context) => context.data.availableTags, + ), + ]); + + return { + provider, + contextData, + // Expose raw data for usage in parser/other components if needed + projects, + missions, + allTopics, + allDatatypes, + allTags, + allFileTypes, + }; +} diff --git a/frontend/src/composables/use-filter-parser.ts b/frontend/src/composables/use-filter-parser.ts new file mode 100644 index 000000000..35f1a5415 --- /dev/null +++ b/frontend/src/composables/use-filter-parser.ts @@ -0,0 +1,301 @@ +import { + parseSearchString, + validateSearchSyntax, +} from 'src/services/suggestions/search-parser'; +import { computed, Ref } from 'vue'; +import { FilterState } from './use-file-filter'; + +export const KEYWORDS = { + PROJECT: 'project:', + MISSION: 'mission:', + TOPIC: 'topic:', + TOPIC_AND: '&topic:', + DATATYPE: 'datatype:', + FILETYPE: 'filetype:', + START: 'date-start:', + END: 'date-end:', + METADATA: 'meta:', +}; + +interface NamedItem { + name: string; + uuid: string; +} + +interface ParsedFilter { + projectUuid?: string; + missionUuid?: string; + topics: string[]; + matchAllTopics: boolean; + datatypes: string[]; + fileTypes: Set; + metadata: Record; + startDate?: string; + endDate?: string; + freeText: string; +} + +const quote = (s: string) => (s.includes(' ') ? `"${s}"` : s); + +export function useFilterParser( + state: FilterState, + setFilterString: (filterString_: string) => void, + options: { + projects: Ref; + missions: Ref; + projectUuid: Ref; + missionUuid: Ref; + setProject: (uuid: string | undefined) => void; + setMission: (uuid: string | undefined) => void; + defaultStartDate?: string; + defaultEndDate?: string; + }, +) { + const filterString = computed(() => { + const parts: string[] = []; + + if (options.projectUuid.value) { + const p = options.projects.value.find( + (x) => x.uuid === options.projectUuid.value, + ); + if (p) parts.push(`${KEYWORDS.PROJECT}${quote(p.name)}`); + } + if (options.missionUuid.value) { + const m = options.missions.value.find( + (x) => x.uuid === options.missionUuid.value, + ); + if (m) parts.push(`${KEYWORDS.MISSION}${quote(m.name)}`); + } + + const topicKeyword = state.matchAllTopics + ? KEYWORDS.TOPIC_AND + : KEYWORDS.TOPIC; + for (const t of state.selectedTopics) + parts.push(`${topicKeyword}${quote(t)}`); + for (const d of state.selectedDatatypes) + parts.push(`${KEYWORDS.DATATYPE}${quote(d)}`); + + if (state.fileTypeFilter) { + const allSelected = state.fileTypeFilter.every((ft) => ft.value); + if (!allSelected) { + for (const ft of state.fileTypeFilter) { + if (ft.value) + parts.push(`${KEYWORDS.FILETYPE}${quote(ft.name)}`); + } + } + } + + for (const tag of Object.values(state.tagFilter)) { + parts.push( + `${KEYWORDS.METADATA}${quote(`${tag.name}=${tag.value}`)}`, + ); + } + + // Add date filters if set (skip if "All" - epoch date 01.01.1970) + const startDateOnly = state.startDates.split(' ')[0]; + const isAllDates = startDateOnly === '01.01.1970'; + + if (!isAllDates && state.startDates && startDateOnly) + parts.push(`${KEYWORDS.START}${startDateOnly}`); + if (!isAllDates && state.endDates) { + const dateOnly = state.endDates.split(' ')[0]; + if (dateOnly) parts.push(`${KEYWORDS.END}${dateOnly}`); + } + + if (state.filter) { + parts.push(state.filter); + } + + return parts.join(' '); + }); + + // eslint-disable-next-line complexity + function parseTokens(input: string): ParsedFilter { + const result: ParsedFilter = { + topics: [], + matchAllTopics: false, + datatypes: [], + fileTypes: new Set(), + metadata: {}, + freeText: '', + }; + + const parsed = parseSearchString(input); + result.freeText = parsed.freeText; + + // Check if there was matchForAll in tokens + let hasAndTopic = false; + + for (const token of parsed.tokens) { + if (!token.key) continue; + + const keyWithColon = token.key + ':'; + + let handled = true; + + switch (keyWithColon) { + case KEYWORDS.PROJECT: { + if (token.value) { + if (options.projects.value.length === 0) { + // Projects not loaded yet, preserve current if set + if (options.projectUuid.value) { + result.projectUuid = options.projectUuid.value; + } + } else { + const project = options.projects.value.find( + (p) => + p.name.toLowerCase() === + token.value.toLowerCase() || + p.uuid === token.value, + ); + if (project) result.projectUuid = project.uuid; + } + } + break; + } + case KEYWORDS.MISSION: { + if (token.value) { + if (options.missions.value.length === 0) { + if (options.missionUuid.value) { + result.missionUuid = options.missionUuid.value; + } + } else { + const mission = options.missions.value.find( + (m) => + m.name.toLowerCase() === + token.value.toLowerCase() || + m.uuid === token.value, + ); + if (mission) result.missionUuid = mission.uuid; + } + } + break; + } + case KEYWORDS.TOPIC: { + if (token.value) result.topics.push(token.value); + break; + } + case KEYWORDS.TOPIC_AND: { + if (token.value) { + result.topics.push(token.value); + hasAndTopic = true; + } + break; + } + case KEYWORDS.DATATYPE: { + if (token.value) result.datatypes.push(token.value); + break; + } + case KEYWORDS.FILETYPE: { + if (token.value) + result.fileTypes.add(token.value.toLowerCase()); + break; + } + case KEYWORDS.METADATA: { + if (token.value.includes('=')) { + const [mKey, mValue] = token.value.split('='); + if (mKey && mValue) { + result.metadata[mKey] = { + name: mKey, + value: mValue, + }; + } + } + break; + } + case KEYWORDS.START: + case KEYWORDS.END: { + break; + } + default: { + handled = false; + } + } + + if (!handled) { + result.freeText += + (result.freeText ? ' ' : '') + token.original; + } + } + + result.matchAllTopics = hasAndTopic; + return result; + } + + function parse(input: string) { + const parsed = parseTokens(input); + + options.setProject(parsed.projectUuid); + options.setMission(parsed.missionUuid); + + state.selectedTopics = parsed.topics; + state.matchAllTopics = parsed.matchAllTopics; + state.selectedDatatypes = parsed.datatypes; + + if (state.fileTypeFilter) { + const implicitlyAll = parsed.fileTypes.size === 0; + + for (const ft of state.fileTypeFilter) { + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition + if (ft) { + ft.value = implicitlyAll + ? true + : parsed.fileTypes.has(ft.name.toLowerCase()); + } + } + } + + state.tagFilter = parsed.metadata; + + // Date Parsing + if (options.defaultStartDate) { + // Append 00:00 if parser extracted just date + const startValue = parsed.startDate + ? `${parsed.startDate} 00:00` + : options.defaultStartDate; + // Check if user typed full time? Assuming they type DD.MM.YYYY (split by space [0] used in reconstruction) + // If user typed '01.01.2023', we make it '01.01.2023 00:00'. + state.startDates = startValue; + } + if (options.defaultEndDate) { + const endValue = parsed.endDate + ? `${parsed.endDate} 23:59` + : options.defaultEndDate; + state.endDates = endValue; + } + + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition + state.filter = (parsed.freeText.trim() ?? '').replaceAll(/\s+/g, ' '); + } + + function validateSyntax(input: string): string | null { + const validKeys = Object.values(KEYWORDS).map((k) => + k.replace(':', ''), + ); + const error = validateSearchSyntax(input, validKeys); + if (error) return error; + + // Check for invalid use of & prefix (only allowed for topic:) + // The generic validator doesn't check specific key rules. + const invalidAndPrefixRegex = + /&(project|mission|datatype|filetype|date-start|date-end|meta):/gi; + const invalidMatch = invalidAndPrefixRegex.exec(input); + if (invalidMatch) { + // eslint-disable-next-line @typescript-eslint/restrict-template-expressions + return `The & prefix is only valid for topics (use &topic: for AND matching). Invalid: &${invalidMatch[1]}:`; + } + + // Check Mission Dependency + if (input.includes(KEYWORDS.MISSION) && !options.projectUuid.value) { + return 'Cannot filter for mission without project filter.'; + } + + return null; + } + + return { + filterString, + parse, + validateSyntax, + }; +} diff --git a/frontend/src/services/queries/file.ts b/frontend/src/services/queries/file.ts index e3cd63b39..56b69cfab 100644 --- a/frontend/src/services/queries/file.ts +++ b/frontend/src/services/queries/file.ts @@ -31,6 +31,7 @@ export interface FilteredFilesConfig { export const fetchFilteredFiles = async ( config: FilteredFilesConfig, + // eslint-disable-next-line complexity ): Promise => { const { filename, diff --git a/frontend/src/services/query-handler.ts b/frontend/src/services/query-handler.ts index bf79125bc..1a5add30b 100644 --- a/frontend/src/services/query-handler.ts +++ b/frontend/src/services/query-handler.ts @@ -214,6 +214,14 @@ export class QueryURLHandler extends QueryHandler { ); } + async safeWriteURL(): Promise { + try { + await this.writeURL(); + } catch (error) { + console.error('Failed to update URL:', error); + } + } + /** * --------------------------------------------- * Setters that update the URL @@ -221,74 +229,53 @@ export class QueryURLHandler extends QueryHandler { */ override setPage(page: number): void { super.setPage(page); - this.writeURL().catch((error: unknown) => { - console.error(error); - }); + void this.safeWriteURL(); } override setTake(take: number): void { super.setTake(take); - this.writeURL().catch((error: unknown) => { - console.error(error); - }); + void this.safeWriteURL(); } override setSort(sortBy: string): void { super.setSort(sortBy); - this.writeURL().catch((error: unknown) => { - console.error(error); - }); + void this.safeWriteURL(); } override setDescending(descending: boolean): void { super.setDescending(descending); - this.writeURL().catch((error: unknown) => { - console.error(error); - }); + void this.safeWriteURL(); } override setProjectUUID(projectUuid: string | undefined): void { super.setProjectUUID(projectUuid); - this.writeURL().catch((error: unknown) => { - console.error(error); - }); + void this.safeWriteURL(); } override setMissionUUID(missionUuid: string | undefined): void { super.setMissionUUID(missionUuid); - this.writeURL().catch((error: unknown) => { - console.error(error); - }); + void this.safeWriteURL(); } override setSearch(searchParameters: Record): void { super.setSearch(searchParameters); - this.writeURL().catch((error: unknown) => { - console.error(error); - }); + void this.safeWriteURL(); } - // Renamed and updated to call the new super method override setFileTypes(fileTypes: FileType[]): void { super.setFileTypes(fileTypes); - this.writeURL().catch((error: unknown) => { - console.error(error); - }); + void this.safeWriteURL(); } override setCategories(categories: string[]): void { super.setCategories(categories); - this.writeURL().catch((error: unknown) => { - console.error(error); - }); + void this.safeWriteURL(); } override addCategory(category: string): void { if (!this.categories.includes(category)) { super.addCategory(category); - this.writeURL().catch((error: unknown) => { - console.error(error); - }); + void this.safeWriteURL(); } } @@ -318,12 +305,30 @@ export class QueryURLHandler extends QueryHandler { : ''; const searchParameters = {} as Record; - if (route.query.name) - searchParameters.name = route.query.name as string; - if (route.query.health) - searchParameters.health = route.query.health as string; - // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition - this.searchParams = searchParameters ?? DEFAULT_SEARCH; + + // Reserved keys that are handled separately + const reservedKeys = new Set([ + 'page', + 'rowsPerPage', + 'sortBy', + 'descending', + 'projectUuid', + 'missionUuid', + 'file_type', + 'categories', + ]); + + for (const key of Object.keys(route.query)) { + if (!reservedKeys.has(key)) { + const value = route.query[key]; + // We only support string filters for now (single value) + if (typeof value === 'string') { + searchParameters[key] = value; + } + } + } + + this.searchParams = searchParameters; const queryFileTypes = route.query.file_type as string | undefined; this.fileTypes = @@ -349,8 +354,6 @@ export class QueryURLHandler extends QueryHandler { // Logic to determine if fileTypes is in its default state const allFileTypesCount = Object.values(FileType).length; const isDefaultFileTypes = - // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition - !this.fileTypes || this.fileTypes.length === 0 || this.fileTypes.length === allFileTypesCount; @@ -377,7 +380,6 @@ export class QueryURLHandler extends QueryHandler { ...this.searchParams, // Join the array into a single comma-separated string - // eslint-disable-next-line @typescript-eslint/naming-convention file_type: isDefaultFileTypes ? undefined @@ -387,7 +389,6 @@ export class QueryURLHandler extends QueryHandler { this.categories.length > 0 ? this.categories : undefined, }; - // check if any query was set before writing to the URL const queries = this.router.currentRoute.value.query; const hasQueries = Object.keys(queries).length > 0; @@ -395,7 +396,7 @@ export class QueryURLHandler extends QueryHandler { // eslint-disable-next-line @typescript-eslint/no-explicit-any const finalQuery: Record = {}; for (const [key, value] of Object.entries(newQuery)) { - if (value !== undefined) { + if (value !== undefined && value !== '') { finalQuery[key] = value; } } diff --git a/frontend/src/services/suggestions/filter-strategies.ts b/frontend/src/services/suggestions/filter-strategies.ts new file mode 100644 index 000000000..64247ea00 --- /dev/null +++ b/frontend/src/services/suggestions/filter-strategies.ts @@ -0,0 +1,266 @@ +import { + Suggestion, + SuggestionContext, + SuggestionProvider, +} from './suggestion-types'; + +export abstract class BaseStrategy implements SuggestionProvider { + abstract getSuggestions(context: SuggestionContext): Suggestion[]; + + protected getLastWord(input: string): string { + const lastWordRegex = + /((?:[a-zA-Z0-9_-]+:"[^"]*)|(?:[a-zA-Z0-9_:=> extends BaseStrategy { + constructor( + private keywords: Record, + private rules?: Record< + string, + (context: SuggestionContext) => { + enabled: boolean; + reason?: string; + hidden?: boolean; + } + >, + ) { + super(); + } + + getSuggestions(context: SuggestionContext): Suggestion[] { + const lastWord = this.getLastWord(context.input); + const lowerLast = lastWord.toLowerCase(); + + // If we are ALREADY typing a value (contains :), keywords are not suggestions + if (lastWord.includes(':')) return []; + + const list: Suggestion[] = []; + + for (const [key, prefix] of Object.entries(this.keywords)) { + // Allow matching "topic" to "&topic:" (ignore leading &) + // This allows suggestions for the "AND" variant even if user just types "topic" + const matches = + prefix.startsWith(lowerLast) || + (prefix.startsWith('&') && + prefix.slice(1).startsWith(lowerLast)); + + if (matches) { + let disabled = false; + let desc = `Filter by ${key.toLowerCase()}`; + + // Check rules + if (this.rules?.[prefix]) { + const ruleResult = this.rules[prefix](context); + if (ruleResult.hidden) { + continue; + } + if (!ruleResult.enabled) { + disabled = true; + desc = ruleResult.reason ?? desc; + } + } + + list.push( + this.createSuggestion( + '', + '', + prefix, + desc, + 'sym_o_filter_alt', + false, + disabled, + ), + ); + } + } + + return list; + } +} + +export class ValueStrategy extends BaseStrategy { + constructor( + private keyword: string, + private icon: string, + private description: string, + private getItems: (context: SuggestionContext) => { name: string }[], + private dependencyCheck?: (context: SuggestionContext) => boolean, + ) { + super(); + } + + getSuggestions(context: SuggestionContext): Suggestion[] { + const lastWord = this.getLastWord(context.input); + const lowerLast = lastWord.toLowerCase(); + + // We only match if the start matches the keyword + if ( + !lowerLast.startsWith(this.keyword) && + !this.keyword.startsWith(lowerLast) + ) { + return []; + } + + const list: Suggestion[] = []; + + if (lowerLast.startsWith(this.keyword)) { + let query = lowerLast.slice(this.keyword.length); + if (query.startsWith('"')) { + query = query.slice(1); + } + + const disabled = this.dependencyCheck + ? !this.dependencyCheck(context) + : false; + + const items = this.getItems(context); + + for (const item of items + .filter((index) => + (index.name || '').toLowerCase().includes(query), + ) + .slice(0, 5)) { + const itemName = item.name || ''; + const label = itemName.includes(' ') + ? `"${itemName}"` + : itemName; + + list.push( + this.createSuggestion( + label, + itemName, + this.keyword, + this.description, + this.icon, + true, + disabled, + ), + ); + } + } + + return list; + } +} + +export interface MetadataTag { + name: string; + datatype: string; +} + +export class GenericMetadataStrategy extends BaseStrategy { + constructor( + private prefix: string, + private getTags: (context: SuggestionContext) => MetadataTag[], + ) { + super(); + } + + getSuggestions(context: SuggestionContext): Suggestion[] { + const lastWord = this.getLastWord(context.input); + const lowerLast = lastWord.toLowerCase(); + + if (!lowerLast.startsWith(this.prefix)) return []; + + const metaPart = lowerLast.slice(this.prefix.length); + const operatorMatch = /([=> t.name.toLowerCase() === metaPart, + ); + if (exactTag) { + for (const op of ['=', '!=', '>', '<', '>=', '<=']) { + list.push( + this.createSuggestion( + op, + op, + this.prefix + exactTag.name, + `Operator ${op}`, + 'sym_o_calculate', + false, + false, + ), + ); + } + } + + // Keys - append = to value so selecting gives meta:description= + for (const tag of availableTags) { + if ( + tag.name.toLowerCase().includes(metaPart) && + tag.name.toLowerCase() !== metaPart + ) { + list.push( + this.createSuggestion( + tag.name, + tag.name + '=', + this.prefix, + `Metadata: ${tag.datatype}`, + 'sym_o_label', + false, + false, + ), + ); + } + } + + // All keys if empty - append = to value + if (metaPart === '') { + for (const tag of availableTags.slice(0, 50)) { + list.push( + this.createSuggestion( + tag.name, + tag.name + '=', + this.prefix, + `Metadata: ${tag.datatype}`, + 'sym_o_label', + false, + false, + ), + ); + } + } + + return list; + } +} + +export class CompositeFilterProvider implements SuggestionProvider { + constructor(private strategies: SuggestionProvider[]) {} + + getSuggestions(context: SuggestionContext): Suggestion[] { + const results: Suggestion[] = []; + for (const s of this.strategies) { + results.push(...s.getSuggestions(context)); + } + return results; + } +} diff --git a/frontend/src/services/suggestions/search-parser.ts b/frontend/src/services/suggestions/search-parser.ts new file mode 100644 index 000000000..d1253dc08 --- /dev/null +++ b/frontend/src/services/suggestions/search-parser.ts @@ -0,0 +1,74 @@ +export interface SearchToken { + original: string; + key?: string; + value: string; +} + +export interface ParsedSearch { + tokens: SearchToken[]; + freeText: string; +} + +// Matches key:value or key:"value with spaces" +// The keys are dynamic: [a-zA-Z0-9_\-&]+: +const TOKEN_REGEX = /([a-zA-Z0-9_\-&]+:)(?:"([^"]*)"?|([^\s]+))/gi; + +export function parseSearchString(input: string): ParsedSearch { + const tokens: SearchToken[] = []; + let freeText = input; + + // Reset lastIndex + TOKEN_REGEX.lastIndex = 0; + + let match; + while ((match = TOKEN_REGEX.exec(input)) !== null) { + const fullMatch = match[0]; + const keyWithColon = match[1]; + const value = match[2] ?? match[3] ?? ''; + + if (keyWithColon) { + const key = keyWithColon.slice(0, -1).toLowerCase(); // remove colon + tokens.push({ + original: fullMatch, + key, + value, + }); + // Remove from free text + freeText = freeText.replace(fullMatch, ''); + } + } + + // Clean up free text (remove extra spaces) + freeText = freeText.replaceAll(/\s+/g, ' ').trim(); + + return { + tokens, + freeText, + }; +} + +export function validateSearchSyntax( + input: string, + validKeys?: string[], +): string | null { + // Check for unclosed quotes + const quoteCount = (input.match(/"/g) ?? []).length; + if (quoteCount % 2 !== 0) { + const isAtEnd = /(?:^|\s)[a-zA-Z0-9_-]+:"[^"]*$/.test(input); + if (!isAtEnd) { + return 'Unclosed quote detected.'; + } + } + + // Check for empty values e.g. "project: " + const emptyKeywordRegex = /([a-zA-Z0-9_\-&]+:)(\s|$)/gi; + const emptyMatch = emptyKeywordRegex.exec(input); + if (emptyMatch) { + const key = emptyMatch[1]?.replace(':', '') ?? 'unknown'; + if (!validKeys || validKeys.includes(key.toLowerCase())) { + return `Missing value for ${key}.`; + } + } + + return null; +} diff --git a/frontend/src/services/suggestions/suggestion-types.ts b/frontend/src/services/suggestions/suggestion-types.ts new file mode 100644 index 000000000..3f084d3e5 --- /dev/null +++ b/frontend/src/services/suggestions/suggestion-types.ts @@ -0,0 +1,20 @@ +export interface Suggestion { + label: string; + value: string; + prefix: string; + description: string; + icon: string; + appendSpace?: boolean; + disabled?: boolean; +} + +export interface SuggestionContext { + input: string; + // Additional context data (e.g. loaded projects, current permissions) + // can be passed here, specific to the implementer. + data: T; +} + +export interface SuggestionProvider { + getSuggestions(context: SuggestionContext): Suggestion[]; +} From e69336b4eb63762ef5327d1095342b248fc1c96b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cyrill=20P=C3=BCntener?= Date: Thu, 18 Dec 2025 21:41:39 +0100 Subject: [PATCH 41/49] feat: searchable project and mission selection and improvements to search (finally implement #78) --- backend/src/services/file.service.ts | 6 +- frontend/src/components/common/app-select.vue | 50 ++- .../common/autocomplete-dropdown.vue | 43 +-- .../src/components/common/scope-selector.vue | 5 + .../components/common/smart-search-input.vue | 27 +- .../src/components/files/files-filter.vue | 194 ++++++---- .../filter/components/datatype-filter.vue | 64 ++++ .../files/filter/components/date-filter.vue | 254 +++++++++++++ .../filter/components/file-type-filter.vue | 29 ++ .../filter/components/metadata-filter.vue | 26 ++ .../filter/components/project-filter.vue | 39 ++ .../files/filter/components/topic-filter.vue | 90 +++++ .../files/filter/composable-filter-popup.vue | 61 +++ .../files/filter/metadata-filter-builder.vue | 36 +- frontend/src/composables/use-file-filter.ts | 2 +- frontend/src/composables/use-file-search.ts | 161 ++++---- frontend/src/composables/use-filter-parser.ts | 349 ++++++++---------- frontend/src/composables/use-filter-sync.ts | 63 ++++ frontend/src/services/filters/base-filter.ts | 98 +++++ .../src/services/filters/filter-interface.ts | 20 + .../implementations/datatype-filter.ts | 68 ++++ .../filters/implementations/date-filter.ts | 46 +++ .../implementations/file-type-filter.ts | 36 ++ .../implementations/metadata-filter.ts | 132 +++++++ .../filters/implementations/mission-filter.ts | 43 +++ .../filters/implementations/project-filter.ts | 47 +++ .../filters/implementations/topic-filter.ts | 63 ++++ .../services/suggestions/filter-strategies.ts | 266 ------------- .../suggestions/strategies/base-strategy.ts | 29 ++ .../strategies/composite-filter-provider.ts | 17 + .../strategies/keyword-strategy.ts | 68 ++++ .../services/suggestions/suggestion-types.ts | 16 + 32 files changed, 1785 insertions(+), 663 deletions(-) create mode 100644 frontend/src/components/files/filter/components/datatype-filter.vue create mode 100644 frontend/src/components/files/filter/components/date-filter.vue create mode 100644 frontend/src/components/files/filter/components/file-type-filter.vue create mode 100644 frontend/src/components/files/filter/components/metadata-filter.vue create mode 100644 frontend/src/components/files/filter/components/project-filter.vue create mode 100644 frontend/src/components/files/filter/components/topic-filter.vue create mode 100644 frontend/src/components/files/filter/composable-filter-popup.vue create mode 100644 frontend/src/composables/use-filter-sync.ts create mode 100644 frontend/src/services/filters/base-filter.ts create mode 100644 frontend/src/services/filters/filter-interface.ts create mode 100644 frontend/src/services/filters/implementations/datatype-filter.ts create mode 100644 frontend/src/services/filters/implementations/date-filter.ts create mode 100644 frontend/src/services/filters/implementations/file-type-filter.ts create mode 100644 frontend/src/services/filters/implementations/metadata-filter.ts create mode 100644 frontend/src/services/filters/implementations/mission-filter.ts create mode 100644 frontend/src/services/filters/implementations/project-filter.ts create mode 100644 frontend/src/services/filters/implementations/topic-filter.ts delete mode 100644 frontend/src/services/suggestions/filter-strategies.ts create mode 100644 frontend/src/services/suggestions/strategies/base-strategy.ts create mode 100644 frontend/src/services/suggestions/strategies/composite-filter-provider.ts create mode 100644 frontend/src/services/suggestions/strategies/keyword-strategy.ts diff --git a/backend/src/services/file.service.ts b/backend/src/services/file.service.ts index 40908d9fa..ae5940e86 100644 --- a/backend/src/services/file.service.ts +++ b/backend/src/services/file.service.ts @@ -682,6 +682,7 @@ export class FileService implements OnModuleInit { const tagWhereClauses: string[] = []; // eslint-disable-next-line @typescript-eslint/no-explicit-any const tagParameters: Record = {}; + const validTagNames = new Set(); let validTagCount = 0; for (const uuid of tagTypeUUIDs) { @@ -719,6 +720,7 @@ export class FileService implements OnModuleInit { tagParameters[valueParameter] = processedValue; validTagCount++; + validTagNames.add(tagtype.name); } if (validTagCount === 0) { @@ -734,8 +736,8 @@ export class FileService implements OnModuleInit { tagParameters, ); - query.having('COUNT(DISTINCT tagtype.uuid) = :tagCount', { - tagCount: validTagCount, + query.having('COUNT(DISTINCT tagtype.name) = :tagCount', { + tagCount: validTagNames.size, }); } diff --git a/frontend/src/components/common/app-select.vue b/frontend/src/components/common/app-select.vue index 6ef213b19..88de67892 100644 --- a/frontend/src/components/common/app-select.vue +++ b/frontend/src/components/common/app-select.vue @@ -13,13 +13,15 @@ dense map-options emit-value - :options="options" + :options="filteredOptions" :option-label="optionLabel" :option-value="optionValue" + :use-input="searchable && !model" :bg-color=" bgColor ?? ($attrs.readonly || $attrs.disable ? 'grey-2' : 'white') " + @filter="filterFunction" > - - - - - - - -
- + + +
- - - - - - - - + +
+ + + +
+ +
@@ -110,17 +85,17 @@ selected
-