diff --git a/.github/workflows/build-sample-actions.yml b/.github/workflows/build-sample-actions.yml index 1f753493f..add9428a4 100644 --- a/.github/workflows/build-sample-actions.yml +++ b/.github/workflows/build-sample-actions.yml @@ -35,10 +35,10 @@ jobs: uses: actions/checkout@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Login to Docker Hub - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} @@ -47,7 +47,11 @@ jobs: id: tag run: | if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then - echo "tag=dev" >> $GITHUB_OUTPUT + if [ "${{ matrix.type }}" == "prod" ] && [ "${{ github.ref }}" == "refs/heads/main" ]; then + echo "tag=latest" >> $GITHUB_OUTPUT + else + echo "tag=dev" >> $GITHUB_OUTPUT + fi echo "kleinkram_version=${{ inputs.kleinkram_version }}" >> $GITHUB_OUTPUT elif [ "${{ matrix.type }}" == "dev" ]; then echo "tag=dev" >> $GITHUB_OUTPUT @@ -58,7 +62,7 @@ jobs: fi - name: Build and push - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v6 with: context: examples/kleinkram-actions/${{ matrix.action }} push: true diff --git a/.github/workflows/check-migrations.yml b/.github/workflows/check-migrations.yml index eb9008639..ea6d96b84 100644 --- a/.github/workflows/check-migrations.yml +++ b/.github/workflows/check-migrations.yml @@ -13,9 +13,7 @@ jobs: uses: actions/checkout@v4 - name: Install pnpm - uses: pnpm/action-setup@v2 - with: - version: 10 + uses: pnpm/action-setup@v4 - name: Setup Node.js uses: actions/setup-node@v4 diff --git a/.github/workflows/draft-pdf.yml b/.github/workflows/draft-pdf.yml index 0591e8009..28a16d6e7 100644 --- a/.github/workflows/draft-pdf.yml +++ b/.github/workflows/draft-pdf.yml @@ -18,7 +18,7 @@ jobs: # This should be the path to the paper within your repo. paper-path: openjournals/paper.md - name: Upload - uses: actions/upload-artifact@v5 + uses: actions/upload-artifact@v6 with: name: paper # This is the output path where Pandoc will write the compiled diff --git a/backend/migration/migrations/1765827049888-action_template_floats.ts b/backend/migration/migrations/1765827049888-action_template_floats.ts new file mode 100644 index 000000000..494b1f316 --- /dev/null +++ b/backend/migration/migrations/1765827049888-action_template_floats.ts @@ -0,0 +1,80 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class ActionTemplateFloats1765827049888 implements MigrationInterface { + name = 'ActionTemplateFloats1765827049888'; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `ALTER TABLE "action_template" ALTER COLUMN "cpuCores" TYPE double precision`, + ); + await queryRunner.query( + `ALTER TABLE "action_template" ALTER COLUMN "cpuMemory" TYPE double precision`, + ); + await queryRunner.query( + `ALTER TABLE "action_template" ALTER COLUMN "gpuMemory" TYPE double precision`, + ); + await queryRunner.query( + `ALTER TABLE "action_template" ALTER COLUMN "maxRuntime" TYPE double precision`, + ); + + // Worker memory floats + // cpuMemory + await queryRunner.query(` + DO $$ + BEGIN + IF EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name='worker' AND column_name='cpuMemory') THEN + ALTER TABLE "worker" ALTER COLUMN "cpuMemory" TYPE double precision; + ELSE + ALTER TABLE "worker" ADD COLUMN "cpuMemory" double precision NOT NULL DEFAULT 512; + END IF; + END $$; + `); + + // gpuMemory + await queryRunner.query(` + DO $$ + BEGIN + IF EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name='worker' AND column_name='gpuMemory') THEN + ALTER TABLE "worker" ALTER COLUMN "gpuMemory" TYPE double precision; + ELSE + ALTER TABLE "worker" ADD COLUMN "gpuMemory" double precision NOT NULL DEFAULT -1; + END IF; + END $$; + `); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `ALTER TABLE "action_template" DROP COLUMN "maxRuntime"`, + ); + await queryRunner.query( + `ALTER TABLE "action_template" ADD "maxRuntime" integer NOT NULL`, + ); + await queryRunner.query( + `ALTER TABLE "action_template" DROP COLUMN "gpuMemory"`, + ); + await queryRunner.query( + `ALTER TABLE "action_template" ADD "gpuMemory" integer NOT NULL`, + ); + await queryRunner.query( + `ALTER TABLE "action_template" DROP COLUMN "cpuMemory"`, + ); + await queryRunner.query( + `ALTER TABLE "action_template" ADD "cpuMemory" integer NOT NULL`, + ); + await queryRunner.query( + `ALTER TABLE "action_template" DROP COLUMN "cpuCores"`, + ); + await queryRunner.query( + `ALTER TABLE "action_template" ADD "cpuCores" integer NOT NULL`, + ); + + // Revert worker memory to integer (might fail if data is not integer, but this is best effort reversion) + await queryRunner.query( + `ALTER TABLE "worker" ALTER COLUMN "cpuMemory" TYPE integer`, + ); + await queryRunner.query( + `ALTER TABLE "worker" ALTER COLUMN "gpuMemory" TYPE integer`, + ); + } +} diff --git a/backend/migration/migrations/1766151003272-add-indexes.ts b/backend/migration/migrations/1766151003272-add-indexes.ts new file mode 100644 index 000000000..2a0fadb17 --- /dev/null +++ b/backend/migration/migrations/1766151003272-add-indexes.ts @@ -0,0 +1,137 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class AddIndexes1766151003272 implements MigrationInterface { + name = 'AddIndexes1766151003272'; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `CREATE INDEX "IDX_2b0c110b36a490a5458d253911" ON "base_entity" ("deletedAt") `, + ); + await queryRunner.query( + `CREATE INDEX "IDX_2cabb849760babe66490f024e1" ON "account" ("deletedAt") `, + ); + await queryRunner.query( + `CREATE INDEX "IDX_aa2316638d2492c7a5e8a4eccd" ON "group_membership" ("deletedAt") `, + ); + await queryRunner.query( + `CREATE INDEX "IDX_82c1e9e16eb675a12a17d17f1f" ON "topic" ("deletedAt") `, + ); + await queryRunner.query( + `CREATE INDEX "IDX_45208f047f6742a3067fd1d49c" ON "file_entity" ("deletedAt") `, + ); + await queryRunner.query( + `CREATE INDEX "IDX_cff24fb6f37558b50778134d28" ON "file_entity" ("missionUuid") `, + ); + await queryRunner.query( + `CREATE INDEX "IDX_294e1e9ea20bb504f382a0d356" ON "category" ("deletedAt") `, + ); + await queryRunner.query( + `CREATE INDEX "IDX_16daeaa9a7c1eeeb4468048d21" ON "tag" ("deletedAt") `, + ); + await queryRunner.query( + `CREATE INDEX "IDX_8102188b489ccc9e092d7c177b" ON "tag_type" ("deletedAt") `, + ); + await queryRunner.query( + `CREATE INDEX "IDX_30310b331b5091f60585828d75" ON "project" ("deletedAt") `, + ); + await queryRunner.query( + `CREATE INDEX "IDX_82e2e7d6f42e4aaac0ed332a92" ON "project_access" ("deletedAt") `, + ); + await queryRunner.query( + `CREATE INDEX "IDX_01369bae653efc42ddf9a572fb" ON "access_group" ("deletedAt") `, + ); + await queryRunner.query( + `CREATE INDEX "IDX_98fc2b4a5cf77e0786d1a43cfa" ON "mission_access" ("deletedAt") `, + ); + await queryRunner.query( + `CREATE INDEX "IDX_7d27162eb31a34e1a5607ec135" ON "ingestion_job" ("deletedAt") `, + ); + await queryRunner.query( + `CREATE INDEX "IDX_39e216d9a16251e209621e8259" ON "mission" ("deletedAt") `, + ); + await queryRunner.query( + `CREATE INDEX "IDX_6d71c0142267fbeaba6cae0e5d" ON "mission" ("projectUuid") `, + ); + await queryRunner.query( + `CREATE INDEX "IDX_2fb2f37b515e0d6a6dc4a98888" ON "apikey" ("deletedAt") `, + ); + await queryRunner.query( + `CREATE INDEX "IDX_92f09bd6964a57bb87891a2acf" ON "user" ("deletedAt") `, + ); + await queryRunner.query( + `CREATE INDEX "IDX_c28b07d4ad69e00608d9453324" ON "action_template" ("deletedAt") `, + ); + await queryRunner.query( + `CREATE INDEX "IDX_945a53fd3adb2200313ebb0766" ON "action" ("deletedAt") `, + ); + await queryRunner.query( + `CREATE INDEX "IDX_5896a6451354b0fef7e7759dbe" ON "worker" ("deletedAt") `, + ); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `DROP INDEX "public"."IDX_5896a6451354b0fef7e7759dbe"`, + ); + await queryRunner.query( + `DROP INDEX "public"."IDX_945a53fd3adb2200313ebb0766"`, + ); + await queryRunner.query( + `DROP INDEX "public"."IDX_c28b07d4ad69e00608d9453324"`, + ); + await queryRunner.query( + `DROP INDEX "public"."IDX_92f09bd6964a57bb87891a2acf"`, + ); + await queryRunner.query( + `DROP INDEX "public"."IDX_2fb2f37b515e0d6a6dc4a98888"`, + ); + await queryRunner.query( + `DROP INDEX "public"."IDX_6d71c0142267fbeaba6cae0e5d"`, + ); + await queryRunner.query( + `DROP INDEX "public"."IDX_39e216d9a16251e209621e8259"`, + ); + await queryRunner.query( + `DROP INDEX "public"."IDX_7d27162eb31a34e1a5607ec135"`, + ); + await queryRunner.query( + `DROP INDEX "public"."IDX_98fc2b4a5cf77e0786d1a43cfa"`, + ); + await queryRunner.query( + `DROP INDEX "public"."IDX_01369bae653efc42ddf9a572fb"`, + ); + await queryRunner.query( + `DROP INDEX "public"."IDX_82e2e7d6f42e4aaac0ed332a92"`, + ); + await queryRunner.query( + `DROP INDEX "public"."IDX_30310b331b5091f60585828d75"`, + ); + await queryRunner.query( + `DROP INDEX "public"."IDX_8102188b489ccc9e092d7c177b"`, + ); + await queryRunner.query( + `DROP INDEX "public"."IDX_16daeaa9a7c1eeeb4468048d21"`, + ); + await queryRunner.query( + `DROP INDEX "public"."IDX_294e1e9ea20bb504f382a0d356"`, + ); + await queryRunner.query( + `DROP INDEX "public"."IDX_cff24fb6f37558b50778134d28"`, + ); + await queryRunner.query( + `DROP INDEX "public"."IDX_45208f047f6742a3067fd1d49c"`, + ); + await queryRunner.query( + `DROP INDEX "public"."IDX_82c1e9e16eb675a12a17d17f1f"`, + ); + await queryRunner.query( + `DROP INDEX "public"."IDX_aa2316638d2492c7a5e8a4eccd"`, + ); + await queryRunner.query( + `DROP INDEX "public"."IDX_2cabb849760babe66490f024e1"`, + ); + await queryRunner.query( + `DROP INDEX "public"."IDX_2b0c110b36a490a5458d253911"`, + ); + } +} diff --git a/backend/package.json b/backend/package.json index ec78fccb5..ef48179ca 100644 --- a/backend/package.json +++ b/backend/package.json @@ -1,6 +1,6 @@ { "name": "kleinkram-backend", - "version": "0.56.0", + "version": "0.57.0", "description": "", "author": "", "license": "MIT", @@ -26,16 +26,16 @@ "@kleinkram/backend-common": "workspace:*", "@kleinkram/shared": "workspace:*", "@kleinkram/validation": "workspace:*", - "@nestjs/common": "^10.4.20", + "@nestjs/common": "^11.1.9", "@nestjs/config": "^4.0.2", - "@nestjs/core": "^10.4.5", - "@nestjs/jwt": "^11.0.1", - "@nestjs/mapped-types": "^2.0.5", + "@nestjs/core": "^11.1.9", + "@nestjs/jwt": "^11.0.2", + "@nestjs/mapped-types": "^2.1.0", "@nestjs/passport": "^11.0.5", - "@nestjs/platform-express": "^10.4.20", - "@nestjs/schedule": "^6.0.1", - "@nestjs/swagger": "^8.0.1", - "@nestjs/typeorm": "^10.0.2", + "@nestjs/platform-express": "^11.1.9", + "@nestjs/schedule": "^6.1.0", + "@nestjs/swagger": "^11.2.3", + "@nestjs/typeorm": "^11.0.0", "@opentelemetry/api": "^1.9.0", "@opentelemetry/exporter-prometheus": "^0.208.0", "@opentelemetry/exporter-trace-otlp-http": "^0.208.0", @@ -73,22 +73,23 @@ "reflect-metadata": "^0.2.0", "rxjs": "^7.8.2", "typeorm": "^0.3.27", + "typeorm-extension": "^3.7.3", "uuid": "^13.0.0", "winston": "^3.18.3", "winston-loki": "^6.0.7" }, "devDependencies": { - "@aws-sdk/client-s3": "3.726.1", + "@aws-sdk/client-s3": "3.952.0", "@jest/globals": "^30.2.0", - "@nestjs/cli": "^11.0.10", + "@nestjs/cli": "^11.0.14", "@nestjs/schematics": "^11.0.9", - "@nestjs/testing": "^10.0.0", + "@nestjs/testing": "^11.1.9", "@swc/core": "^1.13.5", "@swc/jest": "^0.2.39", "@types/cookie-parser": "^1.4.9", "@types/dotenv": "^8.2.3", "@types/express": "^5.0.3", - "@types/jest": "^29.5.2", + "@types/jest": "^30.0.0", "@types/jsonwebtoken": "^9.0.10", "@types/multer": "^2.0.0", "@types/node": "^24.10.1", @@ -101,10 +102,10 @@ "@typescript-eslint/eslint-plugin": "^8.32.1", "@typescript-eslint/parser": "^8.25.0", "concurrently": "^8.2.2", - "eslint": "^9.37.0", + "eslint": "^9.39.2", "eslint-config-prettier": "^10.1.8", "eslint-plugin-prettier": "^5.5.4", - "jest": "^29.7.0", + "jest": "^30.2.0", "jest-junit": "^16.0.0", "node-loader": "^2.0.0", "prettier": "^3.6.2", diff --git a/backend/scripts/generate-openapi.ts b/backend/scripts/generate-openapi.ts index bfc368531..7894b5733 100644 --- a/backend/scripts/generate-openapi.ts +++ b/backend/scripts/generate-openapi.ts @@ -211,7 +211,7 @@ aside: false modules.push({ name: name, path: `/${String(controllerPath)}`, - link: `generated/${name}.html`, + link: `generated/${name}`, description: `Docs for ${name} module`, }); } @@ -230,23 +230,39 @@ aside: false process.exit(0); } -function saveEndpointsAsJson(app: INestApplication, filename: string): void { - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment - const server = app.getHttpServer(); +interface ExpressLayer { + route?: { + path: string; + stack: { method: string }[]; + }; +} - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access - const endpoints = server._events.request._router.stack +interface ExpressRouter { + stack: ExpressLayer[]; +} - // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-return, @typescript-eslint/no-unsafe-member-access - .filter((r: any) => r.route) +interface HttpServerWithRouter { + _events?: { + request?: { + _router?: ExpressRouter; + }; + }; +} - // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-explicit-any - .map((r: any) => ({ - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access - url: r.route.path, +function saveEndpointsAsJson(app: INestApplication, filename: string): void { + const server = app.getHttpServer() as HttpServerWithRouter; + + const router = server._events?.request?._router; + + if (!router) { + return; + } - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access - method: r.route.stack[0].method, + const endpoints = router.stack + .filter((r) => r.route) + .map((r) => ({ + url: r.route?.path, + method: r.route?.stack[0]?.method, })); try { diff --git a/backend/src/app-version.ts b/backend/src/app-version.ts index 97bc87ee5..e3c5c1e02 100644 --- a/backend/src/app-version.ts +++ b/backend/src/app-version.ts @@ -6,7 +6,7 @@ interface PackageJson { description?: string; } -const path = '/usr/src/app/backend/package.json'; +const path = '/app/backend/package.json'; function readFileIfExists(filePath: string): PackageJson | null { try { diff --git a/backend/src/app.module.ts b/backend/src/app.module.ts index e68fe415a..7fbc5734a 100644 --- a/backend/src/app.module.ts +++ b/backend/src/app.module.ts @@ -105,8 +105,8 @@ export class AppModule implements NestModule { consumer .apply(VersionCheckerMiddlewareService) .exclude( - '/auth/(.*)', // excludes auth endpoints - '/integrations/(.*)', // excludes integration endpoints + '/auth/{*path}', // excludes auth endpoints + '/integrations/{*path}', // excludes integration endpoints '/api/health', // excludes health check ) .forRoutes('*'); diff --git a/backend/src/endpoints/file/file.controller.ts b/backend/src/endpoints/file/file.controller.ts index 176a4758b..068eff720 100644 --- a/backend/src/endpoints/file/file.controller.ts +++ b/backend/src/endpoints/file/file.controller.ts @@ -31,8 +31,8 @@ import { FileQueryDto, FileQueueEntriesDto, FileQueueEntryDto, - FileWithTopicDto, FilesDto, + FileWithTopicDto, FoxgloveLinkResponseDto, IsUploadingDto, MoveFilesResponseDto, @@ -158,6 +158,12 @@ export class FileController { endDate: Date | undefined, @QueryOptionalString('topics', 'Name of Topics (coma separated)') topics: string, + @QueryOptionalString( + 'messageDatatypes', + 'Message datatypes to filter by (coma separated). If multiple are given, ' + + 'files containing any of the datatypes are returned (OR).', + ) + messageDatatypes: string, @QueryOptionalString( 'fileTypes', 'File types to filter by (coma separated)', @@ -193,6 +199,7 @@ export class FileController { startDate, endDate, topics, + messageDatatypes, categories, matchAllTopics, fileTypes, @@ -571,6 +578,11 @@ export class FileController { @AddUser() user: AuthHeader, ): Promise { const date = new Date(startDate); + if (Number.isNaN(date.getTime())) { + throw new BadRequestException( + `Invalid startDate: "${startDate}". Expected ISO 8601 format.`, + ); + } const data = (await this.queueService.active( date, diff --git a/backend/src/endpoints/topic/topic.controller.ts b/backend/src/endpoints/topic/topic.controller.ts index b05906dc2..3670df22e 100644 --- a/backend/src/endpoints/topic/topic.controller.ts +++ b/backend/src/endpoints/topic/topic.controller.ts @@ -1,7 +1,7 @@ import { ApiOkResponse } from '@/decorators'; import { TopicService } from '@/services/topic.service'; import { QuerySkip, QueryTake } from '@/validation/query-decorators'; -import { TopicNamesDto, TopicsDto } from '@kleinkram/api-dto'; +import { TopicNamesDto, TopicsDto, TopicTypesDto } from '@kleinkram/api-dto'; import { Controller, Get } from '@nestjs/common'; import { AddUser, AuthHeader } from '../auth/parameter-decorator'; import { LoggedIn } from '../auth/roles.decorator'; @@ -33,4 +33,14 @@ export class TopicController { async allNames(@AddUser() user: AuthHeader): Promise { return await this.topicService.findAllNames(user.user.uuid); } + + @Get('types') + @LoggedIn() + @ApiOkResponse({ + description: 'Get all topic types', + type: TopicTypesDto, + }) + async allTypes(@AddUser() user: AuthHeader): Promise { + return await this.topicService.findAllTypes(user.user.uuid); + } } diff --git a/backend/src/main.ts b/backend/src/main.ts index 8024166ab..ab61100b1 100644 --- a/backend/src/main.ts +++ b/backend/src/main.ts @@ -16,23 +16,39 @@ import { GlobalErrorFilter } from './routing/filters/global-error-filter'; import { GlobalResponseValidationInterceptor } from './routing/interceptors/output-validation'; import { AddVersionInterceptor } from './routing/interceptors/version-injector'; -function saveEndpointsAsJson(app: INestApplication, filename: string): void { - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment - const server = app.getHttpServer(); +interface ExpressLayer { + route?: { + path: string; + stack: { method: string }[]; + }; +} + +interface ExpressRouter { + stack: ExpressLayer[]; +} - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access - const endpoints = server._events.request._router.stack +interface HttpServerWithRouter { + _events?: { + request?: { + _router?: ExpressRouter; + }; + }; +} + +function saveEndpointsAsJson(app: INestApplication, filename: string): void { + const server = app.getHttpServer() as HttpServerWithRouter; - // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-return, @typescript-eslint/no-unsafe-member-access - .filter((r: any) => r.route) + const router = server._events?.request?._router; - // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-explicit-any - .map((r: any) => ({ - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access - url: r.route.path, + if (!router) { + return; + } - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access - method: r.route.stack[0].method, + const endpoints = router.stack + .filter((r) => r.route) + .map((r) => ({ + url: r.route?.path, + method: r.route?.stack[0]?.method, })); try { diff --git a/backend/src/services/file.service.ts b/backend/src/services/file.service.ts index 8556af309..ffce24b4e 100644 --- a/backend/src/services/file.service.ts +++ b/backend/src/services/file.service.ts @@ -1,5 +1,19 @@ -import { SortOrder, UpdateFile } from '@kleinkram/api-dto'; +import { fileEntityToDto, fileEntityToDtoWithTopic } from '@/serialization'; +import { + FileEventsDto, + FileExistsResponseDto, + FilesDto, + FileWithTopicDto, + SortOrder, + StorageOverviewDto, + TemporaryFileAccessesDto, + UpdateFile, +} from '@kleinkram/api-dto'; +import { FileAuditService } from '@kleinkram/backend-common/audit/file-audit.service'; +import { redis } from '@kleinkram/backend-common/consts'; import { ActionEntity } from '@kleinkram/backend-common/entities/action/action.entity'; +import { CategoryEntity } from '@kleinkram/backend-common/entities/category/category.entity'; +import { FileEventEntity } from '@kleinkram/backend-common/entities/file/file-event.entity'; import { FileEntity } from '@kleinkram/backend-common/entities/file/file.entity'; import { IngestionJobEntity } from '@kleinkram/backend-common/entities/file/ingestion-job.entity'; import { MissionEntity } from '@kleinkram/backend-common/entities/mission/mission.entity'; @@ -31,7 +45,6 @@ import { Repository, SelectQueryBuilder, } from 'typeorm'; -import { fileEntityToDto, fileEntityToDtoWithTopic } from '../serialization'; import { addFileFilters, addMissionFilters, @@ -40,19 +53,6 @@ import { convertGlobToLikePattern, } from './utilities'; -import { - FileEventsDto, - FileExistsResponseDto, - FilesDto, - FileWithTopicDto, - StorageOverviewDto, - TemporaryFileAccessesDto, -} from '@kleinkram/api-dto'; -import { FileAuditService } from '@kleinkram/backend-common/audit/file-audit.service'; -import { redis } from '@kleinkram/backend-common/consts'; -import { CategoryEntity } from '@kleinkram/backend-common/entities/category/category.entity'; -import { FileEventEntity } from '@kleinkram/backend-common/entities/file/file-event.entity'; - import { TagTypeEntity } from '@kleinkram/backend-common/entities/tagType/tag-type.entity'; import { UserEntity } from '@kleinkram/backend-common/entities/user/user.entity'; import { StorageService } from '@kleinkram/backend-common/modules/storage/storage.service'; @@ -60,13 +60,13 @@ import Queue from 'bull'; // @ts-ignore import Credentials from 'minio/dist/main/Credentials'; // @ts-ignore -import { BucketItem } from 'minio/dist/main/internal/type'; import { addAccessConstraints, addAccessConstraintsToFileQuery, addAccessConstraintsToMissionQuery, addAccessConstraintsToProjectQuery, -} from '../endpoints/auth/auth-helper'; +} from '@/endpoints/auth/auth-helper'; +import { BucketItem } from 'minio/dist/main/internal/type'; import logger from '../logger'; const FIND_MANY_SORT_KEYS = { @@ -371,6 +371,7 @@ export class FileService implements OnModuleInit { startDate: Date | undefined, endDate: Date | undefined, topics: string, + messageDatatype: string, categories: string, matchAllTopics: boolean, fileTypes: string, @@ -404,9 +405,22 @@ export class FileService implements OnModuleInit { // Apply simple filters if (fileName) { logger.debug(`Filtering files by filename: ${fileName}`); - idQuery.andWhere('file.filename LIKE :fileName', { - fileName: `%${fileName}%`, - }); + const tokens = fileName.trim().split(/\s+/); + + if (tokens.length > 0) { + idQuery.andWhere( + new Brackets((qb) => { + for (const [index, token] of tokens.entries()) { + qb.andWhere( + `file.filename ILIKE :fileName_${String(index)}`, + { + [`fileName_${String(index)}`]: `%${token}%`, + }, + ); + } + }), + ); + } } if (projectUUID) { @@ -419,19 +433,22 @@ export class FileService implements OnModuleInit { idQuery.andWhere('mission.uuid = :missionUUID', { missionUUID }); } - if (startDate && endDate) { + if (startDate) { logger.debug( - `Filtering files by date range: ${startDate.toString()} - ${endDate.toString()}`, + `Filtering files by start date: ${startDate.toString()}`, ); - idQuery.andWhere('file.date BETWEEN :startDate AND :endDate', { - startDate, - endDate, - }); + idQuery.andWhere('file.date >= :startDate', { startDate }); + } + + if (endDate) { + logger.debug(`Filtering files by end date: ${endDate.toString()}`); + idQuery.andWhere('file.date <= :endDate', { endDate }); } // Apply complex filters via helper methods this._applyFileTypeFilter(idQuery, fileTypes); this._applyTopicFilter(idQuery, topics, matchAllTopics); + this._applyMessageDatatypeFilter(idQuery, messageDatatype); // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition if (health) { @@ -610,6 +627,30 @@ export class FileService implements OnModuleInit { } } + /** + * Applies message datatype filtering to the query. + */ + private _applyMessageDatatypeFilter( + query: SelectQueryBuilder, + messageDatatype: string, + ): void { + if (!messageDatatype) { + return; + } + + const splitMessageDatatype = messageDatatype + .split(',') + .filter((t) => t.length > 0); + if (splitMessageDatatype.length === 0) { + return; + } + + // Filter files that have *at least one* of the message datatypes + query.andWhere('topic.type IN (:...splitMessageDatatype)', { + splitMessageDatatype, + }); + } + /** * Applies tag filtering to the query. * This is the most complex filter, requiring a 'relational division' query. @@ -643,6 +684,7 @@ export class FileService implements OnModuleInit { const tagWhereClauses: string[] = []; // eslint-disable-next-line @typescript-eslint/no-explicit-any const tagParameters: Record = {}; + const validTagNames = new Set(); let validTagCount = 0; for (const uuid of tagTypeUUIDs) { @@ -680,6 +722,7 @@ export class FileService implements OnModuleInit { tagParameters[valueParameter] = processedValue; validTagCount++; + validTagNames.add(tagtype.name); } if (validTagCount === 0) { @@ -695,8 +738,8 @@ export class FileService implements OnModuleInit { tagParameters, ); - query.having('COUNT(DISTINCT tagtype.uuid) = :tagCount', { - tagCount: validTagCount, + query.having('COUNT(DISTINCT tagtype.name) = :tagCount', { + tagCount: validTagNames.size, }); } @@ -983,6 +1026,9 @@ export class FileService implements OnModuleInit { // eslint-disable-next-line @typescript-eslint/naming-convention * @param uuid The unique identifier of the file * @param expires Whether the download link should expire + * @param preview_only + * @param actor + * @param action */ async generateDownload( uuid: string, @@ -1117,6 +1163,7 @@ export class FileService implements OnModuleInit { * * @param uuid The unique identifier of the file * @param actor + * @param action */ async deleteFile( uuid: string, @@ -1214,6 +1261,8 @@ export class FileService implements OnModuleInit { * @param filenames list of filenames to upload * @param missionUUID the mission to upload the files to * @param userUUID the user that is uploading the files + * @param action + * @param uploadSource */ async getTemporaryAccess( filenames: string[], diff --git a/backend/src/services/mission.service.ts b/backend/src/services/mission.service.ts index 9f3a6496b..1b19cd2fa 100644 --- a/backend/src/services/mission.service.ts +++ b/backend/src/services/mission.service.ts @@ -304,9 +304,12 @@ export class MissionService { .skip(skip); if (search) { - query.andWhere('mission.name ILIKE :search', { - search: `%${search}%`, - }); + const tokens = search.trim().split(/\s+/); + for (const [index, token] of tokens.entries()) { + query.andWhere(`mission.name ILIKE :search_${String(index)}`, { + [`search_${String(index)}`]: `%${token}%`, + }); + } } if (sortBy) { query.orderBy(`mission.${sortBy}`, sortDirection); @@ -355,9 +358,12 @@ export class MissionService { .addGroupBy('tagType.uuid'); if (search) { - query.andWhere('mission.name ILIKE :search', { - search: `%${search}%`, - }); + const tokens = search.trim().split(/\s+/); + for (const [index, token] of tokens.entries()) { + query.andWhere(`mission.name ILIKE :search_${String(index)}`, { + [`search_${String(index)}`]: `%${token}%`, + }); + } } if (sortBy) { query.orderBy(`mission.${sortBy}`, sortDirection); diff --git a/backend/src/services/project.service.ts b/backend/src/services/project.service.ts index b328364fc..2137cf611 100644 --- a/backend/src/services/project.service.ts +++ b/backend/src/services/project.service.ts @@ -81,6 +81,35 @@ export class ProjectService { this.config = config; } + private async _getProjectSizes( + projectUuids: string[], + ): Promise> { + if (projectUuids.length === 0) { + return new Map(); + } + + const rawResults = await this.projectRepository + .createQueryBuilder('project') + .select('project.uuid', 'projectUuid') + .addSelect('COALESCE(SUM(file.size), 0)', 'totalSize') + .leftJoin( + 'project.missions', + 'mission', + 'mission.deletedAt IS NULL', + ) + .leftJoin('mission.files', 'file', 'file.deletedAt IS NULL') + .where('project.uuid IN (:...projectUuids)', { projectUuids }) + .groupBy('project.uuid') + .getRawMany<{ projectUuid: string; totalSize: string }>(); + + const sizeMap = new Map(); + for (const raw of rawResults) { + const size = Number.parseInt(raw.totalSize) || 0; + sizeMap.set(raw.projectUuid, size); + } + return sizeMap; + } + async findMany( projectUuids: string[], projectPatterns: string[], @@ -117,10 +146,15 @@ export class ProjectService { query.skip(skip).take(take); const [projects, count] = await query.getManyAndCount(); + const foundProjectUuids = projects.map((p) => p.uuid); + const sizes = await this._getProjectSizes(foundProjectUuids); + return { - data: projects.map((element) => - projectEntityToDtoWithMissionCountAndTags(element), - ), + data: projects.map((element) => { + const dto = projectEntityToDtoWithMissionCountAndTags(element); + dto.size = sizes.get(element.uuid) ?? 0; + return dto; + }), count, skip, take, @@ -149,7 +183,10 @@ export class ProjectService { missionPromise, missionCountPromise, ]); - return projectEntityToDtoWithRequiredTags(mission, missionCount); + const sizes = await this._getProjectSizes([uuid]); + const dto = projectEntityToDtoWithRequiredTags(mission, missionCount); + dto.size = sizes.get(uuid) ?? 0; + return dto; } async getRecentProjects( diff --git a/backend/src/services/topic.service.ts b/backend/src/services/topic.service.ts index 4dfe3fc27..412172cb2 100644 --- a/backend/src/services/topic.service.ts +++ b/backend/src/services/topic.service.ts @@ -1,6 +1,6 @@ import { addAccessConstraints } from '@/endpoints/auth/auth-helper'; import { topicEntityToDto } from '@/serialization'; -import { TopicNamesDto, TopicsDto } from '@kleinkram/api-dto'; +import { TopicNamesDto, TopicsDto, TopicTypesDto } from '@kleinkram/api-dto'; import { TopicEntity } from '@kleinkram/backend-common/entities/topic/topic.entity'; import { UserEntity } from '@kleinkram/backend-common/entities/user/user.entity'; import { UserRole } from '@kleinkram/shared'; @@ -51,6 +51,38 @@ export class TopicService { }; } + async findAllTypes(userUuid: string): Promise { + const baseQuery = this.topicRepository + .createQueryBuilder('topic') + .select('DISTINCT topic.type', 'type') + .orderBy('type'); + + const user = await this.userRepository.findOneOrFail({ + where: { uuid: userUuid }, + }); + + const topicsQuery = + user.role === UserRole.ADMIN + ? baseQuery + : addAccessConstraints( + baseQuery + .leftJoin('topic.file', 'file') + .leftJoin('file.mission', 'mission') + .leftJoin('mission.project', 'project'), + userUuid, + ); + + const topics = await topicsQuery.clone().getRawMany(); + const count = await topicsQuery.getCount(); + + return { + count, + data: topics.map((topic: TopicEntity) => topic.type), + take: count, + skip: 0, + }; + } + async findAll( userUUID: string, skip: number, diff --git a/cli/README.md b/cli/README.md index 96f048daf..68d865bbb 100644 --- a/cli/README.md +++ b/cli/README.md @@ -50,7 +50,7 @@ Instead of downloading files from a specified mission you can download arbitrary klein download --dest out *id1* *id2* *id3* ``` -For more information consult the [documentation](https://docs.datasets.leggedrobotics.com/usage/python/getting-started.html). +For more information consult the [documentation](https://docs.datasets.leggedrobotics.com//usage/python/setup). ## Development diff --git a/cli/kleinkram/cli/app.py b/cli/kleinkram/cli/app.py index c467df6fe..5712359d1 100644 --- a/cli/kleinkram/cli/app.py +++ b/cli/kleinkram/cli/app.py @@ -61,7 +61,7 @@ The Kleinkram CLI is a command line interface for Kleinkram. For a list of available commands, run `klein --help` or visit \ -https://docs.datasets.leggedrobotics.com/usage/python/getting-started.html \ +https://docs.datasets.leggedrobotics.com/usage/python/setup \ for more information. """ @@ -193,7 +193,7 @@ def _version_callback(value: bool) -> None: raise typer.Exit() -def check_version_compatiblity() -> None: +def check_version_compatibility() -> None: cli_version = get_supported_api_version() api_version = _get_api_version() api_vers_str = ".".join(map(str, api_version)) @@ -205,10 +205,16 @@ def check_version_compatiblity() -> None: ) if cli_version[1] != api_version[1]: - msg = f"You are using an outdated CLI version ({__version__}). " - msg += f"Please consider upgrading the CLI to version {api_vers_str}." - Console(file=sys.stderr).print(msg, style="red") - logger.warning(msg) + if cli_version < api_version: + msg = f"You are using an outdated CLI version ({__version__}). " + msg += f"Please consider upgrading the CLI to version {api_vers_str}." + Console(file=sys.stderr).print(msg, style="red") + logger.warning(msg) + elif cli_version > api_version: + msg = f"You are using a CLI version ({__version__}) that is newer than the server version ({api_vers_str}). " + msg += "Please ask the admin to update the server." + Console(file=sys.stderr).print(msg, style="yellow") + logger.warning(msg) @app.callback() @@ -246,7 +252,7 @@ def cli( logger.info(f"CLI version: {__version__}") try: - check_version_compatiblity() + check_version_compatibility() except InvalidCLIVersion as e: logger.error(format_traceback(e)) raise diff --git a/cli/setup.cfg b/cli/setup.cfg index c1051814c..5ace8acf5 100644 --- a/cli/setup.cfg +++ b/cli/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = kleinkram -version = 0.56.0 +version = 0.57.0 description = give me your bags long_description = file: README.md long_description_content_type = text/markdown diff --git a/docker/backend.Dockerfile b/docker/backend.Dockerfile index b18cad60f..a1b20b762 100644 --- a/docker/backend.Dockerfile +++ b/docker/backend.Dockerfile @@ -36,6 +36,7 @@ FROM gcr.io/distroless/nodejs22-debian12 AS production WORKDIR /app COPY --from=build /app/backend/dist/main.js ./backend/dist/main.js +COPY --from=build /app/backend/package.json ./backend/package.json WORKDIR /app/backend diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index 9b980bfda..40998a0c7 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -5,6 +5,7 @@ const require = createRequire(import.meta.url); export default withMermaid({ lang: 'en-US', + cleanUrls: true, title: 'Kleinkram', description: 'A structured bag and mcap dataset storage.', diff --git a/docs/nginx.conf b/docs/nginx.conf index a1fee2828..7995a90a2 100644 --- a/docs/nginx.conf +++ b/docs/nginx.conf @@ -6,6 +6,6 @@ server { # Serve VitePress from the root URL location / { - try_files $uri $uri/ /index.html; + try_files $uri $uri.html $uri/ /index.html; } } diff --git a/docs/package.json b/docs/package.json index 2be307028..d1745303a 100644 --- a/docs/package.json +++ b/docs/package.json @@ -1,6 +1,6 @@ { "name": "kleinkram-docs", - "version": "0.56.0", + "version": "0.57.0", "license": "MIT", "target": "es2022", "scripts": { @@ -18,21 +18,12 @@ "vue-json-pretty": "^2.6.0" }, "devDependencies": { - "@types/node": "^24.10.1", + "@types/node": "^25.0.2", "mermaid": "^11.12.1", "ts-morph": "^27.0.2", "tsx": "^4.21.0", - "vite": "^7.2.6", + "vite": "^7.3.0", "vitepress-plugin-mermaid": "2.0.17", "vue": "^3.5.25" - }, - "pnpm": { - "overrides": { - "@nestjs/common@<10.4.16": ">=10.4.16", - "esbuild@<=0.24.2": ">=0.25.0", - "got@<11.8.5": ">=11.8.5", - "js-yaml@>=4.0.0 <4.1.1": ">=4.1.1", - "tmp@<=0.2.3": ">=0.2.4" - } } } diff --git a/docs/pnpm-lock.yaml b/docs/pnpm-lock.yaml index c7b61da5b..703fd4a71 100644 --- a/docs/pnpm-lock.yaml +++ b/docs/pnpm-lock.yaml @@ -4,12 +4,6 @@ settings: autoInstallPeers: true excludeLinksFromLockfile: false -overrides: - '@nestjs/common@<10.4.16': '>=10.4.16' - esbuild@<=0.24.2: '>=0.25.0' - got@<11.8.5: '>=11.8.5' - js-yaml@>=4.0.0 <4.1.1: '>=4.1.1' - tmp@<=0.2.3: '>=0.2.4' importers: @@ -17,14 +11,14 @@ importers: dependencies: vitepress: specifier: ^1.6.4 - version: 1.6.4(@algolia/client-search@5.46.0)(@types/node@24.10.1)(axios@1.13.2)(jwt-decode@4.0.0)(postcss@8.5.6)(sass-embedded@1.93.3)(sass@1.93.3)(search-insights@2.17.3)(terser@5.44.1)(typescript@5.9.3) + version: 1.6.4(@algolia/client-search@5.46.0)(@types/node@25.0.3)(axios@1.13.2)(jwt-decode@4.0.0)(postcss@8.5.6)(sass-embedded@1.93.3)(sass@1.93.3)(search-insights@2.17.3)(terser@5.44.1)(typescript@5.9.3) vue-json-pretty: specifier: ^2.6.0 version: 2.6.0(vue@3.5.25(typescript@5.9.3)) devDependencies: '@types/node': - specifier: ^24.10.1 - version: 24.10.1 + specifier: ^25.0.2 + version: 25.0.3 mermaid: specifier: ^11.12.1 version: 11.12.2 @@ -35,11 +29,11 @@ importers: specifier: ^4.21.0 version: 4.21.0 vite: - specifier: ^7.2.6 - version: 7.2.6(@types/node@24.10.1)(jiti@2.6.1)(sass-embedded@1.93.3)(sass@1.93.3)(terser@5.44.1)(tsx@4.21.0) + specifier: ^7.3.0 + version: 7.3.0(@types/node@25.0.3)(jiti@2.6.1)(sass-embedded@1.93.3)(sass@1.93.3)(terser@5.44.1)(tsx@4.21.0) vitepress-plugin-mermaid: specifier: 2.0.17 - version: 2.0.17(mermaid@11.12.2)(vitepress@1.6.4(@algolia/client-search@5.46.0)(@types/node@24.10.1)(axios@1.13.2)(jwt-decode@4.0.0)(postcss@8.5.6)(sass-embedded@1.93.3)(sass@1.93.3)(search-insights@2.17.3)(terser@5.44.1)(typescript@5.9.3)) + version: 2.0.17(mermaid@11.12.2)(vitepress@1.6.4(@algolia/client-search@5.46.0)(@types/node@25.0.3)(axios@1.13.2)(jwt-decode@4.0.0)(postcss@8.5.6)(sass-embedded@1.93.3)(sass@1.93.3)(search-insights@2.17.3)(terser@5.44.1)(typescript@5.9.3)) vue: specifier: ^3.5.25 version: 3.5.25(typescript@5.9.3) @@ -3882,6 +3876,9 @@ packages: '@types/node@24.10.1': resolution: {integrity: sha512-GNWcUTRBgIRJD5zj+Tq0fKOJ5XZajIiBroOF0yvj2bSU1WvNdYS/dn9UxwsujGW4JX06dnHyjV2y9rRaybH0iQ==} + '@types/node@25.0.3': + resolution: {integrity: sha512-W609buLVRVmeW693xKfzHeIV6nJGGz98uCPfeXI1ELMLXVeKYZ9m15fAMSaUPBHYLGFsVRcMmSCksQOrZV9BYA==} + '@types/normalize-package-data@2.4.4': resolution: {integrity: sha512-37i+OaWTh9qeK4LSHPsyRC7NahnGotNuZvjLSgcPzblpHB3rrCJxAOgI5gCdKm7coonsaX1Of0ILiTcnZjbfxA==} @@ -9707,6 +9704,46 @@ packages: yaml: optional: true + vite@7.3.0: + resolution: {integrity: sha512-dZwN5L1VlUBewiP6H9s2+B3e3Jg96D0vzN+Ry73sOefebhYr9f94wwkMNN/9ouoU8pV1BqA1d1zGk8928cx0rg==} + engines: {node: ^20.19.0 || >=22.12.0} + hasBin: true + peerDependencies: + '@types/node': ^20.19.0 || >=22.12.0 + jiti: '>=1.21.0' + less: ^4.0.0 + lightningcss: ^1.21.0 + sass: ^1.70.0 + sass-embedded: ^1.70.0 + stylus: '>=0.54.8' + sugarss: ^5.0.0 + terser: ^5.16.0 + tsx: ^4.8.1 + yaml: ^2.4.2 + peerDependenciesMeta: + '@types/node': + optional: true + jiti: + optional: true + less: + optional: true + lightningcss: + optional: true + sass: + optional: true + sass-embedded: + optional: true + stylus: + optional: true + sugarss: + optional: true + terser: + optional: true + tsx: + optional: true + yaml: + optional: true + vitepress-plugin-mermaid@2.0.17: resolution: {integrity: sha512-IUzYpwf61GC6k0XzfmAmNrLvMi9TRrVRMsUyCA8KNXhg/mQ1VqWnO0/tBVPiX5UoKF1mDUwqn5QV4qAJl6JnUg==} peerDependencies: @@ -13964,6 +14001,10 @@ snapshots: dependencies: undici-types: 7.16.0 + '@types/node@25.0.3': + dependencies: + undici-types: 7.16.0 + '@types/normalize-package-data@2.4.4': {} '@types/oauth@0.9.6': @@ -14274,9 +14315,9 @@ snapshots: '@ungap/structured-clone@1.3.0': {} - '@vitejs/plugin-vue@5.2.4(vite@5.4.21(@types/node@24.10.1)(sass-embedded@1.93.3)(sass@1.93.3)(terser@5.44.1))(vue@3.5.25(typescript@5.9.3))': + '@vitejs/plugin-vue@5.2.4(vite@5.4.21(@types/node@25.0.3)(sass-embedded@1.93.3)(sass@1.93.3)(terser@5.44.1))(vue@3.5.25(typescript@5.9.3))': dependencies: - vite: 5.4.21(@types/node@24.10.1)(sass-embedded@1.93.3)(sass@1.93.3)(terser@5.44.1) + vite: 5.4.21(@types/node@25.0.3)(sass-embedded@1.93.3)(sass@1.93.3)(terser@5.44.1) vue: 3.5.25(typescript@5.9.3) '@vitejs/plugin-vue@6.0.2(vite@7.2.6(@types/node@24.10.1)(jiti@2.6.1)(sass-embedded@1.93.3)(sass@1.93.3)(terser@5.44.1)(tsx@4.21.0))(vue@3.5.25(typescript@5.9.3))': @@ -20934,13 +20975,13 @@ snapshots: dependencies: vite: 7.2.6(@types/node@24.10.1)(jiti@2.6.1)(sass-embedded@1.93.3)(sass@1.93.3)(terser@5.44.1)(tsx@4.21.0) - vite@5.4.21(@types/node@24.10.1)(sass-embedded@1.93.3)(sass@1.93.3)(terser@5.44.1): + vite@5.4.21(@types/node@25.0.3)(sass-embedded@1.93.3)(sass@1.93.3)(terser@5.44.1): dependencies: esbuild: 0.25.12 postcss: 8.5.6 rollup: 4.53.3 optionalDependencies: - '@types/node': 24.10.1 + '@types/node': 25.0.3 fsevents: 2.3.3 sass: 1.93.3 sass-embedded: 1.93.3 @@ -20963,14 +21004,31 @@ snapshots: terser: 5.44.1 tsx: 4.21.0 - vitepress-plugin-mermaid@2.0.17(mermaid@11.12.2)(vitepress@1.6.4(@algolia/client-search@5.46.0)(@types/node@24.10.1)(axios@1.13.2)(jwt-decode@4.0.0)(postcss@8.5.6)(sass-embedded@1.93.3)(sass@1.93.3)(search-insights@2.17.3)(terser@5.44.1)(typescript@5.9.3)): + vite@7.3.0(@types/node@25.0.3)(jiti@2.6.1)(sass-embedded@1.93.3)(sass@1.93.3)(terser@5.44.1)(tsx@4.21.0): + dependencies: + esbuild: 0.27.1 + fdir: 6.5.0(picomatch@4.0.3) + picomatch: 4.0.3 + postcss: 8.5.6 + rollup: 4.53.3 + tinyglobby: 0.2.15 + optionalDependencies: + '@types/node': 25.0.3 + fsevents: 2.3.3 + jiti: 2.6.1 + sass: 1.93.3 + sass-embedded: 1.93.3 + terser: 5.44.1 + tsx: 4.21.0 + + vitepress-plugin-mermaid@2.0.17(mermaid@11.12.2)(vitepress@1.6.4(@algolia/client-search@5.46.0)(@types/node@25.0.3)(axios@1.13.2)(jwt-decode@4.0.0)(postcss@8.5.6)(sass-embedded@1.93.3)(sass@1.93.3)(search-insights@2.17.3)(terser@5.44.1)(typescript@5.9.3)): dependencies: mermaid: 11.12.2 - vitepress: 1.6.4(@algolia/client-search@5.46.0)(@types/node@24.10.1)(axios@1.13.2)(jwt-decode@4.0.0)(postcss@8.5.6)(sass-embedded@1.93.3)(sass@1.93.3)(search-insights@2.17.3)(terser@5.44.1)(typescript@5.9.3) + vitepress: 1.6.4(@algolia/client-search@5.46.0)(@types/node@25.0.3)(axios@1.13.2)(jwt-decode@4.0.0)(postcss@8.5.6)(sass-embedded@1.93.3)(sass@1.93.3)(search-insights@2.17.3)(terser@5.44.1)(typescript@5.9.3) optionalDependencies: '@mermaid-js/mermaid-mindmap': 9.3.0 - vitepress@1.6.4(@algolia/client-search@5.46.0)(@types/node@24.10.1)(axios@1.13.2)(jwt-decode@4.0.0)(postcss@8.5.6)(sass-embedded@1.93.3)(sass@1.93.3)(search-insights@2.17.3)(terser@5.44.1)(typescript@5.9.3): + vitepress@1.6.4(@algolia/client-search@5.46.0)(@types/node@25.0.3)(axios@1.13.2)(jwt-decode@4.0.0)(postcss@8.5.6)(sass-embedded@1.93.3)(sass@1.93.3)(search-insights@2.17.3)(terser@5.44.1)(typescript@5.9.3): dependencies: '@docsearch/css': 3.8.2 '@docsearch/js': 3.8.2(@algolia/client-search@5.46.0)(search-insights@2.17.3) @@ -20979,7 +21037,7 @@ snapshots: '@shikijs/transformers': 2.5.0 '@shikijs/types': 2.5.0 '@types/markdown-it': 14.1.2 - '@vitejs/plugin-vue': 5.2.4(vite@5.4.21(@types/node@24.10.1)(sass-embedded@1.93.3)(sass@1.93.3)(terser@5.44.1))(vue@3.5.25(typescript@5.9.3)) + '@vitejs/plugin-vue': 5.2.4(vite@5.4.21(@types/node@25.0.3)(sass-embedded@1.93.3)(sass@1.93.3)(terser@5.44.1))(vue@3.5.25(typescript@5.9.3)) '@vue/devtools-api': 7.7.9 '@vue/shared': 3.5.25 '@vueuse/core': 12.8.2(typescript@5.9.3) @@ -20988,7 +21046,7 @@ snapshots: mark.js: 8.11.1 minisearch: 7.2.0 shiki: 2.5.0 - vite: 5.4.21(@types/node@24.10.1)(sass-embedded@1.93.3)(sass@1.93.3)(terser@5.44.1) + vite: 5.4.21(@types/node@25.0.3)(sass-embedded@1.93.3)(sass@1.93.3)(terser@5.44.1) vue: 3.5.25(typescript@5.9.3) optionalDependencies: postcss: 8.5.6 diff --git a/frontend/package.json b/frontend/package.json index b68fcfb55..ce9757984 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "kleinkram-frontend", - "version": "0.56.0", + "version": "0.57.0", "description": "Data storage of ROS bags", "productName": "Kleinkram", "author": "Johann Schwabe ", @@ -12,7 +12,7 @@ "build": "sh ./create_build_info.sh && quasar build" }, "dependencies": { - "@aws-sdk/client-s3": "3.726.1", + "@aws-sdk/client-s3": "3.952.0", "@foxglove/rosbag": "^0.4.1", "@foxglove/rosmsg": "^5.0.5", "@foxglove/rosmsg-serialization": "^2.0.4", @@ -45,20 +45,20 @@ "vue": "^3.5.25", "vue-echarts": "^7.0.3", "vue-json-pretty": "^2.6.0", - "vue-router": "^4.6.3" + "vue-router": "^4.6.4" }, "devDependencies": { "@quasar/app-vite": "^2.4.0", - "@types/node": "^24.10.1", + "@types/node": "^25.0.2", "@types/qs": "^6.14.0", "@types/spark-md5": "^3.0.5", "@types/sql.js": "^1.4.9", - "autoprefixer": "^10.4.22", + "autoprefixer": "^10.4.23", "jiti": "^2.6.1", "tsx": "^4.20.6", "typescript": "^5.9.3", "vite": "^7.2.6", - "vue-tsc": "^3.1.5" + "vue-tsc": "^3.1.8" }, "engines": { "node": "^22 || ^24", diff --git a/frontend/src/components/actions/actions-table.vue b/frontend/src/components/actions/actions-table.vue index fbd0c2a39..7191d6b97 100644 --- a/frontend/src/components/actions/actions-table.vue +++ b/frontend/src/components/actions/actions-table.vue @@ -50,7 +50,7 @@ /> No executions found - There are no executions matching your criteria. + {{ noDataSubtitle }} @@ -128,8 +128,14 @@ properties.handler.setSort('createdAt'); properties.handler.setDescending(true); const queryFilters = computed(() => ({ - projectUuid: (route.query.projectUuid as string) || undefined, - missionUuid: (route.query.missionUuid as string) || undefined, + projectUuid: + (route.query.projectUuid as string) || + (route.params.projectUuid as string) || + undefined, + missionUuid: + (route.query.missionUuid as string) || + (route.params.missionUuid as string) || + undefined, take: route.query.rowsPerPage ? Number(route.query.rowsPerPage) : 100, skip: route.query.page ? (Number(route.query.page) - 1) * @@ -141,6 +147,13 @@ const queryFilters = computed(() => ({ templateName: (route.query.name as string) || undefined, })); +const noDataSubtitle = computed(() => { + if (queryFilters.value.missionUuid && !queryFilters.value.templateName) { + return 'No actions have been executed for this mission yet.'; + } + return 'There are no executions matching your criteria.'; +}); + const { data: rawData, isLoading } = useActionList(queryFilters); const tableReference: Ref = ref(undefined); diff --git a/frontend/src/components/common/app-select.vue b/frontend/src/components/common/app-select.vue index 6ef213b19..88de67892 100644 --- a/frontend/src/components/common/app-select.vue +++ b/frontend/src/components/common/app-select.vue @@ -13,13 +13,15 @@ dense map-options emit-value - :options="options" + :options="filteredOptions" :option-label="optionLabel" :option-value="optionValue" + :use-input="searchable && !model" :bg-color=" bgColor ?? ($attrs.readonly || $attrs.disable ? 'grey-2' : 'white') " + @filter="filterFunction" > @@ -188,4 +181,31 @@ const missionRules = computed(() => { ] : []; }); + +// Dynamic placeholders showing example based on first available option +const dynamicProjectPlaceholder = computed(() => { + if (selectedProjectUuid.value) return; + const first = projects.value[0]; + return first ? `e.g. ${first.name}` : props.projectPlaceholder; +}); + +const dynamicMissionPlaceholder = computed(() => { + if (selectedMissionUuid.value) return; + if (!selectedProjectUuid.value) return 'Select a Project'; + const first = missions.value[0]; + return first ? `e.g. ${first.name}` : props.missionPlaceholder; +}); + +// Check if mission is disabled to show tooltip +const isMissionDisabled = computed(() => { + return ( + props.disabled || !selectedProjectUuid.value || isMissionsLoading.value + ); +}); + +const missionDisabledReason = computed(() => { + if (!selectedProjectUuid.value) return 'Select a project first'; + if (isMissionsLoading.value) return 'Loading missions...'; + return ''; +}); diff --git a/frontend/src/components/common/selection-button-group.vue b/frontend/src/components/common/selection-button-group.vue new file mode 100644 index 000000000..165138ac4 --- /dev/null +++ b/frontend/src/components/common/selection-button-group.vue @@ -0,0 +1,108 @@ + + + + + diff --git a/frontend/src/components/common/smart-search-input.vue b/frontend/src/components/common/smart-search-input.vue new file mode 100644 index 000000000..ee96e7567 --- /dev/null +++ b/frontend/src/components/common/smart-search-input.vue @@ -0,0 +1,441 @@ + + + + + diff --git a/frontend/src/components/common/table-empty-state.vue b/frontend/src/components/common/table-empty-state.vue new file mode 100644 index 000000000..005433d64 --- /dev/null +++ b/frontend/src/components/common/table-empty-state.vue @@ -0,0 +1,67 @@ + + + + + diff --git a/frontend/src/components/documentation-icon.vue b/frontend/src/components/documentation-icon.vue index d620f9fa5..df8b3e857 100644 --- a/frontend/src/components/documentation-icon.vue +++ b/frontend/src/components/documentation-icon.vue @@ -14,7 +14,7 @@ const { link } = defineProps<{ link?: string }>(); const documentationBasePath = 'https://docs.datasets.leggedrobotics.com'; -const documentationDefaultPath = '/usage/getting-started.html'; +const documentationDefaultPath = '/usage/getting-started'; const documentationLink = link ?? documentationBasePath + documentationDefaultPath; diff --git a/frontend/src/components/explorer-page/explorer-page-files-table.vue b/frontend/src/components/explorer-page/explorer-page-files-table.vue index 15049827d..b4ea56786 100644 --- a/frontend/src/components/explorer-page/explorer-page-files-table.vue +++ b/frontend/src/components/explorer-page/explorer-page-files-table.vue @@ -215,7 +215,11 @@ import type { CategoryDto } from '@kleinkram/api-dto/types/category.dto'; import type { FileWithTopicDto } from '@kleinkram/api-dto/types/file/file.dto'; import type { FilesDto } from '@kleinkram/api-dto/types/file/files.dto'; import { FileType, HealthStatus } from '@kleinkram/shared'; -import { useQuery, UseQueryReturnType } from '@tanstack/vue-query'; +import { + keepPreviousData, + useQuery, + UseQueryReturnType, +} from '@tanstack/vue-query'; import DeleteFileDialogOpener from 'components/button-wrapper/delete-file-dialog-opener.vue'; import CreateFileDialogOpener from 'components/button-wrapper/dialog-opener-create-file.vue'; import EditFileDialogOpener from 'components/button-wrapper/edit-file-dialog-opener.vue'; @@ -225,6 +229,7 @@ import { QTable } from 'quasar'; import { useMission, useMissionsOfProjectMinimal } from 'src/hooks/query-hooks'; import { useMissionUUID, useProjectUUID } from 'src/hooks/router-hooks'; import ROUTES from 'src/router/routes'; +import { parseDate } from 'src/services/date-formating'; import { _downloadFile, getColorFileState, @@ -264,6 +269,11 @@ const hasActiveFilters = computed(() => { return ( ((h.searchParams.name && h.searchParams.name.length > 0) ?? (h.searchParams.health && h.searchParams.health.length > 0) ?? + (h.searchParams.startDate && h.searchParams.startDate.length > 0) ?? + (h.searchParams.endDate && h.searchParams.endDate.length > 0) ?? + (h.searchParams.topics && h.searchParams.topics.length > 0) ?? + (h.searchParams.messageDatatypes && + h.searchParams.messageDatatypes.length > 0) ?? isTypeFilterActive) || // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition (h.categories && h.categories.length > 0) @@ -313,14 +323,20 @@ function setPagination(update: TableRequest): void { properties.urlHandler.setDescending(update.pagination.descending); } -const pagination = computed(() => { - return { +const pagination = computed({ + get: () => ({ page: properties.urlHandler.page, rowsPerPage: properties.urlHandler.take, rowsNumber: properties.urlHandler.rowsNumber, sortBy: properties.urlHandler.sortBy, descending: properties.urlHandler.descending, - }; + }), + set: (v) => { + properties.urlHandler.setPage(v.page); + properties.urlHandler.setTake(v.rowsPerPage); + properties.urlHandler.setSort(v.sortBy); + properties.urlHandler.setDescending(v.descending); + }, }); const queryKey = computed(() => [ @@ -335,18 +351,37 @@ const { isLoading, }: UseQueryReturnType = useQuery({ queryKey: queryKey, - queryFn: () => - filesOfMission( + queryFn: () => { + const h = properties.urlHandler; + return filesOfMission( missionUuid.value ?? '', - properties.urlHandler.take, - properties.urlHandler.skip, - properties.urlHandler.fileTypes, - properties.urlHandler.searchParams.name, - properties.urlHandler.categories, - properties.urlHandler.sortBy, - properties.urlHandler.descending, - properties.urlHandler.searchParams.health as HealthStatus, - ), + h.take, + h.skip, + h.fileTypes, + h.searchParams.name, + h.categories, + h.sortBy, + h.descending, + + h.searchParams.health as HealthStatus, + h.searchParams.startDate + ? parseDate(h.searchParams.startDate) + : undefined, + h.searchParams.endDate + ? parseDate(h.searchParams.endDate) + : undefined, + // Topics and Datatypes + h.searchParams.topics && h.searchParams.topics.length > 0 + ? h.searchParams.topics.split(',') + : undefined, + h.searchParams.messageDatatypes && + h.searchParams.messageDatatypes.length > 0 + ? h.searchParams.messageDatatypes.split(',') + : undefined, + h.searchParams.matchAllTopics === 'true', + ); + }, + placeholderData: keepPreviousData, }); const data = computed(() => (rawData.value ? rawData.value.data : [])); const total = computed(() => (rawData.value ? rawData.value.count : 0)); diff --git a/frontend/src/components/explorer-page/explorer-page-mission-table.vue b/frontend/src/components/explorer-page/explorer-page-mission-table.vue index 6e8d9cad8..9d648c730 100644 --- a/frontend/src/components/explorer-page/explorer-page-mission-table.vue +++ b/frontend/src/components/explorer-page/explorer-page-mission-table.vue @@ -150,7 +150,7 @@ diff --git a/frontend/src/components/explorer-page/mission-files.vue b/frontend/src/components/explorer-page/mission-files.vue new file mode 100644 index 000000000..2db4b5b12 --- /dev/null +++ b/frontend/src/components/explorer-page/mission-files.vue @@ -0,0 +1,491 @@ + + + + + diff --git a/frontend/src/components/file-type-selector.vue b/frontend/src/components/file-type-selector.vue index d52c3941f..dd3e15e22 100644 --- a/frontend/src/components/file-type-selector.vue +++ b/frontend/src/components/file-type-selector.vue @@ -1,52 +1,17 @@ - - diff --git a/frontend/src/components/files/files-filter.vue b/frontend/src/components/files/files-filter.vue new file mode 100644 index 000000000..ecdefc744 --- /dev/null +++ b/frontend/src/components/files/files-filter.vue @@ -0,0 +1,353 @@ + + + diff --git a/frontend/src/components/files/filter/components/category-filter.vue b/frontend/src/components/files/filter/components/category-filter.vue new file mode 100644 index 000000000..0dd2e36b1 --- /dev/null +++ b/frontend/src/components/files/filter/components/category-filter.vue @@ -0,0 +1,83 @@ + + + diff --git a/frontend/src/components/files/filter/components/datatype-filter.vue b/frontend/src/components/files/filter/components/datatype-filter.vue new file mode 100644 index 000000000..aded0b938 --- /dev/null +++ b/frontend/src/components/files/filter/components/datatype-filter.vue @@ -0,0 +1,64 @@ + + + diff --git a/frontend/src/components/files/filter/components/date-filter.vue b/frontend/src/components/files/filter/components/date-filter.vue new file mode 100644 index 000000000..6875ad6c2 --- /dev/null +++ b/frontend/src/components/files/filter/components/date-filter.vue @@ -0,0 +1,254 @@ + + + diff --git a/frontend/src/components/files/filter/components/file-type-filter.vue b/frontend/src/components/files/filter/components/file-type-filter.vue new file mode 100644 index 000000000..ddbc90d90 --- /dev/null +++ b/frontend/src/components/files/filter/components/file-type-filter.vue @@ -0,0 +1,29 @@ + + + diff --git a/frontend/src/components/files/filter/components/health-filter.vue b/frontend/src/components/files/filter/components/health-filter.vue new file mode 100644 index 000000000..d81ce879c --- /dev/null +++ b/frontend/src/components/files/filter/components/health-filter.vue @@ -0,0 +1,101 @@ + + + diff --git a/frontend/src/components/files/filter/components/metadata-filter.vue b/frontend/src/components/files/filter/components/metadata-filter.vue new file mode 100644 index 000000000..ac473976e --- /dev/null +++ b/frontend/src/components/files/filter/components/metadata-filter.vue @@ -0,0 +1,26 @@ + + + diff --git a/frontend/src/components/files/filter/components/project-filter.vue b/frontend/src/components/files/filter/components/project-filter.vue new file mode 100644 index 000000000..46f2d4c01 --- /dev/null +++ b/frontend/src/components/files/filter/components/project-filter.vue @@ -0,0 +1,39 @@ + + + diff --git a/frontend/src/components/files/filter/components/topic-filter.vue b/frontend/src/components/files/filter/components/topic-filter.vue new file mode 100644 index 000000000..b63f7a528 --- /dev/null +++ b/frontend/src/components/files/filter/components/topic-filter.vue @@ -0,0 +1,90 @@ + + + diff --git a/frontend/src/components/files/filter/composable-filter-popup.vue b/frontend/src/components/files/filter/composable-filter-popup.vue new file mode 100644 index 000000000..9dbc1b1c4 --- /dev/null +++ b/frontend/src/components/files/filter/composable-filter-popup.vue @@ -0,0 +1,61 @@ + + + diff --git a/frontend/src/components/files/filter/filter-popup.vue b/frontend/src/components/files/filter/filter-popup.vue new file mode 100644 index 000000000..4161efa9a --- /dev/null +++ b/frontend/src/components/files/filter/filter-popup.vue @@ -0,0 +1,485 @@ + + + + + diff --git a/frontend/src/components/files/filter/metadata-filter-builder.vue b/frontend/src/components/files/filter/metadata-filter-builder.vue new file mode 100644 index 000000000..4547749ce --- /dev/null +++ b/frontend/src/components/files/filter/metadata-filter-builder.vue @@ -0,0 +1,188 @@ + + + diff --git a/frontend/src/components/inspect-file/viewers/string-viewer.vue b/frontend/src/components/inspect-file/viewers/string-viewer.vue index 70757717c..db4eb58a6 100644 --- a/frontend/src/components/inspect-file/viewers/string-viewer.vue +++ b/frontend/src/components/inspect-file/viewers/string-viewer.vue @@ -6,6 +6,26 @@ {{ messages.length }} messages + + XML + + + JSON + - Copy JSON + Copy Messages (JSON) @@ -32,7 +52,12 @@ {{ formatTime(msg.logTime) }} -
+
- 
{{
-                                        parseContent(msg.data.data).json
+                                    
{{
+                                        parseContent(msg.data.data).formatted
                                     }}

                                     
                                                 copyText(
                                                     parseContent(msg.data.data)
-                                                        .json,
+                                                        .formatted,
                                                 )
                                         "
                                     >
-                                        Copy JSON
+                                        {{
+                                            parseContent(msg.data.data).isJson
+                                                ? 'Copy JSON'
+                                                : 'Copy XML'
+                                        }}
@@ -112,6 +141,7 @@ + + diff --git a/frontend/src/components/queue-items.vue b/frontend/src/components/queue-items.vue index f5c81819e..a1a2190b2 100644 --- a/frontend/src/components/queue-items.vue +++ b/frontend/src/components/queue-items.vue @@ -192,7 +192,7 @@ import { useQueryClient } from '@tanstack/vue-query'; import { QTable, useQuasar } from 'quasar'; import ConfirmDeleteFile from 'src/dialogs/confirm-delete-file-dialog.vue'; import ROUTES from 'src/router/routes'; -import { dateMask, formatDate } from 'src/services/date-formating'; +import { dateMask, formatDate, parseDate } from 'src/services/date-formating'; import { _downloadFile, getColor, @@ -269,9 +269,16 @@ const fileStateFilterEnums = computed(() => { }); // Use Composable -// @ts-ignore +const startDateIso = computed(() => { + try { + return parseDate(startDate.value).toISOString(); + } catch { + return ''; + } +}); + const { data: queueEntries, isLoading } = useQueue( - startDate, + startDateIso, // @ts-ignore fileStateFilterEnums, queryPagination, diff --git a/frontend/src/components/user-profile/admin-settings.vue b/frontend/src/components/user-profile/admin-settings.vue index eb3710fe0..cc0cc3831 100644 --- a/frontend/src/components/user-profile/admin-settings.vue +++ b/frontend/src/components/user-profile/admin-settings.vue @@ -103,7 +103,7 @@ async function recalculateHashes(): Promise { async function reextractTopics(): Promise { // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment - const { data } = await axios.post('file/reextractTopics'); + const { data } = await axios.post('files/reextractTopics'); $q.notify({ // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/restrict-template-expressions diff --git a/frontend/src/composables/use-file-filter.ts b/frontend/src/composables/use-file-filter.ts new file mode 100644 index 000000000..1bca6ecbb --- /dev/null +++ b/frontend/src/composables/use-file-filter.ts @@ -0,0 +1,235 @@ +import { FileType, HealthStatus } from '@kleinkram/shared'; +import { useQuery } from '@tanstack/vue-query'; +import { useHandler } from 'src/hooks/query-hooks'; +import { formatDate, parseDate } from 'src/services/date-formating'; +import { allTopicsNames, allTopicTypes } from 'src/services/queries/topic'; +import { FileTypeOption } from 'src/types/file-type-option'; +import { computed, reactive, ref, watch } from 'vue'; + +export interface FilterState { + filter: string; + startDates: string; + endDates: string; + selectedTopics: string[]; + selectedDatatypes: string[]; + matchAllTopics: boolean; + fileTypeFilter: FileTypeOption[] | undefined; + tagFilter: Record; + health: HealthStatus | undefined; +} + +export const DEFAULT_STATE = (): FilterState => { + const start = new Date(0); + const end = new Date(); + end.setHours(23, 59, 59, 999); + + const allFileTypes = Object.values(FileType) + .filter((t) => t !== FileType.ALL) + .map((name) => ({ + name, + value: true, + })); + + return { + filter: '', + startDates: formatDate(start), + endDates: formatDate(end), + selectedTopics: [], + selectedDatatypes: [], + matchAllTopics: false, + fileTypeFilter: allFileTypes, + tagFilter: {}, + health: undefined, + }; +}; + +export function useFileFilter() { + const handler = useHandler(); + + const defaultState = DEFAULT_STATE(); + + // Sync from URL (handler) if present + if (handler.value.searchParams.name) { + defaultState.filter = handler.value.searchParams.name; + } + if (handler.value.searchParams.startDate) { + defaultState.startDates = handler.value.searchParams.startDate; + } + if (handler.value.searchParams.endDate) { + defaultState.endDates = handler.value.searchParams.endDate; + } + + // Sync FileTypes + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition + if (handler.value.fileTypes && defaultState.fileTypeFilter) { + const activeTypes = new Set(handler.value.fileTypes); + defaultState.fileTypeFilter = defaultState.fileTypeFilter.map((ft) => ({ + ...ft, + value: activeTypes.has(ft.name as FileType), + })); + } + + const state = reactive(defaultState); + + // -- Queries for Options -- + const { data: allTopics } = useQuery({ + queryKey: ['topics'], + queryFn: allTopicsNames, + }); + + const { data: allDatatypes } = useQuery({ + queryKey: ['topicTypes'], + queryFn: allTopicTypes, + }); + + // -- Computed -- + const startDate = computed(() => parseDate(state.startDates)); + const endDate = computed(() => parseDate(state.endDates)); + + const selectedFileTypesFilter = computed(() => { + const list = state.fileTypeFilter ?? []; + return list + .filter((option) => option.value) + .map((option) => option.name) as FileType[]; + }); + + const tagFilterQuery = computed(() => { + const query: Record = {}; + for (const key of Object.keys(state.tagFilter)) { + const value = state.tagFilter[key]?.value; + + if (value === undefined || value === '') continue; + query[key] = String(value); + } + return query; + }); + + // -- Actions -- + function resetStartDate(): void { + const defaultS = DEFAULT_STATE(); + state.startDates = defaultS.startDates; + } + + function resetEndDate(): void { + const defaultS = DEFAULT_STATE(); + state.endDates = defaultS.endDates; + } + + function resetFilter(): void { + handler.value.setProjectUUID(undefined); + handler.value.setMissionUUID(undefined); + handler.value.setSearch({ name: '' }); + + // Preserve fileTypeFilter structure if it exists, just selecting all + const currentFileTypes = state.fileTypeFilter; + + Object.assign(state, DEFAULT_STATE()); + + if (currentFileTypes) { + state.fileTypeFilter = currentFileTypes.map((it) => ({ + ...it, + value: true, + })); + } + } + + function useAndTopicFilter(): void { + state.matchAllTopics = true; + } + + function useOrTopicFilter(): void { + state.matchAllTopics = false; + } + + function applyDateShortcut( + type: 'today' | '7days' | 'lastmonth' | 'ytd' | 'all', + ): void { + if (type === 'all') { + resetStartDate(); + resetEndDate(); + // Clear from Search params + const newParameters = { ...handler.value.searchParams }; + delete newParameters.startDate; + delete newParameters.endDate; + + handler.value.setSearch({ + ...newParameters, + startDate: undefined as unknown as string, + endDate: undefined as unknown as string, + }); + return; + } + + const end = new Date(); + // End of day today + end.setHours(23, 59, 59, 999); + + const start = new Date(); + start.setHours(0, 0, 0, 0); + + switch (type) { + case 'today': { + // start is already beginning of today + break; + } + case '7days': { + start.setDate(start.getDate() - 7); + break; + } + case 'lastmonth': { + // 1 month ago + start.setMonth(start.getMonth() - 1); + break; + } + case 'ytd': { + // Jan 1st + start.setMonth(0, 1); + break; + } + // No default + } + + const s = formatDate(start); + const dateEnd = formatDate(end); + state.startDates = s; + state.endDates = dateEnd; + + // Immediately apply to handler (Search) + handler.value.setSearch({ + ...handler.value.searchParams, + startDate: s, + endDate: dateEnd, + }); + } + + // -- Debounce Logic -- + const debouncedFilter = ref(state.filter); + let timeout: ReturnType; + + watch( + () => state.filter, + (value: string) => { + clearTimeout(timeout); + timeout = setTimeout(() => { + debouncedFilter.value = value; + }, 300); + }, + ); + + return { + state, + startDate, + endDate, + selectedFileTypesFilter, + tagFilterQuery, + debouncedFilter, + allTopics, + allDatatypes, + resetStartDate, + resetEndDate, + resetFilter, + useAndTopicFilter, + useOrTopicFilter, + applyDateShortcut, + }; +} diff --git a/frontend/src/composables/use-file-search.ts b/frontend/src/composables/use-file-search.ts new file mode 100644 index 000000000..8b37538d0 --- /dev/null +++ b/frontend/src/composables/use-file-search.ts @@ -0,0 +1,194 @@ +import { FileType } from '@kleinkram/shared'; +import { useQuery } from '@tanstack/vue-query'; +import { + useAllTags, + useFilteredProjects, + useMissionsOfProjectMinimal, + useProjectQuery, +} from 'src/hooks/query-hooks'; +import { Filter } from 'src/services/filters/filter-interface'; +import { DatatypeFilter } from 'src/services/filters/implementations/datatype-filter'; +import { + EndDateFilter, + StartDateFilter, +} from 'src/services/filters/implementations/date-filter'; +import { FileTypeFilter } from 'src/services/filters/implementations/file-type-filter'; +import { MetadataFilter } from 'src/services/filters/implementations/metadata-filter'; +import { MissionFilter } from 'src/services/filters/implementations/mission-filter'; +import { ProjectFilter } from 'src/services/filters/implementations/project-filter'; +import { TopicFilter } from 'src/services/filters/implementations/topic-filter'; +import { allTopicsNames, allTopicTypes } from 'src/services/queries/topic'; +import { CompositeFilterProvider } from 'src/services/suggestions/strategies/composite-filter-provider'; +import { KeywordStrategy } from 'src/services/suggestions/strategies/keyword-strategy'; +import { + MetadataTag, + SuggestionProvider, +} from 'src/services/suggestions/suggestion-types'; +import { computed, Ref } from 'vue'; +import { FilterState } from './use-file-filter'; +import { FilterParserContext } from './use-filter-parser'; + +export interface FileSearchContextData extends FilterParserContext { + projects: { name: string; uuid: string }[]; + missions: { name: string; uuid: string }[]; + topics: string[]; + datatypes: string[]; + fileTypes: string[]; + availableTags: MetadataTag[]; + hasProjectSelected: boolean; + projectUuid?: string | undefined; + missionUuid?: string | undefined; + setProject?: ((uuid: string | undefined) => void) | undefined; + setMission?: ((uuid: string | undefined) => void) | undefined; +} + +export function useFileSearch( + currentProjectUuid: Ref, + currentMissionUuid?: Ref, + setProject?: (uuid: string | undefined) => void, + setMission?: (uuid: string | undefined) => void, +) { + // --- Data Fetching --- + + // Projects + const { data: projectsData } = useFilteredProjects(100, 0, 'name', false); + const { data: selectedProject } = useProjectQuery(currentProjectUuid); + + const projects = computed(() => { + const list = + projectsData.value?.data.map((p) => ({ + name: p.name, + uuid: p.uuid, + })) ?? []; + + if (currentProjectUuid.value) { + const p = list.find((x) => x.uuid === currentProjectUuid.value); + if (!p) { + list.push({ + name: + selectedProject.value?.name ?? currentProjectUuid.value, + uuid: currentProjectUuid.value, + }); + } + } + return list; + }); + + // Missions (dependent on project) + const { data: missionsData } = useMissionsOfProjectMinimal( + currentProjectUuid, + 100, + 0, + ); + const missions = computed(() => { + const list = + missionsData.value?.data.map((m) => ({ + name: m.name, + uuid: m.uuid, + })) ?? []; + + if (currentMissionUuid?.value) { + const m = list.find((x) => x.uuid === currentMissionUuid.value); + if (!m) { + list.push({ + name: currentMissionUuid.value, // We don't have a mission query yet, so just use UUID + uuid: currentMissionUuid.value, + }); + } + } + return list; + }); + + // Topics + const { data: allTopics } = useQuery({ + queryKey: ['topics'], + queryFn: allTopicsNames, + }); + + // Datatypes + const { data: allDatatypes } = useQuery({ + queryKey: ['topicTypes'], + queryFn: allTopicTypes, + }); + + // Tags + const { data: allTags } = useAllTags(); + + // FileTypes (Static) + const allFileTypes = Object.values(FileType).filter( + (f) => f !== FileType.ALL, + ); + + // --- Context Construction --- + + const contextData = computed(() => ({ + projects: projects.value, + missions: missions.value, + topics: allTopics.value ?? [], + datatypes: allDatatypes.value ?? [], + fileTypes: allFileTypes, + availableTags: + allTags.value?.map((t) => ({ + name: t.name, + uuid: t.uuid, + datatype: t.datatype, + })) ?? [], + hasProjectSelected: !!currentProjectUuid.value, + projectUuid: currentProjectUuid.value, + missionUuid: currentMissionUuid?.value, + setProject, + setMission, + })); + + // --- Instantiate Filters --- + const filters: Filter[] = [ + new ProjectFilter(), + new MissionFilter(), + new TopicFilter(), + new DatatypeFilter(), + new FileTypeFilter(), + new StartDateFilter(), + new EndDateFilter(), + new MetadataFilter(), + ]; + + // --- Keyword Strategy (for suggesting filter keywords like "project:", "mission:", etc.) --- + const keywordMap: Record = {}; + for (const f of filters) { + // Convert key like "project:" to label like "PROJECT" for display + keywordMap[f.label.toUpperCase()] = f.key; + } + // Add &topic: variant + keywordMap.TOPIC_AND = '&topic:'; + + const keywordStrategy = new KeywordStrategy( + keywordMap, + { + // Mission requires project to be selected + // eslint-disable-next-line @typescript-eslint/naming-convention + 'mission:': (context) => ({ + enabled: context.data.hasProjectSelected, + reason: 'Select a project first', + }), + }, + ); + + // --- Provider --- + // Combine KeywordStrategy with filter suggestions + const provider = new CompositeFilterProvider([ + keywordStrategy, + ...(filters as unknown as SuggestionProvider[]), + ]); + + return { + provider, + filters, + contextData, + projects, + missions, + allTopics, + allDatatypes, + allTags, + allFileTypes, + }; +} diff --git a/frontend/src/composables/use-filter-parser.ts b/frontend/src/composables/use-filter-parser.ts new file mode 100644 index 000000000..6c9aa5613 --- /dev/null +++ b/frontend/src/composables/use-filter-parser.ts @@ -0,0 +1,280 @@ +import { Filter } from 'src/services/filters/filter-interface'; +import { + parseSearchString, + validateSearchSyntax, +} from 'src/services/suggestions/search-parser'; +import { computed } from 'vue'; +import { FilterState } from './use-file-filter'; + +export const KEYWORDS = { + PROJECT: 'project:', + MISSION: 'mission:', + TOPIC: 'topic:', + TOPIC_AND: '&topic:', + DATATYPE: 'datatype:', + FILETYPE: 'filetype:', + START: 'date-start:', + END: 'date-end:', + METADATA: 'meta:', + HEALTH: 'health:', + CATEGORY: 'category:', +}; + +function quote(s: string) { + return s.includes(' ') ? `"${s}"` : s; +} + +export interface FilterParserContext { + projects: { name: string; uuid: string }[]; + missions: { name: string; uuid: string }[]; + projectUuid?: string | undefined; + missionUuid?: string | undefined; + setProject?: ((uuid: string | undefined) => void) | undefined; + setMission?: ((uuid: string | undefined) => void) | undefined; + [key: string]: unknown; +} + +export function useFilterParser( + state: FilterState, + filters: Filter[], + contextAccessor: () => TContext, + defaults?: { + defaultStartDate?: string; + defaultEndDate?: string; + }, +) { + // Generate filter string from State + const filterString = computed(() => { + const parts: string[] = []; + const keys = new Set(filters.map((f) => f.key)); + const context = contextAccessor(); + + if (keys.has(KEYWORDS.PROJECT) && context.projectUuid) { + const p = context.projects.find( + (x) => x.uuid === context.projectUuid, + ); + if (p) parts.push(`${KEYWORDS.PROJECT}${quote(p.name)}`); + } + if (keys.has(KEYWORDS.MISSION) && context.missionUuid) { + const m = context.missions.find( + (x) => x.uuid === context.missionUuid, + ); + if (m) parts.push(`${KEYWORDS.MISSION}${quote(m.name)}`); + } + if (keys.has(KEYWORDS.TOPIC) || keys.has(KEYWORDS.TOPIC_AND)) { + const topicKeyword = state.matchAllTopics + ? KEYWORDS.TOPIC_AND + : KEYWORDS.TOPIC; + for (const t of state.selectedTopics) + parts.push(`${topicKeyword}${quote(t)}`); + } + + if (keys.has(KEYWORDS.DATATYPE)) { + for (const d of state.selectedDatatypes) + parts.push(`${KEYWORDS.DATATYPE}${quote(d)}`); + } + + if (keys.has(KEYWORDS.FILETYPE) && state.fileTypeFilter) { + const allSelected = state.fileTypeFilter.every((ft) => ft.value); + if (!allSelected) { + for (const ft of state.fileTypeFilter) { + if (ft.value) + parts.push(`${KEYWORDS.FILETYPE}${quote(ft.name)}`); + } + } + } + + if (keys.has(KEYWORDS.METADATA)) { + for (const tag of Object.values(state.tagFilter)) { + parts.push( + `${KEYWORDS.METADATA}${quote(`${tag.name}=${tag.value}`)}`, + ); + } + } + + if (keys.has(KEYWORDS.HEALTH) && state.health) { + parts.push(`${KEYWORDS.HEALTH}${state.health}`); + } + + if ( + keys.has(KEYWORDS.CATEGORY) && + 'categories' in state && + // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access + Array.isArray((state as any).categories) + ) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access + const cats = (state as any).categories as string[]; + // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access + const availableCats = (context as any).availableCategories as + | { + name: string; + uuid: string; + }[] + | undefined; + + if (availableCats) { + for (const catUuid of cats) { + const cat = availableCats.find((c) => c.uuid === catUuid); + if (cat) { + parts.push(`${KEYWORDS.CATEGORY}${quote(cat.name)}`); + } + } + } + } + + // Dates + if (keys.has(KEYWORDS.START)) { + const startDateOnly = state.startDates.split(' ')[0]; + const isAllDates = startDateOnly === '01.01.1970'; + if (!isAllDates && state.startDates && startDateOnly) + parts.push(`${KEYWORDS.START}${startDateOnly}`); + } + + if (keys.has(KEYWORDS.END)) { + const startDateOnly = state.startDates.split(' ')[0]; + const isAllDates = startDateOnly === '01.01.1970'; + if (!isAllDates && state.endDates) { + const dateOnly = state.endDates.split(' ')[0]; + if (dateOnly) parts.push(`${KEYWORDS.END}${dateOnly}`); + } + } + + if (state.filter) { + parts.push(state.filter); + } + + return parts.join(' '); + }); + + // eslint-disable-next-line complexity + function parse(input: string) { + const parsedContext = contextAccessor(); + + // Reset State + state.selectedTopics = []; + state.selectedDatatypes = []; + state.tagFilter = {}; + state.filter = ''; + state.health = undefined; + if (defaults?.defaultStartDate) + state.startDates = defaults.defaultStartDate; + if (defaults?.defaultEndDate) state.endDates = defaults.defaultEndDate; + + if ('categories' in state) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access + (state as any).categories = []; + } + + // Tokenize + const { tokens, freeText } = parseSearchString(input); + + // FileType logic + const fileTypeKey = KEYWORDS.FILETYPE.replace(':', ''); + const hasFileTypeToken = tokens.some((t) => t.key === fileTypeKey); + if (hasFileTypeToken && state.fileTypeFilter) { + for (const ft of state.fileTypeFilter) ft.value = false; + } else if (state.fileTypeFilter) { + for (const ft of state.fileTypeFilter) ft.value = true; + } + + let hasAndTopic = false; + + // Iterate Tokens + for (const token of tokens) { + if (!token.key) continue; + const fullKey = token.key + ':'; + + // Handle &topic: specially + if (fullKey === KEYWORDS.TOPIC_AND) { + const topicFilter = filters.find( + (f) => f.key === KEYWORDS.TOPIC, + ); + if (topicFilter) { + topicFilter.parse(token.value, state, parsedContext); + hasAndTopic = true; + } + continue; + } + + const filter = filters.find((f) => f.key === fullKey); + + if (filter) { + filter.parse(token.value, state, parsedContext); + } + } + + state.matchAllTopics = hasAndTopic; + + // Check if project/mission were in the input. If not, clear them from context. + const hasProjectToken = tokens.some((t) => t.key === 'project'); + const hasMissionToken = tokens.some((t) => t.key === 'mission'); + + if (!hasProjectToken && parsedContext.setProject) { + parsedContext.setProject(undefined); + } + if (!hasMissionToken && parsedContext.setMission) { + parsedContext.setMission(undefined); + } + + // Collect unknown tokens + let extraFreeText = ''; + for (const token of tokens) { + if (!token.key) continue; + const fullKey = token.key + ':'; + const filter = filters.find( + (f) => + f.key === fullKey || + (fullKey === KEYWORDS.TOPIC_AND && + f.key === KEYWORDS.TOPIC), + ); + if (!filter) { + extraFreeText += (extraFreeText ? ' ' : '') + token.original; + } + } + + const combinedFreeText = ( + freeText + (extraFreeText ? ' ' + extraFreeText : '') + ) + .trim() + .replaceAll(/\s+/g, ' '); + + state.filter = (state.filter + ' ' + combinedFreeText).trim(); + } + + function validateSyntax(input: string): string | null { + const validKeys = filters.flatMap((f) => { + const k = f.key.replace(':', ''); + if (f.key === KEYWORDS.TOPIC) return [k, '&topic']; + return [k]; + }); + + const error = validateSearchSyntax(input, validKeys); + if (error) return error; + + // Check for invalid & prefix + const invalidAndPrefixRegex = + /&(project|mission|datatype|filetype|date-start|date-end|meta):/gi; + const invalidMatch = invalidAndPrefixRegex.exec(input); + if (invalidMatch) { + return `The & prefix is only valid for topics (use &topic: for AND matching). Invalid: &${invalidMatch[1] ?? ''}:`; + } + + // Mission dependency check + const hasMission = input.toLowerCase().includes('mission:'); + if (hasMission) { + const context = contextAccessor(); + const hasProject = input.toLowerCase().includes('project:'); + if (!context.projectUuid && !hasProject) { + return 'Cannot filter for mission without project filter.'; + } + } + + return null; + } + + return { + filterString, + parse, + validateSyntax, + }; +} diff --git a/frontend/src/composables/use-filter-sync.ts b/frontend/src/composables/use-filter-sync.ts new file mode 100644 index 000000000..d283f2add --- /dev/null +++ b/frontend/src/composables/use-filter-sync.ts @@ -0,0 +1,63 @@ +import { useHandler } from 'src/hooks/query-hooks'; +import { computed, nextTick, ref, watch } from 'vue'; + +export function useFilterSync( + handler: ReturnType, + refreshCallback: () => void, +) { + const draftProjectUuid = ref(handler.value.projectUuid); + const draftMissionUuid = ref(handler.value.missionUuid); + + // Sync draft from handler initially (and if URL changes externally) + watch( + () => handler.value.projectUuid, + (value) => { + if (value !== draftProjectUuid.value) + draftProjectUuid.value = value; + }, + ); + watch( + () => handler.value.missionUuid, + (value) => { + if (value !== draftMissionUuid.value) + draftMissionUuid.value = value; + }, + ); + + const projectUuid = computed( + () => draftProjectUuid.value ?? handler.value.projectUuid, + ); + const missionUuid = computed( + () => draftMissionUuid.value ?? handler.value.missionUuid, + ); + + function setProjectUUID(v: string | undefined) { + draftProjectUuid.value = v; + } + function setMissionUUID(v: string | undefined) { + draftMissionUuid.value = v; + } + + // Watch for changes from Advanced Filter UI to trigger URL update + // Using nextTick to ensure all reactive updates (filterString) have propagated first + watch([draftProjectUuid, draftMissionUuid], ([newProject, newMission]) => { + // Only trigger refresh if the change wasn't from URL sync (handler watch) + const projectChangedFromUI = newProject !== handler.value.projectUuid; + const missionChangedFromUI = newMission !== handler.value.missionUuid; + + if (projectChangedFromUI || missionChangedFromUI) { + void nextTick(() => { + refreshCallback(); + }); + } + }); + + return { + draftProjectUuid, + draftMissionUuid, + projectUuid, + missionUuid, + setProjectUUID, + setMissionUUID, + }; +} diff --git a/frontend/src/composables/use-mission-file-filter.ts b/frontend/src/composables/use-mission-file-filter.ts new file mode 100644 index 000000000..892933901 --- /dev/null +++ b/frontend/src/composables/use-mission-file-filter.ts @@ -0,0 +1,209 @@ +import { FileType, HealthStatus } from '@kleinkram/shared'; +import { useQuery } from '@tanstack/vue-query'; +import { useHandler } from 'src/hooks/query-hooks'; +import { formatDate, parseDate } from 'src/services/date-formating'; +import { allTopicsNames, allTopicTypes } from 'src/services/queries/topic'; +import { FileTypeOption } from 'src/types/file-type-option'; +import { computed, reactive, ref, watch } from 'vue'; + +export interface MissionFilterState { + filter: string; // filename + health: HealthStatus | undefined; + categories: string[]; // UUIDs + startDates: string; + endDates: string; + selectedTopics: string[]; + selectedDatatypes: string[]; + matchAllTopics: boolean; + fileTypeFilter: FileTypeOption[] | undefined; + tagFilter: Record; +} + +export const DEFAULT_MISSION_STATE = (): MissionFilterState => { + const start = new Date(0); + const end = new Date(); + end.setHours(23, 59, 59, 999); + + const allFileTypes = Object.values(FileType) + .filter((t) => t !== FileType.ALL) + .map((name) => ({ + name, + value: true, + })); + + return { + filter: '', + health: undefined, + categories: [], + startDates: formatDate(start), + endDates: formatDate(end), + selectedTopics: [], + selectedDatatypes: [], + matchAllTopics: false, + fileTypeFilter: allFileTypes, + tagFilter: {}, + }; +}; + +export function useMissionFileFilter() { + const handler = useHandler(); + + const defaultState = DEFAULT_MISSION_STATE(); + + // Sync from URL (handler) if present + if (handler.value.searchParams.name) { + defaultState.filter = handler.value.searchParams.name; + } + if (handler.value.searchParams.health) { + // @ts-ignore + defaultState.health = handler.value.searchParams.health as HealthStatus; + } + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition + if (handler.value.categories) { + defaultState.categories = handler.value.categories; + } + if (handler.value.searchParams.startDate) { + defaultState.startDates = handler.value.searchParams.startDate; + } + if (handler.value.searchParams.endDate) { + defaultState.endDates = handler.value.searchParams.endDate; + } + if (handler.value.searchParams.messageDatatypes) { + defaultState.selectedDatatypes = + handler.value.searchParams.messageDatatypes.split(','); + } + if (handler.value.searchParams.topics) { + defaultState.selectedTopics = + handler.value.searchParams.topics.split(','); + } + if (handler.value.searchParams.matchAllTopics) { + defaultState.matchAllTopics = + handler.value.searchParams.matchAllTopics === 'true'; + } + + // Sync FileTypes + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition + if (handler.value.fileTypes && defaultState.fileTypeFilter) { + const activeTypes = new Set(handler.value.fileTypes); + defaultState.fileTypeFilter = defaultState.fileTypeFilter.map((ft) => ({ + ...ft, + value: activeTypes.has(ft.name as FileType), + })); + } + + const state = reactive(defaultState); + + // -- Queries for Options -- + const { data: allTopics } = useQuery({ + queryKey: ['topics'], + queryFn: allTopicsNames, + }); + + const { data: allDatatypes } = useQuery({ + queryKey: ['topicTypes'], + queryFn: allTopicTypes, + }); + + // -- Computed -- + const startDate = computed(() => parseDate(state.startDates)); + const endDate = computed(() => parseDate(state.endDates)); + + const selectedFileTypesFilter = computed(() => { + const list = state.fileTypeFilter ?? []; + return list + .filter((option) => option.value) + .map((option) => option.name) as FileType[]; + }); + + const tagFilterQuery = computed(() => { + const query: Record = {}; + for (const key of Object.keys(state.tagFilter)) { + const value = state.tagFilter[key]?.value; + + if (value === undefined || value === '') continue; + + if (typeof value === 'string') { + if (value.trim() !== '') { + query[key] = value; + } + } else { + // Handle numbers, booleans, dates + query[key] = String(value); + } + } + return query; + }); + + // -- Actions -- + function resetStartDate(): void { + const defaultS = DEFAULT_MISSION_STATE(); + state.startDates = defaultS.startDates; + } + + function resetEndDate(): void { + const defaultS = DEFAULT_MISSION_STATE(); + state.endDates = defaultS.endDates; + } + + function resetFilter(): void { + handler.value.setSearch({ name: '', health: '' }); + handler.value.setCategories([]); + + // Preserve fileTypeFilter structure if it exists, just selecting all + const currentFileTypes = state.fileTypeFilter; + + Object.assign(state, DEFAULT_MISSION_STATE()); + + if (currentFileTypes) { + state.fileTypeFilter = currentFileTypes.map((it) => ({ + ...it, + value: true, + })); + } + } + + function useAndTopicFilter(): void { + state.matchAllTopics = true; + } + + function useOrTopicFilter(): void { + state.matchAllTopics = false; + } + + // -- Debounce Logic -- + const debouncedFilter = ref(state.filter); + let timeout: ReturnType; + + watch( + () => state.filter, + (value: string) => { + clearTimeout(timeout); + timeout = setTimeout(() => { + debouncedFilter.value = value; + }, 300); + }, + ); + + watch( + () => handler.value.categories, + (newCategories) => { + state.categories = newCategories; + }, + ); + + return { + state, + startDate, + endDate, + selectedFileTypesFilter, + tagFilterQuery, + debouncedFilter, + allTopics, + allDatatypes, + resetStartDate, + resetEndDate, + resetFilter, + useAndTopicFilter, + useOrTopicFilter, + }; +} diff --git a/frontend/src/composables/use-mission-file-search.ts b/frontend/src/composables/use-mission-file-search.ts new file mode 100644 index 000000000..ee135f866 --- /dev/null +++ b/frontend/src/composables/use-mission-file-search.ts @@ -0,0 +1,172 @@ +import { FileType } from '@kleinkram/shared'; +import { useQuery } from '@tanstack/vue-query'; +import { useAllTags } from 'src/hooks/query-hooks'; +import { Filter } from 'src/services/filters/filter-interface'; +import { CategoryFilter } from 'src/services/filters/implementations/category-filter'; +import { DatatypeFilter } from 'src/services/filters/implementations/datatype-filter'; +import { + EndDateFilter, + StartDateFilter, +} from 'src/services/filters/implementations/date-filter'; +import { FileTypeFilter } from 'src/services/filters/implementations/file-type-filter'; +import { FilenameFilter } from 'src/services/filters/implementations/filename-filter'; +import { HealthFilter } from 'src/services/filters/implementations/health-filter'; +import { TopicFilter } from 'src/services/filters/implementations/topic-filter'; +import { getCategories } from 'src/services/queries/categories'; +import { allTopicsNames, allTopicTypes } from 'src/services/queries/topic'; +import { CompositeFilterProvider } from 'src/services/suggestions/strategies/composite-filter-provider'; +import { KeywordStrategy } from 'src/services/suggestions/strategies/keyword-strategy'; +import { + MetadataTag, + SuggestionProvider, +} from 'src/services/suggestions/suggestion-types'; +import { computed, Ref } from 'vue'; +import { FilterParserContext } from './use-filter-parser'; +import { MissionFilterState } from './use-mission-file-filter'; + +import { FileWithTopicDto } from '@kleinkram/api-dto/types/file/file.dto'; + +export interface MissionFileSearchContextData extends FilterParserContext { + topics: string[]; + datatypes: string[]; + fileTypes: string[]; + filenames: string[]; + availableTags: MetadataTag[]; + availableCategories: { name: string; uuid: string; description?: string }[]; + // Compatibility properties + projects: { name: string; uuid: string }[]; + missions: { name: string; uuid: string }[]; +} + +export function useMissionFileSearch( + projectUuid: Ref, + files: Ref, +) { + // --- Data Fetching --- + + // Topics + const { data: allTopics } = useQuery({ + queryKey: ['topics'], + queryFn: allTopicsNames, + }); + + // Datatypes + const { data: allDatatypes } = useQuery({ + queryKey: ['topicTypes'], + queryFn: allTopicTypes, + }); + + // Tags + const { data: allTags } = useAllTags(); + + // Categories (fetched from project) + const { data: categoriesDto } = useQuery({ + queryKey: ['categories', projectUuid], + queryFn: () => getCategories(projectUuid.value ?? ''), + enabled: computed(() => !!projectUuid.value), + }); + + const availableCategories = computed(() => { + const categories = categoriesDto.value?.data ?? []; + return categories + .map((c) => ({ name: c.name, uuid: c.uuid, description: c.name })) + .sort((a, b) => a.name.localeCompare(b.name)); + }); + + // FileTypes (Static) + const allFileTypes = Object.values(FileType).filter( + (f) => f !== FileType.ALL, + ); + + // Filenames + const allFilenames = computed(() => { + return files.value?.map((f) => f.filename) ?? []; + }); + + // --- Context Construction --- + + const contextData = computed(() => ({ + topics: allTopics.value ?? [], + datatypes: allDatatypes.value ?? [], + fileTypes: allFileTypes, + filenames: allFilenames.value, + availableTags: + allTags.value?.map((t) => ({ + name: t.name, + uuid: t.uuid, + datatype: t.datatype, + })) ?? [], + availableCategories: availableCategories.value, + hasProjectSelected: true, // Always true in mission context + projectUuid: projectUuid.value, + // Legacy compatibility for FileSearchContextData + projects: [], + missions: [], + missionUuid: undefined, // Already have mission context implicitly, but filter might check this. + // eslint-disable-next-line @typescript-eslint/no-empty-function + setProject: () => {}, + // eslint-disable-next-line @typescript-eslint/no-empty-function + setMission: () => {}, + })); + + // --- Instantiate Filters --- + const filters: Filter[] = + [ + // Filename needs to be handled? FilenameFilter handles "filename:foo". + // Free text "foo" is handled by the parser default. + new FilenameFilter(), + new HealthFilter(), + new CategoryFilter(), + new TopicFilter() as unknown as Filter< + MissionFilterState, + MissionFileSearchContextData + >, + new DatatypeFilter() as unknown as Filter< + MissionFilterState, + MissionFileSearchContextData + >, + new FileTypeFilter() as unknown as Filter< + MissionFilterState, + MissionFileSearchContextData + >, + new StartDateFilter() as unknown as Filter< + MissionFilterState, + MissionFileSearchContextData + >, + new EndDateFilter() as unknown as Filter< + MissionFilterState, + MissionFileSearchContextData + >, + ]; + + // --- Keyword Strategy (for suggesting filter keywords like "health:", "category:", etc.) --- + const keywordMap: Record = {}; + for (const f of filters) { + keywordMap[f.label.toUpperCase()] = f.key; + } + + // Add &topic: variant + keywordMap.TOPIC_AND = '&topic:'; + + const keywordStrategy = new KeywordStrategy( + keywordMap, + {}, + ); + + // --- Provider --- + const provider = new CompositeFilterProvider([ + keywordStrategy, + ...(filters as unknown as SuggestionProvider[]), + ]); + + return { + provider, + filters, + contextData, + allTopics, + allDatatypes, + allTags, + allFileTypes, + availableCategories, + }; +} diff --git a/frontend/src/dialogs/tag-filter.vue b/frontend/src/dialogs/tag-filter.vue index d6620c6df..ddb975e39 100644 --- a/frontend/src/dialogs/tag-filter.vue +++ b/frontend/src/dialogs/tag-filter.vue @@ -172,6 +172,6 @@ const updateTagValues = (newTagValues: Record): void => { }; const applyAction = (): void => { - onDialogOK(convertedTagValues); + onDialogOK(convertedTagValues.value); }; diff --git a/frontend/src/pages/data-table-page.vue b/frontend/src/pages/data-table-page.vue index 9eef41d74..8cf41db69 100644 --- a/frontend/src/pages/data-table-page.vue +++ b/frontend/src/pages/data-table-page.vue @@ -1,197 +1,7 @@ + diff --git a/frontend/src/pages/files-explorer-page.vue b/frontend/src/pages/files-explorer-page.vue index a23bacbc6..b372ef55a 100644 --- a/frontend/src/pages/files-explorer-page.vue +++ b/frontend/src/pages/files-explorer-page.vue @@ -33,19 +33,7 @@ - - -
-
- - - - - - - - - - - -
- -
- - + + + + + + - - - - - - -
-
- - - - -
-
- - - - - -
-
+ + + + + + + + +