Merge pull request #224 from import-ai/refactor/files

refactor(s3): use s3

Author: ycdzj

package.json

@@ -76,6 +76,7 @@
     "@opentelemetry/sdk-node": "^0.203.0",
     "@opentelemetry/sdk-trace-base": "^2.1.0",
     "@opentelemetry/semantic-conventions": "^1.37.0",
+    "aws4fetch": "^1.0.20",
     "bcrypt": "^5.1.1",
     "cache-manager": "^6.4.3",
     "cacheable": "^2.1.1",

pnpm-lock.yaml

@@ -69,6 +69,9 @@ import KeyvRedis from '@keyv/redis';
 import { Keyv } from 'keyv';
 import { CacheableMemory } from 'cacheable';
 import { isEmpty } from 'omniboxd/utils/is-empty';
+import { Files1761556143000 } from 'omniboxd/migrations/1761556143000-files';
+import { FilesModule } from 'omniboxd/files/files.module';
+import { AddFileIdToResources1761726974942 } from 'omniboxd/migrations/1761726974942-add-file-id-to-resources';
 
 @Module({})
 export class AppModule implements NestModule {
@@ -140,6 +143,7 @@ export class AppModule implements NestModule {
         FeedbackModule,
         ApplicationsModule,
         WebSocketModule,
+        FilesModule,
         CacheModule.registerAsync({
           isGlobal: true,
           imports: [ConfigModule],
@@ -187,6 +191,8 @@ export class AppModule implements NestModule {
               NullableUserId1757844448000,
               AddShareIdToConversations1757844449000,
               ShareUser1760171824000,
+              Files1761556143000,
+              AddFileIdToResources1761726974942,
               ...extraMigrations,
             ],
             migrationsRun: true,

src/app/app.module.ts

@@ -0,0 +1,20 @@
+import { Base } from 'omniboxd/common/base.entity';
+import { Column, Entity, PrimaryGeneratedColumn } from 'typeorm';
+
+@Entity('files')
+export class File extends Base {
+  @PrimaryGeneratedColumn()
+  id: string;
+
+  @Column()
+  namespaceId: string;
+
+  @Column()
+  userId: string;
+
+  @Column()
+  name: string;
+
+  @Column()
+  mimetype: string;
+}

src/files/entities/file.entity.ts

@@ -0,0 +1,12 @@
+import { Module } from '@nestjs/common';
+import { FilesService } from './files.service';
+import { ConfigModule } from '@nestjs/config';
+import { TypeOrmModule } from '@nestjs/typeorm';
+import { File } from './entities/file.entity';
+
+@Module({
+  imports: [ConfigModule, TypeOrmModule.forFeature([File])],
+  providers: [FilesService],
+  exports: [FilesService],
+})
+export class FilesModule {}

src/files/files.module.ts

@@ -0,0 +1,125 @@
+import { HttpStatus, Injectable } from '@nestjs/common';
+import { AwsClient } from 'aws4fetch';
+import { ConfigService } from '@nestjs/config';
+import { Repository } from 'typeorm';
+import { File } from './entities/file.entity';
+import { InjectRepository } from '@nestjs/typeorm';
+import { AppException } from 'omniboxd/common/exceptions/app.exception';
+import { I18nService } from 'nestjs-i18n';
+
+@Injectable()
+export class FilesService {
+  private readonly awsClient: AwsClient;
+  private readonly s3Url: URL;
+  private readonly s3InternalUrl: URL;
+
+  constructor(
+    configService: ConfigService,
+
+    @InjectRepository(File)
+    private readonly fileRepo: Repository<File>,
+    private readonly i18n: I18nService,
+  ) {
+    const accessKeyId = configService.get<string>('OBB_S3_ACCESS_KEY_ID');
+    const secretAccessKey = configService.get<string>(
+      'OBB_S3_SECRET_ACCESS_KEY',
+    );
+    if (!accessKeyId || !secretAccessKey) {
+      throw new Error('S3 credentials not set');
+    }
+
+    let s3Url = configService.get<string>('OBB_S3_URL');
+    if (!s3Url) {
+      throw new Error('S3 URL not set');
+    }
+    if (!s3Url.endsWith('/')) {
+      s3Url += '/';
+    }
+
+    let s3InternalUrl = configService.get<string>('OBB_S3_INTERNAL_URL');
+    if (!s3InternalUrl) {
+      s3InternalUrl = s3Url;
+    } else if (!s3InternalUrl.endsWith('/')) {
+      s3InternalUrl += '/';
+    }
+
+    this.awsClient = new AwsClient({ accessKeyId, secretAccessKey });
+    this.s3Url = new URL(s3Url);
+    this.s3InternalUrl = new URL(s3InternalUrl);
+  }
+
+  async createFile(
+    userId: string,
+    namespaceId: string,
+    filename: string,
+    mimetype: string,
+  ): Promise<File> {
+    return await this.fileRepo.save(
+      this.fileRepo.create({
+        namespaceId,
+        userId,
+        name: filename,
+        mimetype,
+      }),
+    );
+  }
+
+  async getFile(namespaceId: string, fileId: string): Promise<File | null> {
+    return await this.fileRepo.findOne({ where: { namespaceId, id: fileId } });
+  }
+
+  async generateUploadUrl(fileId: string): Promise<string> {
+    const fileUrl = new URL(fileId, this.s3Url);
+    fileUrl.searchParams.set('X-Amz-Expires', '900'); // 900 seconds
+    const signedReq = await this.awsClient.sign(fileUrl.toString(), {
+      method: 'PUT',
+      aws: {
+        service: 's3',
+        signQuery: true,
+      },
+    });
+    return signedReq.url;
+  }
+
+  private async generateDownloadUrl(
+    namespaceId: string,
+    fileId: string,
+    s3Url: URL,
+  ): Promise<string> {
+    const file = await this.getFile(namespaceId, fileId);
+    if (!file) {
+      const message = this.i18n.t('resource.errors.fileNotFound');
+      throw new AppException(message, 'FILE_NOT_FOUND', HttpStatus.NOT_FOUND);
+    }
+
+    const fileUrl = new URL(fileId, s3Url);
+    fileUrl.searchParams.set('X-Amz-Expires', '900'); // 900 seconds
+    fileUrl.searchParams.set(
+      'response-content-disposition',
+      `attachment; filename*=UTF-8''${encodeURIComponent(file.name)}`,
+    );
+
+    const signedReq = await this.awsClient.sign(fileUrl.toString(), {
+      method: 'GET',
+      aws: {
+        service: 's3',
+        signQuery: true,
+      },
+    });
+    return signedReq.url;
+  }
+
+  async generatePublicDownloadUrl(
+    namespaceId: string,
+    fileId: string,
+  ): Promise<string> {
+    return this.generateDownloadUrl(namespaceId, fileId, this.s3Url);
+  }
+
+  async generateInternalDownloadUrl(
+    namespaceId: string,
+    fileId: string,
+  ): Promise<string> {
+    return this.generateDownloadUrl(namespaceId, fileId, this.s3InternalUrl);
+  }
+}

src/files/files.service.ts

@@ -3,12 +3,14 @@
     "resourceNotFile": "Resource is not a file",
     "parentOrResourceIdRequired": "parent_id or resource_id is required",
     "resourceNotFound": "Resource not found",
+    "fileNotFound": "File not found",
     "cannotDeleteRoot": "Cannot delete root resource",
     "cannotDuplicateRoot": "Cannot duplicate root resource",
     "cannotRestoreRoot": "Cannot restore root resource",
     "contentRequired": "Content is required for the resource",
     "fileResourceNotFound": "File resource not found",
     "cycleDetected": "Cycle detected in the resource tree",
-    "cannotSetParentToSubResource": "Cannot set parent to a sub-resource"
+    "cannotSetParentToSubResource": "Cannot set parent to a sub-resource",
+    "invalidResourceType": "Invalid resource type"
   }
 }

src/i18n/en/resource.json

@@ -3,12 +3,14 @@
     "resourceNotFile": "资源不是文件",
     "parentOrResourceIdRequired": "需要 parent_id 或 resource_id",
     "resourceNotFound": "资源未找到",
+    "fileNotFound": "文件未找到",
     "cannotDeleteRoot": "无法删除根资源",
     "cannotDuplicateRoot": "无法复制根资源",
     "cannotRestoreRoot": "无法恢复根资源",
     "contentRequired": "资源需要内容",
     "fileResourceNotFound": "文件资源未找到",
     "cycleDetected": "资源树中检测到循环引用",
-    "cannotSetParentToSubResource": "无法将父资源设置为子资源"
+    "cannotSetParentToSubResource": "无法将父资源设置为子资源",
+    "invalidResourceType": "资源类型无效"
   }
 }

src/i18n/zh/resource.json

@@ -0,0 +1,56 @@
+import { MigrationInterface, QueryRunner, Table } from 'typeorm';
+import { BaseColumns } from './base-columns';
+
+export class Files1761556143000 implements MigrationInterface {
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    const table = new Table({
+      name: 'files',
+      columns: [
+        {
+          name: 'id',
+          type: 'uuid',
+          isPrimary: true,
+          default: 'uuid_generate_v4()',
+        },
+        {
+          name: 'namespace_id',
+          type: 'character varying',
+          isNullable: false,
+        },
+        {
+          name: 'user_id',
+          type: 'uuid',
+          isNullable: false,
+        },
+        {
+          name: 'name',
+          type: 'character varying',
+          isNullable: false,
+        },
+        {
+          name: 'mimetype',
+          type: 'character varying',
+          isNullable: false,
+        },
+        ...BaseColumns(),
+      ],
+      foreignKeys: [
+        {
+          columnNames: ['namespace_id'],
+          referencedTableName: 'namespaces',
+          referencedColumnNames: ['id'],
+        },
+        {
+          columnNames: ['user_id'],
+          referencedTableName: 'users',
+          referencedColumnNames: ['id'],
+        },
+      ],
+    });
+    await queryRunner.createTable(table, true, true, true);
+  }
+
+  public down(): Promise<void> {
+    throw new Error('Not supported.');
+  }
+}

src/migrations/1761556143000-files.ts

@@ -0,0 +1,32 @@
+import {
+  MigrationInterface,
+  QueryRunner,
+  TableColumn,
+  TableForeignKey,
+} from 'typeorm';
+
+export class AddFileIdToResources1761726974942 implements MigrationInterface {
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.addColumn(
+      'resources',
+      new TableColumn({
+        name: 'file_id',
+        type: 'uuid',
+        isNullable: true,
+      }),
+    );
+
+    await queryRunner.createForeignKey(
+      'resources',
+      new TableForeignKey({
+        columnNames: ['file_id'],
+        referencedTableName: 'files',
+        referencedColumnNames: ['id'],
+      }),
+    );
+  }
+
+  public down(): Promise<void> {
+    throw new Error('Not supported.');
+  }
+}