fix: tenant s3 credentials fixes and refactor

migrations/multitenant/0017-tenants-s3-credentials-fix-notify-key.sql

@@ -0,0 +1,21 @@
+CREATE OR REPLACE FUNCTION tenants_s3_credentials_update_notify_trigger ()
+    RETURNS TRIGGER
+AS $$
+BEGIN
+    PERFORM
+        pg_notify('tenants_s3_credentials_update', '"' || NEW.tenant_id || ':' || NEW.access_key || '"');
+    RETURN NULL;
+END;
+$$
+    LANGUAGE plpgsql;
+
+CREATE OR REPLACE FUNCTION tenants_s3_credentials_delete_notify_trigger ()
+    RETURNS TRIGGER
+AS $$
+BEGIN
+    PERFORM
+        pg_notify('tenants_s3_credentials_update', '"' || OLD.tenant_id || ':' || OLD.access_key || '"');
+    RETURN NULL;
+END;
+$$
+    LANGUAGE plpgsql;

src/http/error-handler.ts

@@ -64,7 +64,7 @@ export const setErrorHandler = (app: FastifyInstance) => {
     // Fastify errors
     if ('statusCode' in error) {
       const err = error as FastifyError
-      return reply.status((error as any).statusCode || 500).send({
+      return reply.status(err.statusCode || 500).send({
         statusCode: `${err.statusCode}`,
         error: err.name,
         message: err.message,

src/http/plugins/db.ts

@@ -171,7 +171,7 @@ export const migrations = fastifyPlugin(
     })
 
     if (dbMigrationStrategy === MultitenantMigrationStrategy.ON_REQUEST) {
-      const migrationsMutex = createMutexByKey()
+      const migrationsMutex = createMutexByKey<void>()
 
       fastify.addHook('preHandler', async (request) => {
         // migrations are handled via async migrations

src/http/plugins/jwt.ts

@@ -25,7 +25,7 @@ export const jwt = fastifyPlugin(
     fastify.decorateRequest('jwt', '')
     fastify.decorateRequest('jwtPayload', undefined)
 
-    fastify.addHook('preHandler', async (request, reply) => {
+    fastify.addHook('preHandler', async (request) => {
       request.jwt = (request.headers.authorization || '').replace(BEARER, '')
 
       if (!request.jwt && request.routeOptions.config.allowInvalidJwt) {
@@ -41,13 +41,14 @@ export const jwt = fastifyPlugin(
         request.jwtPayload = payload
         request.owner = payload.sub
         request.isAuthenticated = true
-      } catch (err: any) {
+      } catch (e) {
         request.jwtPayload = { role: 'anon' }
         request.isAuthenticated = false
 
         if (request.routeOptions.config.allowInvalidJwt) {
           return
         }
+        const err = e as Error
         throw ERRORS.AccessDenied(err.message, err)
       }
     })

src/http/plugins/signature-v4.ts

@@ -1,6 +1,6 @@
 import { FastifyInstance, FastifyRequest } from 'fastify'
 import fastifyPlugin from 'fastify-plugin'
-import { getJwtSecret, getS3CredentialsByAccessKey, getTenantConfig } from '@internal/database'
+import { getJwtSecret, getTenantConfig, s3CredentialsManager } from '@internal/database'
 import { ClientSignature, SignatureV4 } from '@storage/protocols/s3'
 import { signJWT, verifyJWT } from '@internal/auth'
 import { ERRORS } from '@internal/errors'
@@ -160,7 +160,7 @@ async function createServerSignature(tenantId: string, clientSignature: ClientSi
   }
 
   if (isMultitenant) {
-    const credential = await getS3CredentialsByAccessKey(
+    const credential = await s3CredentialsManager.getS3CredentialsByAccessKey(
       tenantId,
       clientSignature.credentials.accessKey
     )

src/http/routes/admin/jwks.ts

@@ -110,7 +110,7 @@ export default async function routes(fastify: FastifyInstance) {
       }
 
       const result = await jwksManager.addJwk(params.tenantId, body.jwk, body.kind)
-      return reply.send(result)
+      return reply.status(201).send(result)
     }
   )
 

src/http/routes/admin/s3.ts

@@ -1,7 +1,9 @@
 import { FastifyInstance, RequestGenericInterface } from 'fastify'
 import apiKey from '../../plugins/apikey'
-import { createS3Credentials, deleteS3Credential, listS3Credentials } from '@internal/database'
+import { s3CredentialsManager } from '@internal/database'
 import { FromSchema } from 'json-schema-to-ts'
+import { isUuid } from '@storage/limits'
+import { ERRORS } from '@internal/errors'
 
 const createCredentialsSchema = {
   description: 'Create S3 Credentials',
@@ -88,7 +90,7 @@ export default async function routes(fastify: FastifyInstance) {
       schema: createCredentialsSchema,
     },
     async (req, reply) => {
-      const credentials = await createS3Credentials(req.params.tenantId, {
+      const credentials = await s3CredentialsManager.createS3Credentials(req.params.tenantId, {
         description: req.body.description,
         claims: req.body.claims,
       })
@@ -106,8 +108,7 @@ export default async function routes(fastify: FastifyInstance) {
     '/:tenantId/credentials',
     { schema: listCredentialsSchema },
     async (req, reply) => {
-      const credentials = await listS3Credentials(req.params.tenantId)
-
+      const credentials = await s3CredentialsManager.listS3Credentials(req.params.tenantId)
       return reply.send(credentials)
     }
   )
@@ -116,7 +117,10 @@ export default async function routes(fastify: FastifyInstance) {
     '/:tenantId/credentials',
     { schema: deleteCredentialsSchema },
     async (req, reply) => {
-      await deleteS3Credential(req.params.tenantId, req.body.id)
+      if (!isUuid(req.body.id)) {
+        throw ERRORS.InvalidParameter('id not uuid')
+      }
+      await s3CredentialsManager.deleteS3Credential(req.params.tenantId, req.body.id)
 
       return reply.code(204).send()
     }

src/internal/concurrency/mutex.ts

@@ -1,9 +1,9 @@
 import { Semaphore } from '@shopify/semaphore'
 
-export function createMutexByKey() {
+export function createMutexByKey<T>() {
   const semaphoreMap = new Map<string, { semaphore: Semaphore; count: number }>()
 
-  return async (key: string, fn: () => Promise<any>) => {
+  return async (key: string, fn: () => Promise<T>) => {
     let entry = semaphoreMap.get(key)
     if (!entry) {
       entry = { semaphore: new Semaphore(1), count: 0 }

src/internal/database/jwks-manager/manager.ts

@@ -9,7 +9,7 @@ const TENANTS_JWKS_UPDATE_CHANNEL = 'tenants_jwks_update'
 const JWK_KIND_STORAGE_URL_SIGNING = 'storage-url-signing-key'
 const JWK_KID_SEPARATOR = '_'
 
-const tenantJwksMutex = createMutexByKey()
+const tenantJwksMutex = createMutexByKey<JwksConfig>()
 const tenantJwksConfigCache = new Map<string, JwksConfig>()
 
 function createJwkKid({ kind, id }: { id: string; kind: string }): string {

src/internal/database/s3-credentials-manager/index.ts

@@ -0,0 +1 @@
+export * from './manager'

src/internal/database/s3-credentials-manager/manager.ts

@@ -0,0 +1,126 @@
+import crypto from 'node:crypto'
+import { LRUCache } from 'lru-cache'
+import objectSizeOf from 'object-sizeof'
+import { S3Credentials, S3CredentialsManagerStore, S3CredentialsRaw } from './store'
+import { createMutexByKey } from '@internal/concurrency'
+import { ERRORS } from '@internal/errors'
+import { getConfig } from '../../../config'
+import { decrypt, encrypt } from '@internal/auth'
+import { PubSubAdapter } from '@internal/pubsub'
+
+const TENANTS_S3_CREDENTIALS_UPDATE_CHANNEL = 'tenants_s3_credentials_update'
+
+const tenantS3CredentialsCache = new LRUCache<string, S3Credentials>({
+  maxSize: 1024 * 1024 * 50, // 50MB
+  ttl: 1000 * 60 * 60, // 1 hour
+  sizeCalculation: (value) => objectSizeOf(value),
+  updateAgeOnGet: true,
+  allowStale: false,
+})
+
+const s3CredentialsMutex = createMutexByKey<S3Credentials>()
+
+export class S3CredentialsManager {
+  private dbServiceRole: string
+
+  constructor(private storage: S3CredentialsManagerStore) {
+    const { dbServiceRole } = getConfig()
+    this.dbServiceRole = dbServiceRole
+  }
+
+  /**
+   * Keeps the in memory config cache up to date
+   */
+  async listenForTenantUpdate(pubSub: PubSubAdapter): Promise<void> {
+    await pubSub.subscribe(TENANTS_S3_CREDENTIALS_UPDATE_CHANNEL, (cacheKey) => {
+      tenantS3CredentialsCache.delete(cacheKey)
+    })
+  }
+
+  /**
+   * Create S3 Credential for a tenant
+   * @param tenantId
+   * @param data
+   */
+  async createS3Credentials(
+    tenantId: string,
+    data: { description: string; claims?: S3Credentials['claims'] }
+  ) {
+    const existingCount = await this.countS3Credentials(tenantId)
+
+    if (existingCount >= 50) {
+      throw ERRORS.MaximumCredentialsLimit()
+    }
+
+    const accessKey = crypto.randomBytes(32).toString('hex').slice(0, 32)
+    const secretKey = crypto.randomBytes(64).toString('hex').slice(0, 64)
+
+    if (data.claims) {
+      delete data.claims.iss
+      delete data.claims.issuer
+      delete data.claims.exp
+      delete data.claims.iat
+    }
+
+    const claims = {
+      ...(data.claims || {}),
+      role: data.claims?.role ?? this.dbServiceRole,
+      issuer: `supabase.storage.${tenantId}`,
+      sub: data.claims?.sub,
+    }
+
+    const id = await this.storage.insert(tenantId, {
+      description: data.description,
+      claims,
+      accessKey,
+      secretKey: encrypt(secretKey),
+    })
+
+    return {
+      id,
+      access_key: accessKey,
+      secret_key: secretKey,
+    }
+  }
+
+  async getS3CredentialsByAccessKey(tenantId: string, accessKey: string): Promise<S3Credentials> {
+    const cacheKey = `${tenantId}:${accessKey}`
+    const cachedCredentials = tenantS3CredentialsCache.get(cacheKey)
+
+    if (cachedCredentials) {
+      return cachedCredentials
+    }
+
+    return s3CredentialsMutex(cacheKey, async () => {
+      const cachedCredentials = tenantS3CredentialsCache.get(cacheKey)
+
+      if (cachedCredentials) {
+        return cachedCredentials
+      }
+
+      const data = await this.storage.getOneByAccessKey(tenantId, accessKey)
+
+      if (!data) {
+        throw ERRORS.MissingS3Credentials()
+      }
+
+      data.secretKey = decrypt(data.secretKey)
+
+      tenantS3CredentialsCache.set(cacheKey, data)
+
+      return data
+    })
+  }
+
+  deleteS3Credential(tenantId: string, credentialId: string): Promise<number> {
+    return this.storage.delete(tenantId, credentialId)
+  }
+
+  listS3Credentials(tenantId: string): Promise<S3CredentialsRaw[]> {
+    return this.storage.list(tenantId)
+  }
+
+  async countS3Credentials(tenantId: string) {
+    return this.storage.count(tenantId)
+  }
+}

src/internal/database/s3-credentials-manager/store-knex.ts

@@ -0,0 +1,59 @@
+import { Knex } from 'knex'
+import {
+  S3Credentials,
+  S3CredentialsManagerStore,
+  S3CredentialsRaw,
+  S3CredentialWithDescription,
+} from './store'
+
+export class S3CredentialsManagerStoreKnex implements S3CredentialsManagerStore {
+  constructor(private knex: Knex) {}
+
+  async insert(tenantId: string, credential: S3CredentialWithDescription): Promise<string> {
+    const credentials = await this.knex
+      .table('tenants_s3_credentials')
+      .insert({
+        tenant_id: tenantId,
+        description: credential.description,
+        access_key: credential.accessKey,
+        secret_key: credential.secretKey,
+        claims: JSON.stringify(credential.claims),
+      })
+      .returning('id')
+    return credentials[0].id
+  }
+
+  list(tenantId: string): Promise<S3CredentialsRaw[]> {
+    return this.knex
+      .table('tenants_s3_credentials')
+      .select<S3CredentialsRaw[]>('id', 'description', 'access_key', 'created_at')
+      .where('tenant_id', tenantId)
+      .orderBy('created_at', 'asc')
+  }
+
+  getOneByAccessKey(tenantId: string, accessKey: string): Promise<S3Credentials> {
+    return this.knex
+      .table('tenants_s3_credentials')
+      .select({ accessKey: 'access_key', secretKey: 'secret_key', claims: 'claims' })
+      .where('tenant_id', tenantId)
+      .where('access_key', accessKey)
+      .first()
+  }
+
+  async count(tenantId: string): Promise<number> {
+    const data = await this.knex
+      .table('tenants_s3_credentials')
+      .count<{ count: number }>('id')
+      .where('tenant_id', tenantId)
+      .first()
+    return Number(data?.count || 0)
+  }
+
+  delete(tenantId: string, credentialId: string): Promise<number> {
+    return this.knex
+      .table('tenants_s3_credentials')
+      .where('tenant_id', tenantId)
+      .where('id', credentialId)
+      .delete()
+  }
+}

src/internal/database/s3-credentials-manager/store.ts

@@ -0,0 +1,56 @@
+export interface S3Credentials {
+  accessKey: string
+  secretKey: string
+  claims: { role: string; sub?: string; [key: string]: unknown }
+}
+
+export interface S3CredentialWithDescription extends S3Credentials {
+  description: string
+}
+
+export interface S3CredentialsRaw {
+  id: string
+  description: string
+  access_key: string
+  created_at: string
+}
+
+export interface S3CredentialsManagerStore {
+  /**
+   * Inserts a new credential and returns the id
+   *
+   * @param tenantId
+   */
+  insert(tenantId: string, credential: S3CredentialWithDescription): Promise<string>
+
+  /**
+   * List all credentials for the specified tenant
+   * Returns data in the database style (snake case) format because the endpoint is expected to return data in this format
+   *
+   * @param tenantId
+   */
+  list(tenantId: string): Promise<S3CredentialsRaw[]>
+
+  /**
+   * Get one credential for the specified tenant / access key
+   *
+   * @param tenantId
+   * @param accessKey
+   */
+  getOneByAccessKey(tenantId: string, accessKey: string): Promise<S3Credentials>
+
+  /**
+   * Gets the count of credentials for the specified tenant
+   *
+   * @param tenantId
+   */
+  count(tenantId: string): Promise<number>
+
+  /**
+   * Deletes a credential and returns the count of items deleted
+   *
+   * @param tenantId
+   * @param credentialId
+   */
+  delete(tenantId: string, credentialId: string): Promise<number>
+}