@uppy/golden-retriever: Store metaData in IndexedDB (#6362)

This PR fixes #6280, Adds **IndexedDB** as MetaDataStore falling back to
`localStorage` when IndexedDB isn't available.

AI Disclaimer : AI Used

  ### Changes
- **`IndexedDBMetaDataStore.ts`** : IndexedDB-backed `MetaDataStore`;
sync `get`/`set` via in-memory cache, async `load`/`save`.
- **`IndexedDBStore.ts`** : add a `state` store to the existing
`uppy-blobs` DB (v3→4, additive); expire it in `cleanup()`; close on
`versionchange`.
- **`index.ts`** : choose IndexedDB vs localStorage; restore is async
and failure-safe.
- **`MetaDataStore.ts`** : `setItem` wrapped in try/catch so the
fallback can't throw either.
  - **Added Tests**

  ### Notes for reviewers
- **Snapshot is `JSON.stringify`'d before `put()`** (not stored raw):
structured clone *throws* on non-cloneable values; JSON drops them like
localStorage did. Storing raw froze the snapshot and ghosted files.
- **No migration** : an in-flight recovery at upgrade time starts fresh
(rare, no data loss).
- **Best-effort persistence** : write failures are swallowed; they must
never break uploading.
  
  #### Manual Testing :  Manually Tested with large Assemblies

Author: qxprakash

.changeset/shy-knives-read.md

@@ -0,0 +1,5 @@
+---
+"@uppy/golden-retriever": minor
+---
+
+Use IndexedDB as a metadata store and fall back to localStorage when IndexedDB is not available.

packages/@uppy/golden-retriever/src/IndexedDBMetaDataStore.ts

@@ -0,0 +1,114 @@
+import type { Body, Meta } from '@uppy/core'
+import throttle from 'lodash/throttle.js'
+import {
+  connect,
+  DB_NAME,
+  METADATA_STORE_NAME,
+  waitForRequest,
+} from './IndexedDBStore.js'
+import type { StoredState } from './MetaDataStore.js'
+
+type MetaData<M extends Meta, B extends Body> = StoredState<M, B>['metadata']
+
+// `metadata` is stored as a JSON string, not a live object. IndexedDB persists
+// values via the structured clone algorithm, which *throws* on anything
+// non-cloneable (e.g. a function) anywhere in the graph — whereas the
+// localStorage path used `JSON.stringify`, which silently drops such values.
+// Cloning the live Uppy/Transloadit state directly made `put` throw, the error
+// was swallowed, and the snapshot froze at an early state — so restored files
+// looked not-yet-uploaded and were ghosted. Serializing to JSON ourselves keeps
+// the exact semantics of the (working) localStorage path.
+type StateRecord = {
+  id: string
+  expires: number
+  metadata: string
+}
+
+type Options = { storeName: string; expires: number; throttleTime?: number }
+
+/**
+ * IndexedDB-backed twin of MetaDataStore. Same `load`/`get`/`set` contract, but
+ * the recovery snapshot lives in IndexedDB instead of localStorage so large
+ * Transloadit assemblies don't blow past localStorage's ~5MB quota (issue #6280).
+ *
+ * `get` stays synchronous (served from an in-memory cache) because it runs on
+ * every state update; only loading and persisting touch IndexedDB.
+ */
+export default class IndexedDBMetaDataStore<M extends Meta, B extends Body> {
+  #db: Promise<IDBDatabase>
+
+  #cache: MetaData<M, B> | null | undefined
+
+  #key: string
+
+  #expires: number
+
+  #saveThrottled: () => void
+
+  constructor(opts: Options) {
+    this.#db = connect(DB_NAME)
+    // Attach a handler now so a connection failure can't surface as an
+    // unhandledrejection before load()/#save() get a chance to await it; both
+    // of those handle the rejection themselves (persistence is best-effort).
+    this.#db.catch(() => {})
+    this.#key = opts.storeName
+    this.#expires = opts.expires
+    this.#saveThrottled =
+      opts.throttleTime === 0
+        ? this.#save
+        : throttle(this.#save, opts.throttleTime ?? 500, {
+            leading: true,
+            trailing: true,
+          })
+  }
+
+  async load(): Promise<MetaData<M, B> | undefined> {
+    try {
+      const db = await this.#db
+      const record = await waitForRequest<StateRecord | undefined>(
+        db
+          .transaction([METADATA_STORE_NAME])
+          .objectStore(METADATA_STORE_NAME)
+          .get(this.#key),
+      )
+      if (!record || record.expires < Date.now()) return undefined
+      this.#cache = JSON.parse(record.metadata)
+      return this.#cache ?? undefined
+    } catch {
+      // Best-effort: a corrupt record, failed read, or unavailable DB must
+      // never break restore — mirrors the localStorage path's tolerant parse.
+      return undefined
+    }
+  }
+
+  get(): MetaData<M, B> | undefined {
+    return this.#cache ?? undefined
+  }
+
+  set(metadata: MetaData<M, B> | null): void {
+    this.#cache = metadata
+    this.#saveThrottled()
+  }
+
+  #save = async (): Promise<void> => {
+    try {
+      const db = await this.#db
+      const store = db
+        .transaction([METADATA_STORE_NAME], 'readwrite')
+        .objectStore(METADATA_STORE_NAME)
+      if (this.#cache == null) {
+        store.delete(this.#key)
+      } else {
+        // JSON.stringify (not raw structured clone) so non-cloneable values are
+        // dropped instead of throwing. See the StateRecord comment above.
+        store.put({
+          id: this.#key,
+          expires: Date.now() + this.#expires,
+          metadata: JSON.stringify(this.#cache),
+        })
+      }
+    } catch {
+      // Persistence is best-effort; a failed write must never break uploading.
+    }
+  }
+}

packages/@uppy/golden-retriever/src/IndexedDBStore.ts

@@ -14,10 +14,12 @@ const indexedDB =
 
 const isSupported = !!indexedDB
 
-const DB_NAME = 'uppy-blobs'
+export const DB_NAME = 'uppy-blobs'
 const STORE_NAME = 'files' // maybe have a thumbnail store in the future
+
+export const METADATA_STORE_NAME = 'metadata'
 const DEFAULT_EXPIRY = 24 * 60 * 60 * 1000 // 24 hours
-const DB_VERSION = 3
+const DB_VERSION = 4
 const MiB = 0x10_00_00
 
 /**
@@ -36,11 +38,12 @@ function migrateExpiration(store: IDBObjectStore): void {
   }
 }
 
-function connect(dbName: string): Promise<IDBDatabase> {
+export function connect(dbName: string): Promise<IDBDatabase> {
   const request = (indexedDB as IDBFactory).open(dbName, DB_VERSION)
   return new Promise((resolve, reject) => {
     request.onupgradeneeded = (event) => {
       const db: IDBDatabase = (event.target as IDBOpenDBRequest).result
+      closeOnVersionChange(db)
       const transaction = (event.currentTarget as IDBOpenDBRequest)
         .transaction as IDBTransaction
 
@@ -58,18 +61,38 @@ function connect(dbName: string): Promise<IDBDatabase> {
         migrateExpiration(store)
       }
 
+      if (event.oldVersion < 4) {
+        // Added in v4: a store for GoldenRetriever's recovery snapshot, which
+        // moved out of localStorage to escape its ~5MB quota. See issue #6280.
+        const store = db.createObjectStore(METADATA_STORE_NAME, {
+          keyPath: 'id',
+        })
+        store.createIndex('expires', 'expires', { unique: false })
+      }
+
       transaction.oncomplete = () => {
         resolve(db)
       }
     }
     request.onsuccess = (event) => {
-      resolve((event.target as IDBRequest).result)
+      const db = (event.target as IDBRequest).result as IDBDatabase
+      closeOnVersionChange(db)
+      resolve(db)
     }
     request.onerror = reject
   })
 }
 
-function waitForRequest<T>(request: IDBRequest): Promise<T> {
+/**
+ * Close this connection when another tab/instance requests a higher DB version,
+ * so we never block its upgrade (and it never blocks ours). The connection is
+ * unusable afterwards, but recovery persistence is best-effort.
+ */
+function closeOnVersionChange(db: IDBDatabase): void {
+  db.onversionchange = () => db.close()
+}
+
+export function waitForRequest<T>(request: IDBRequest): Promise<T> {
   return new Promise((resolve, reject) => {
     request.onsuccess = (event) => {
       resolve((event.target as IDBRequest).result)
@@ -230,17 +253,13 @@ class IndexedDBStore {
   }
 
   /**
-   * Delete all stored blobs that have an expiry date that is before Date.now().
-   * This is a static method because it deletes expired blobs from _all_ Uppy instances.
+   * Delete every expired entry in a store, using its `expires` index.
    */
-  static async cleanup(): Promise<void> {
-    const db = await connect(DB_NAME)
-    const transaction = db.transaction([STORE_NAME], 'readwrite')
-    const store = transaction.objectStore(STORE_NAME)
+  static #deleteExpired(store: IDBObjectStore): Promise<void> {
     const request = store
       .index('expires')
       .openCursor(IDBKeyRange.upperBound(Date.now()))
-    await new Promise<void>((resolve, reject) => {
+    return new Promise<void>((resolve, reject) => {
       request.onsuccess = (event) => {
         const cursor = (event.target as IDBRequest).result
         if (cursor) {
@@ -252,7 +271,31 @@ class IndexedDBStore {
       }
       request.onerror = reject
     })
-    db.close()
+  }
+
+  /**
+   * Delete all stored blobs and recovery snapshots that have an expiry date
+   * before Date.now(). This is a static method because it deletes expired data
+   * from _all_ Uppy instances.
+   */
+  static async cleanup(): Promise<void> {
+    const db = await connect(DB_NAME)
+    try {
+      const transaction = db.transaction(
+        [STORE_NAME, METADATA_STORE_NAME],
+        'readwrite',
+      )
+      await Promise.all([
+        IndexedDBStore.#deleteExpired(transaction.objectStore(STORE_NAME)),
+        IndexedDBStore.#deleteExpired(
+          transaction.objectStore(METADATA_STORE_NAME),
+        ),
+      ])
+    } finally {
+      // Always release the connection, even if expiry fails — otherwise a
+      // rejected delete would leak the connection (and could block upgrades).
+      db.close()
+    }
   }
 }
 

packages/@uppy/golden-retriever/src/MetaDataStore.ts

@@ -115,8 +115,11 @@ export default class MetaDataStore<M extends Meta, B extends Body> {
       localStorage.removeItem(this.name)
       return
     }
-    const state = JSON.stringify(this.#state)
-    localStorage.setItem(this.name, state)
+    try {
+      localStorage.setItem(this.name, JSON.stringify(this.#state))
+    } catch {
+      // Best-effort: out of quota / disabled storage must not break uploading.
+    }
   }
 
   /**