diff --git a/lib/stores/filesystem.js b/lib/stores/filesystem.js
index 576da202..9dfc9def 100644
--- a/lib/stores/filesystem.js
+++ b/lib/stores/filesystem.js
@@ -377,14 +377,14 @@ class FilesystemStore {
     const parts = key.split('/');
     // the last part isn't a directory (it's embedded into the file name)
     parts.pop();
-    while (
-      parts.length &&
-      !readdirSync(path.join(bucketPath, ...parts)).length
-    ) {
-      await fs.rmdir(path.join(bucketPath, ...parts));
+    while (parts.length) {
+      await fs.rmdir(path.join(bucketPath, ...parts)).catch(err => {
+        if (err.code !== 'ENOENT' && err.code !== 'ENOTEMPTY') throw err;
+        parts.length = 0;
+      });
       parts.pop();
     }
-  }
+}
 
   async initiateUpload(bucket, key, uploadId, metadata) {
     const uploadDir = path.join(