diff --git a/src/libstore/include/nix/store/remote-fs-accessor.hh b/src/libstore/include/nix/store/remote-fs-accessor.hh
index 6e3aef335dd..62f4dd72602 100644
--- a/src/libstore/include/nix/store/remote-fs-accessor.hh
+++ b/src/libstore/include/nix/store/remote-fs-accessor.hh
@@ -11,7 +11,17 @@ class RemoteFSAccessor : public SourceAccessor
 {
     ref<Store> store;
 
-    std::map<std::string, ref<SourceAccessor>> nars;
+    /**
+     * Map from store path hash part to NAR hash. Used to then look up
+     * in `nars`. The indirection allows avoiding opening multiple
+     * redundant NAR accessors for the same NAR.
+     */
+    std::map<std::string, Hash, std::less<>> narHashes;
+
+    /**
+     * Map from NAR hash to NAR accessor.
+     */
+    std::map<Hash, ref<SourceAccessor>> nars;
 
     bool requireValidPath;
 
@@ -21,9 +31,7 @@ class RemoteFSAccessor : public SourceAccessor
 
     friend struct BinaryCacheStore;
 
-    std::filesystem::path makeCacheFile(std::string_view hashPart, const std::string & ext);
-
-    ref<SourceAccessor> addToCache(std::string_view hashPart, std::string && nar);
+    std::filesystem::path makeCacheFile(const Hash & narHash, const std::string & ext);
 
 public:
 
diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc
index 0fd0945da02..1af82d279ce 100644
--- a/src/libstore/remote-fs-accessor.cc
+++ b/src/libstore/remote-fs-accessor.cc
@@ -18,40 +18,15 @@ RemoteFSAccessor::RemoteFSAccessor(
         createDirs(*cacheDir);
 }
 
-std::filesystem::path RemoteFSAccessor::makeCacheFile(std::string_view hashPart, const std::string & ext)
+std::filesystem::path RemoteFSAccessor::makeCacheFile(const Hash & narHash, const std::string & ext)
 {
     assert(cacheDir);
-    auto res = (*cacheDir / hashPart);
-    res.concat(concatStrings(".", ext));
+    auto res = *cacheDir / narHash.to_string(HashFormat::Nix32, false);
+    res += ".";
+    res += ext;
     return res;
 }
 
-ref<SourceAccessor> RemoteFSAccessor::addToCache(std::string_view hashPart, std::string && nar)
-{
-    if (cacheDir) {
-        try {
-            /* FIXME: do this asynchronously. */
-            writeFile(makeCacheFile(hashPart, "nar"), nar);
-        } catch (...) {
-            ignoreExceptionExceptInterrupt();
-        }
-    }
-
-    auto narAccessor = makeNarAccessor(std::move(nar));
-    nars.emplace(hashPart, narAccessor);
-
-    if (cacheDir) {
-        try {
-            nlohmann::json j = listNarDeep(*narAccessor, CanonPath::root);
-            writeFile(makeCacheFile(hashPart, "ls"), j.dump());
-        } catch (...) {
-            ignoreExceptionExceptInterrupt();
-        }
-    }
-
-    return narAccessor;
-}
-
 std::pair<ref<SourceAccessor>, CanonPath> RemoteFSAccessor::fetch(const CanonPath & path)
 {
     auto [storePath, restPath] = store->toStorePath(store->storeDir + path.abs());
@@ -62,37 +37,65 @@ std::pair<ref<SourceAccessor>, CanonPath> RemoteFSAccessor::fetch(const CanonPat
 
 std::shared_ptr<SourceAccessor> RemoteFSAccessor::accessObject(const StorePath & storePath)
 {
-    auto i = nars.find(std::string(storePath.hashPart()));
-    if (i != nars.end())
-        return i->second;
+    if (auto * narHash = get(narHashes, storePath.hashPart())) {
+        if (auto * accessor = get(nars, *narHash))
+            return *accessor;
+    }
 
-    std::string listing;
-    std::filesystem::path cacheFile;
+    auto info = store->queryPathInfo(storePath);
 
-    if (cacheDir && nix::pathExists(cacheFile = makeCacheFile(storePath.hashPart(), "nar"))) {
+    auto cacheAccessor = [&](ref<SourceAccessor> accessor) {
+        narHashes.emplace(storePath.hashPart(), info->narHash);
+        nars.emplace(info->narHash, accessor);
+        return accessor;
+    };
 
-        try {
-            listing = nix::readFile(makeCacheFile(storePath.hashPart(), "ls"));
-            auto listingJson = nlohmann::json::parse(listing);
-            auto narAccessor = makeLazyNarAccessor(listingJson, seekableGetNarBytes(cacheFile));
+    auto getNar = [&]() {
+        StringSink sink;
+        store->narFromPath(storePath, sink);
+        return std::move(sink.s);
+    };
 
-            nars.emplace(storePath.hashPart(), narAccessor);
-            return narAccessor;
+    if (cacheDir) {
+        auto cacheFile = makeCacheFile(info->narHash, "nar");
+        auto listingFile = makeCacheFile(info->narHash, "ls");
+
+        if (nix::pathExists(cacheFile)) {
+            try {
+                auto listing = nix::readFile(listingFile);
+                auto listingJson = nlohmann::json::parse(listing);
+                return cacheAccessor(makeLazyNarAccessor(listingJson, seekableGetNarBytes(cacheFile)));
+            } catch (SystemError &) {
+            }
+
+            try {
+                return cacheAccessor(makeNarAccessor(nix::readFile(cacheFile)));
+            } catch (SystemError &) {
+            }
+        }
 
-        } catch (SystemError &) {
+        auto nar = getNar();
+
+        try {
+            /* FIXME: do this asynchronously. */
+            writeFile(cacheFile, nar);
+        } catch (...) {
+            ignoreExceptionExceptInterrupt();
         }
 
+        auto narAccessor = makeNarAccessor(std::move(nar));
+
         try {
-            auto narAccessor = makeNarAccessor(nix::readFile(cacheFile));
-            nars.emplace(storePath.hashPart(), narAccessor);
-            return narAccessor;
-        } catch (SystemError &) {
+            nlohmann::json j = listNarDeep(*narAccessor, CanonPath::root);
+            writeFile(listingFile, j.dump());
+        } catch (...) {
+            ignoreExceptionExceptInterrupt();
         }
+
+        return cacheAccessor(narAccessor);
     }
 
-    StringSink sink;
-    store->narFromPath(storePath, sink);
-    return addToCache(storePath.hashPart(), std::move(sink.s));
+    return cacheAccessor(makeNarAccessor(getNar()));
 }
 
 std::optional<SourceAccessor::Stat> RemoteFSAccessor::maybeLstat(const CanonPath & path)