Reload config when reading from a JSON file

Author: franzpoeschel

include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp

@@ -410,8 +410,15 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl
 
     std::string m_originalExtension;
 
+    /*
+     * In read mode, we can only open the external block storage backend upon
+     * opening the JSON file, because it contains meta information relevant
+     * for configuring the backend.
+     */
+    std::optional<openPMD::json::TracingJSON>
+        m_deferredExternalBlockstorageConfig;
     DatasetMode_s m_datasetMode;
-    DatasetMode_s retrieveDatasetMode(openPMD::json::TracingJSON &config) const;
+    DatasetMode_s retrieveDatasetMode(openPMD::json::TracingJSON &config);
 
     AttributeMode_s m_attributeMode;
     AttributeMode_s

src/IO/JSON/JSONIOHandlerImpl.cpp

@@ -24,6 +24,7 @@
 #include "openPMD/Error.hpp"
 #include "openPMD/IO/AbstractIOHandler.hpp"
 #include "openPMD/IO/AbstractIOHandlerImpl.hpp"
+#include "openPMD/IO/Access.hpp"
 #include "openPMD/ThrowError.hpp"
 #include "openPMD/auxiliary/Filesystem.hpp"
 #include "openPMD/auxiliary/JSONMatcher.hpp"
@@ -349,8 +350,8 @@ namespace
 
         auto get_mandatory = [&](char const *key,
                                  bool lowercase) -> std::string {
-            auto const &val = *optionalOrElse(
-                get_key("mode"), [&]() -> nlohmann::json const * {
+            auto const &val =
+                *optionalOrElse(get_key(key), [&]() -> nlohmann::json const * {
                     throw error::BackendConfigSchema(
                         {configLocation, "mode", key}, "Mandatory key.");
                 });
@@ -394,7 +395,7 @@ namespace
             }
             static_cast<decltype(then)>(then)(val.get<bool>());
         };
-        auto modeString = get_mandatory("type", true);
+        auto modeString = get_mandatory("provider", true);
 
         if (modeString == "stdio")
         {
@@ -465,8 +466,8 @@ namespace
     }
 } // namespace
 
-auto JSONIOHandlerImpl::retrieveDatasetMode(
-    openPMD::json::TracingJSON &config) const -> DatasetMode_s
+auto JSONIOHandlerImpl::retrieveDatasetMode(openPMD::json::TracingJSON &config)
+    -> DatasetMode_s
 {
     // start with / copy from current config
     auto res = m_datasetMode;
@@ -484,8 +485,20 @@ auto JSONIOHandlerImpl::retrieveDatasetMode(
                 auto mode = datasetConfig["mode"];
                 if (mode.json().is_object())
                 {
-                    parse_external_mode(
-                        std::move(mode), std::nullopt, configLocation, res);
+                    if (access::writeOnly(m_handler->m_backendAccess))
+                    {
+                        parse_external_mode(
+                            std::move(mode), std::nullopt, configLocation, res);
+                    }
+                    else
+                    {
+                        // sic! initialize the deferred json config as a new
+                        // tracing object
+                        m_deferredExternalBlockstorageConfig =
+                            std::make_optional<openPMD::json::TracingJSON>(
+                                config.json(), config.originallySpecifiedAs);
+                        config.declareFullyRead();
+                    }
                 }
                 else
                 {
@@ -661,6 +674,14 @@ void JSONIOHandlerImpl::createFile(
         access::write(m_handler->m_backendAccess),
         "[JSON] Creating a file in read-only mode is not possible.");
 
+    if (m_deferredExternalBlockstorageConfig.has_value())
+    {
+        throw error::Internal(
+            "Creation of external block storage backend was deferred until "
+            "opening the first file, but a file is created before any was "
+            "opened.");
+    }
+
     /*
      * Need to resolve this later than init() since the openPMD version might be
      * specified after the creation of the IOHandler.
@@ -1104,6 +1125,28 @@ void JSONIOHandlerImpl::openFile(
 
     auto file = std::get<0>(getPossiblyExisting(name));
 
+    if (m_deferredExternalBlockstorageConfig.has_value())
+    {
+        auto const &contents = obtainJsonContents(file);
+        auto previousConfig = [&]() -> std::optional<nlohmann::json const *> {
+            if (contents->contains("external_storage"))
+            {
+                return std::make_optional<nlohmann::json const *>(
+                    &contents->at("external_storage"));
+            }
+            else
+            {
+                return std::nullopt;
+            }
+        }();
+        parse_external_mode(
+            std::move(*m_deferredExternalBlockstorageConfig),
+            previousConfig,
+            backendConfigKey(),
+            m_datasetMode);
+        m_attributeMode.m_specificationVia = SpecificationVia::Manually;
+    }
+
     associateWithFile(writable, file);
 
     writable->written = true;
@@ -2242,6 +2285,9 @@ JSONIOHandlerImpl::obtainJsonContents(File const &file)
     auto res = serialImplementation();
 #endif
 
+    bool initialize_external_block_storage =
+        m_deferredExternalBlockstorageConfig.has_value();
+
     if (res->contains(JSONDefaults::openpmd_internal))
     {
         auto const &openpmd_internal = res->at(JSONDefaults::openpmd_internal);
@@ -2272,6 +2318,10 @@ JSONIOHandlerImpl::obtainJsonContents(File const &file)
             {
                 m_datasetMode.m_mode = DatasetMode::Template;
             }
+            else if (modeOption.value() == "external")
+            {
+                initialize_external_block_storage = true;
+            }
             else
             {
                 std::cerr << "[JSON/TOML backend] Warning: Invalid value '"
@@ -2315,6 +2365,31 @@ JSONIOHandlerImpl::obtainJsonContents(File const &file)
             }
         }
     }
+
+    if (initialize_external_block_storage)
+    {
+        auto previousConfig = [&]() -> std::optional<nlohmann::json const *> {
+            if (res->contains("external_storage"))
+            {
+                return std::make_optional<nlohmann::json const *>(
+                    &res->at("external_storage"));
+            }
+            else
+            {
+                return std::nullopt;
+            }
+        }();
+        parse_external_mode(
+            m_deferredExternalBlockstorageConfig.has_value()
+                ? std::move(*m_deferredExternalBlockstorageConfig)
+                : openPMD::json::TracingJSON(),
+            previousConfig,
+            backendConfigKey(),
+            m_datasetMode);
+        m_attributeMode.m_specificationVia = SpecificationVia::Manually;
+        m_deferredExternalBlockstorageConfig.reset();
+    }
+
     m_jsonVals.emplace(file, res);
     return res;
 }