Untested read impl

Author: franzpoeschel

include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp

@@ -177,6 +177,21 @@ struct JsonDatatypeHandling
         }
     }
 
+    template <typename T_required>
+    static auto checkDatatype(nlohmann::json const &j) -> bool
+    {
+        auto const &needed_datatype =
+            jsonDatatypeToString(determineDatatype<T_required>());
+        if (auto it = j.find("datatype"); it != j.end())
+        {
+            return it.value().get<std::string>() == needed_datatype;
+        }
+        else
+        {
+            return false;
+        }
+    }
+
     template <typename Functor, typename... Args>
     static auto decodeDatatype(nlohmann::json const &j, Args &&...args) -> bool
     {

include/openPMD/toolkit/ExternalBlockStorage.hpp

@@ -46,6 +46,13 @@ struct DatatypeHandling_Interface
     template <typename T>
     static auto encodeDatatype(nlohmann::json &) -> bool;
 
+    /*
+     * Returns false if the encoded datatype does not match T_required
+     * or if no datatype has been encoded.
+     */
+    template <typename T_required>
+    static auto checkDatatype(nlohmann::json const &j) -> bool;
+
     /*
      * Returns false if no encoded datatype could be found
      */

src/toolkit/Aws.cpp

@@ -67,7 +67,8 @@ auto ExternalBlockStorageAws::put(
 
     auto input_data = Aws::MakeShared<imemstream>(
         "PutObjectInputStream", reinterpret_cast<char const *>(data), len);
-    std::static_pointer_cast<Aws::IOStream>(input_data);
+    put_request.SetBody(input_data);
+    put_request.SetContentLength(static_cast<long long>(len));
 
     auto put_outcome = m_client.PutObject(put_request);
 

src/toolkit/ExternalBlockStorage.cpp

@@ -2,6 +2,7 @@
 
 #include "openPMD/DatatypeMacros.hpp"
 #include "openPMD/IO/JSON/JSONIOHandlerImpl.hpp"
+#include "openPMD/auxiliary/StringManip.hpp"
 
 #include <nlohmann/json.hpp>
 
@@ -16,6 +17,30 @@ ExternalBlockStorageBackend::~ExternalBlockStorageBackend() = default;
 
 namespace openPMD
 {
+
+namespace
+{
+    auto flat_extent(Extent const &e) -> size_t
+    {
+        return std::accumulate(
+            e.begin(), e.end(), 1, [](size_t left, size_t right) {
+                return left * right;
+            });
+    }
+
+    template <typename T>
+    void read_impl(
+        internal::ExternalBlockStorageBackend *backend,
+        nlohmann::json const &external_block,
+        T *data,
+        size_t len)
+    {
+        auto const &external_ref =
+            external_block.at("external_ref").get<std::string>();
+        backend->get(external_ref, data, sizeof(T) * len);
+    }
+} // namespace
+
 ExternalBlockStorage::ExternalBlockStorage() = default;
 ExternalBlockStorage::ExternalBlockStorage(
     std::unique_ptr<internal::ExternalBlockStorageBackend> worker)
@@ -125,25 +150,11 @@ auto ExternalBlockStorage::store(
     auto escaped_filesystem_identifier = m_worker->put(
         filesystem_identifier.str(),
         data,
-        std::accumulate(
-            blockExtent.begin(),
-            blockExtent.end(),
-            sizeof(T),
-            [](size_t left, size_t right) { return left * right; }));
+        sizeof(T) * flat_extent(blockExtent));
     block["external_ref"] = escaped_filesystem_identifier;
     return index_as_str;
 }
 
-namespace
-{
-    template <typename T>
-    void read_impl(
-        internal::ExternalBlockStorageBackend *backend,
-        nlohmann::json const &external_block,
-        T *data)
-    {}
-} // namespace
-
 template <typename DatatypeHandling, typename T>
 void ExternalBlockStorage::read(
     std::string const &identifier,
@@ -161,6 +172,43 @@ void ExternalBlockStorage::read(
     T *data)
 {
     auto &dataset = fullJsonDataset[path];
+    if (!DatatypeHandling::template checkDatatype<T>(dataset))
+    {
+        throw std::runtime_error("Inconsistent chunk storage in datatype.");
+    }
+    auto external_blocks = dataset["external_blocks"];
+    bool found_a_precise_match = false;
+    for (auto it = external_blocks.begin(); it != external_blocks.end(); ++it)
+    {
+        auto const &block = it.value();
+        try
+        {
+            auto const &o = block.at("offset").get<Offset>();
+            auto const &e = block.at("extent").get<Extent>();
+            // Look only for exact matches for now
+            if (o != blockOffset || e != blockExtent)
+            {
+                continue;
+            }
+            found_a_precise_match = true;
+            read_impl(m_worker.get(), block, data, flat_extent(blockExtent));
+            break;
+        }
+        catch (nlohmann::json::exception const &e)
+        {
+            std::cerr << "[ExternalBlockStorage::read] Could not parse block '"
+                      << it.key() << "'. Original error was:\n"
+                      << e.what();
+        }
+    }
+    if (!found_a_precise_match)
+    {
+        throw std::runtime_error(
+            "[ExternalBlockStorage::read] Unable to find a precise match for "
+            "offset " +
+            auxiliary::vec_as_string(blockOffset) + " and extent " +
+            auxiliary::vec_as_string(blockExtent));
+    }
 }
 
 [[nodiscard]] auto ExternalBlockStorage::externalStorageLocation() const