diff --git a/core/crypto/CMakeLists.txt b/core/crypto/CMakeLists.txt index 67b1d89a67..db44902eb3 100644 --- a/core/crypto/CMakeLists.txt +++ b/core/crypto/CMakeLists.txt @@ -5,6 +5,7 @@ add_subdirectory(blake2) add_subdirectory(bls) +add_subdirectory(hasher) add_subdirectory(murmur) add_subdirectory(randomness) add_subdirectory(vrf) diff --git a/core/crypto/hasher/CMakeLists.txt b/core/crypto/hasher/CMakeLists.txt new file mode 100644 index 0000000000..2a1c7b8403 --- /dev/null +++ b/core/crypto/hasher/CMakeLists.txt @@ -0,0 +1,11 @@ +# +# Copyright Soramitsu Co., Ltd. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +# + +add_library(filecoin_hasher + hasher.cpp + ) +target_link_libraries(filecoin_hasher + blake2 + ) diff --git a/core/crypto/hasher/hasher.cpp b/core/crypto/hasher/hasher.cpp new file mode 100644 index 0000000000..1c5b34f751 --- /dev/null +++ b/core/crypto/hasher/hasher.cpp @@ -0,0 +1,37 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "hasher.hpp" + +#include +#include "crypto/blake2/blake2b160.hpp" + +namespace fc::crypto { + std::map Hasher::methods_{ + {HashType::sha256, Hasher::sha2_256}, + {HashType::blake2b_256, Hasher::blake2b_256}}; + + Hasher::Multihash Hasher::calculate(HashType hash_type, + gsl::span buffer) { + HashMethod hash_method = methods_.at(hash_type); + return std::invoke(hash_method, buffer); + } + + Hasher::Multihash Hasher::sha2_256(gsl::span buffer) { + auto digest = libp2p::crypto::sha256(buffer); + auto multi_hash = Multihash::create(HashType::sha256, digest); + BOOST_ASSERT_MSG(multi_hash.has_value(), + "fc::crypto::Hasher - failed to create sha2-256 hash"); + return multi_hash.value(); + } + + Hasher::Multihash Hasher::blake2b_256(gsl::span buffer) { + auto digest = crypto::blake2b::blake2b_256(buffer); + auto multi_hash = Multihash::create(HashType::blake2b_256, digest); + BOOST_ASSERT_MSG(multi_hash.has_value(), + "fc::crypto::Hasher - failed to create blake2b_256 hash"); + return multi_hash.value(); + } +} // namespace fc::crypto diff --git a/core/crypto/hasher/hasher.hpp b/core/crypto/hasher/hasher.hpp new file mode 100644 index 0000000000..7c687b03d2 --- /dev/null +++ b/core/crypto/hasher/hasher.hpp @@ -0,0 +1,48 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef FILECOIN_CORE_CRYPTO_HASHER_HPP +#define FILECOIN_CORE_CRYPTO_HASHER_HPP + +#include + +#include + +namespace fc::crypto { + /** + * @class Supported methods: + * sha2-256 + * blakeb2-256 + */ + class Hasher { + protected: + using HashType = libp2p::multi::HashType; + using Multihash = libp2p::multi::Multihash; + using HashMethod = Multihash (*)(gsl::span); + + private: + static std::map methods_; + + public: + static Multihash calculate(HashType hash_type, + gsl::span buffer); + + /** + * @brief Calculate SHA2-256 hash + * @param buffer - source data + * @return SHA2-256 hash + */ + static Multihash sha2_256(gsl::span buffer); + + /** + * @brief Calculate Blake2b-256 hash + * @param buffer - source data + * @return Blake2b-256 hash + */ + static Multihash blake2b_256(gsl::span buffer); + }; +} // namespace fc::crypto + +#endif diff --git a/core/primitives/CMakeLists.txt b/core/primitives/CMakeLists.txt index fbd2d9a559..38b48d123f 100644 --- a/core/primitives/CMakeLists.txt +++ b/core/primitives/CMakeLists.txt @@ -8,7 +8,6 @@ add_subdirectory(block) add_subdirectory(chain) add_subdirectory(chain_epoch) add_subdirectory(cid) -add_subdirectory(persistent_block) add_subdirectory(rle_bitset) add_subdirectory(ticket) add_subdirectory(tipset) diff --git a/core/primitives/persistent_block/CMakeLists.txt b/core/primitives/persistent_block/CMakeLists.txt deleted file mode 100644 index 6e84492bc0..0000000000 --- a/core/primitives/persistent_block/CMakeLists.txt +++ /dev/null @@ -1,11 +0,0 @@ -# Copyright Soramitsu Co., Ltd. All Rights Reserved. -# SPDX-License-Identifier: Apache-2.0 - -add_library(persistent_block - persistent_block.cpp - ) -target_link_libraries(persistent_block - buffer - cbor - cid - ) diff --git a/core/primitives/persistent_block/persistent_block.cpp b/core/primitives/persistent_block/persistent_block.cpp deleted file mode 100644 index b6d4680c65..0000000000 --- a/core/primitives/persistent_block/persistent_block.cpp +++ /dev/null @@ -1,22 +0,0 @@ -/** - * Copyright Soramitsu Co., Ltd. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include "primitives/persistent_block/persistent_block.hpp" - -namespace fc::primitives::blockchain::block { - PersistentBlock::PersistentBlock(CID cid, gsl::span data) - : cid_(std::move(cid)) { - data_.put(data); - } - - const CID &PersistentBlock::getCID() const { - return cid_; - } - - const common::Buffer &PersistentBlock::getRawBytes() const { - return data_; - } - -} // namespace fc::primitives::blockchain::block diff --git a/core/primitives/persistent_block/persistent_block.hpp b/core/primitives/persistent_block/persistent_block.hpp deleted file mode 100644 index 37067aed02..0000000000 --- a/core/primitives/persistent_block/persistent_block.hpp +++ /dev/null @@ -1,28 +0,0 @@ -/** - * Copyright Soramitsu Co., Ltd. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -#ifndef CPP_FILECOIN_CORE_BLOCKCHAIN_BLOCK_PERSISTENT_BLOCK_HPP -#define CPP_FILECOIN_CORE_BLOCKCHAIN_BLOCK_PERSISTENT_BLOCK_HPP - -#include "storage/ipfs/ipfs_block.hpp" - -namespace fc::primitives::blockchain::block { - class PersistentBlock : public storage::ipfs::IpfsBlock { - public: - virtual ~PersistentBlock() override = default; - - PersistentBlock(CID cid, gsl::span data); - - const CID &getCID() const override; - - const common::Buffer &getRawBytes() const override; - - private: - CID cid_; - common::Buffer data_; - }; -} // namespace fc::blockchain::block - -#endif // CPP_FILECOIN_CORE_BLOCKCHAIN_BLOCK_PERSISTENT_BLOCK_HPP diff --git a/core/storage/CMakeLists.txt b/core/storage/CMakeLists.txt index f3e4f5640e..40e2b8d579 100644 --- a/core/storage/CMakeLists.txt +++ b/core/storage/CMakeLists.txt @@ -10,6 +10,7 @@ add_subdirectory(filestore) add_subdirectory(hamt) add_subdirectory(in_memory) add_subdirectory(ipfs) +add_subdirectory(ipld) add_subdirectory(keystore) add_subdirectory(leveldb) add_subdirectory(repository) diff --git a/core/storage/chain/CMakeLists.txt b/core/storage/chain/CMakeLists.txt index ae45847746..0620314607 100644 --- a/core/storage/chain/CMakeLists.txt +++ b/core/storage/chain/CMakeLists.txt @@ -9,7 +9,6 @@ target_link_libraries(chain_store datastore_key ipfs_blockservice logger - persistent_block ) add_library(datastore_key diff --git a/core/storage/ipfs/ipfs_block.hpp b/core/storage/ipfs/ipfs_block.hpp deleted file mode 100644 index 19efbea032..0000000000 --- a/core/storage/ipfs/ipfs_block.hpp +++ /dev/null @@ -1,39 +0,0 @@ -/** - * Copyright Soramitsu Co., Ltd. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -#ifndef FILECOIN_STORAGE_IPFS_BLOCK_HPP -#define FILECOIN_STORAGE_IPFS_BLOCK_HPP - -#include - -#include "common/buffer.hpp" -#include "common/outcome.hpp" -#include "primitives/cid/cid.hpp" - -namespace fc::storage::ipfs { - /** - * @interface Piece of data, which is used with BlockService - */ - struct IpfsBlock { - /** - * @brief Default destructor - */ - virtual ~IpfsBlock() = default; - - /** - * @brief Get content identifier - * @return Block CID - */ - virtual const CID &getCID() const = 0; - - /** - * @brief Get block content - * @return Block's raw data for store in the BlockService - */ - virtual const common::Buffer &getRawBytes() const = 0; - }; -} // namespace fc::storage::ipfs - -#endif diff --git a/core/storage/ipfs/merkledag/CMakeLists.txt b/core/storage/ipfs/merkledag/CMakeLists.txt index f1eac25dad..0400b3127a 100644 --- a/core/storage/ipfs/merkledag/CMakeLists.txt +++ b/core/storage/ipfs/merkledag/CMakeLists.txt @@ -1,34 +1,15 @@ -find_package(Protobuf REQUIRED) -set(PB_SCHEME "merkledag.proto") -set(PB_BUILD_DIR ${CMAKE_CURRENT_BINARY_DIR}) - -execute_process(COMMAND ${PROTOBUF_PROTOC_EXECUTABLE} --cpp_out=${PB_BUILD_DIR} ${PB_SCHEME} - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/impl/protobuf - RESULT_VARIABLE CMD_OUTPUT - ) - -add_library(ipfs_merkledag_service_protobuf - ${PB_BUILD_DIR}/merkledag.pb.h - ${PB_BUILD_DIR}/merkledag.pb.cc - ) -target_include_directories(ipfs_merkledag_service_protobuf PUBLIC ${PB_BUILD_DIR}) -target_link_libraries(ipfs_merkledag_service_protobuf - protobuf::libprotobuf - ) -disable_clang_tidy(ipfs_merkledag_service_protobuf) +# +# Copyright Soramitsu Co., Ltd. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +# add_library(ipfs_merkledag_service - impl/link_impl.cpp - impl/node_impl.cpp impl/merkledag_service_impl.cpp - impl/pb_node_encoder.cpp - impl/pb_node_decoder.cpp impl/leaf_impl.cpp ) target_link_libraries(ipfs_merkledag_service - cid Boost::boost + ipld_node ipfs_blockservice ipfs_datastore_in_memory - ipfs_merkledag_service_protobuf ) diff --git a/core/storage/ipfs/merkledag/impl/link_impl.cpp b/core/storage/ipfs/merkledag/impl/link_impl.cpp deleted file mode 100644 index e7bd7ff03a..0000000000 --- a/core/storage/ipfs/merkledag/impl/link_impl.cpp +++ /dev/null @@ -1,27 +0,0 @@ -/** - * Copyright Soramitsu Co., Ltd. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include "storage/ipfs/merkledag/impl/link_impl.hpp" - -namespace fc::storage::ipfs::merkledag { - LinkImpl::LinkImpl(libp2p::multi::ContentIdentifier id, - std::string name, - size_t size) - : cid_{std::move(id)}, - name_{std::move(name)}, - size_{size} {} - - const std::string &LinkImpl::getName() const { - return name_; - } - - const CID &LinkImpl::getCID() const { - return cid_; - } - - size_t LinkImpl::getSize() const { - return size_; - } -} // namespace fc::storage::ipfs::merkledag diff --git a/core/storage/ipfs/merkledag/impl/merkledag_service_impl.cpp b/core/storage/ipfs/merkledag/impl/merkledag_service_impl.cpp index ac3988f64d..c35bcad38e 100644 --- a/core/storage/ipfs/merkledag/impl/merkledag_service_impl.cpp +++ b/core/storage/ipfs/merkledag/impl/merkledag_service_impl.cpp @@ -7,11 +7,13 @@ #include #include -#include "storage/ipfs/merkledag/impl/node_impl.hpp" +#include "storage/ipld/impl/ipld_node_impl.hpp" using libp2p::multi::ContentIdentifierCodec; namespace fc::storage::ipfs::merkledag { + using ipld::IPLDNodeImpl; + MerkleDagServiceImpl::MerkleDagServiceImpl( std::shared_ptr service) : block_service_{std::move(service)} { @@ -20,14 +22,14 @@ namespace fc::storage::ipfs::merkledag { } outcome::result MerkleDagServiceImpl::addNode( - std::shared_ptr node) { + std::shared_ptr node) { return block_service_->set(node->getCID(), node->getRawBytes()); } - outcome::result> MerkleDagServiceImpl::getNode( + outcome::result> MerkleDagServiceImpl::getNode( const CID &cid) const { OUTCOME_TRY(content, block_service_->get(cid)); - return NodeImpl::createFromRawBytes(content); + return IPLDNodeImpl::createFromRawBytes(content); } outcome::result MerkleDagServiceImpl::removeNode(const CID &cid) { @@ -37,15 +39,15 @@ namespace fc::storage::ipfs::merkledag { outcome::result MerkleDagServiceImpl::select( gsl::span root_cid, gsl::span selector, - std::function)> handler) const { + std::function)> handler) const { std::ignore = selector; OUTCOME_TRY(content_id, ContentIdentifierCodec::decode(root_cid)); CID cid{std::move(content_id)}; OUTCOME_TRY(root_node, getNode(cid)); - std::vector> node_set{}; + std::vector> node_set{}; node_set.emplace_back(std::move(root_node)); const auto &links = node_set.front()->getLinks(); - for (const auto& link : links) { + for (const auto &link : links) { auto request = getNode(link.get().getCID()); if (request.has_error()) return ServiceError::UNRESOLVED_LINK; node_set.emplace_back(std::move(request.value())); @@ -79,7 +81,7 @@ namespace fc::storage::ipfs::merkledag { outcome::result MerkleDagServiceImpl::buildGraph( const std::shared_ptr &root, - const std::vector> &links, + const std::vector> &links, bool depth_limit, const size_t max_depth, size_t current_depth) const { @@ -89,7 +91,7 @@ namespace fc::storage::ipfs::merkledag { for (const auto &link : links) { auto request = getNode(link.get().getCID()); if (request.has_error()) return ServiceError::UNRESOLVED_LINK; - std::shared_ptr node = request.value(); + std::shared_ptr node = request.value(); auto child_leaf = std::make_shared(node->content()); auto build_result = buildGraph(child_leaf, node->getLinks(), diff --git a/core/storage/ipfs/merkledag/impl/merkledag_service_impl.hpp b/core/storage/ipfs/merkledag/impl/merkledag_service_impl.hpp index 88f93fab60..30b0c65a3c 100644 --- a/core/storage/ipfs/merkledag/impl/merkledag_service_impl.hpp +++ b/core/storage/ipfs/merkledag/impl/merkledag_service_impl.hpp @@ -11,8 +11,11 @@ #include "storage/ipfs/datastore.hpp" #include "storage/ipfs/merkledag/impl/leaf_impl.hpp" #include "storage/ipfs/merkledag/merkledag_service.hpp" +#include "storage/ipld/ipld_link.hpp" namespace fc::storage::ipfs::merkledag { + using ipld::IPLDLink; + class MerkleDagServiceImpl : public MerkleDagService { public: /** @@ -21,9 +24,10 @@ namespace fc::storage::ipfs::merkledag { */ explicit MerkleDagServiceImpl(std::shared_ptr service); - outcome::result addNode(std::shared_ptr node) override; + outcome::result addNode( + std::shared_ptr node) override; - outcome::result> getNode( + outcome::result> getNode( const CID &cid) const override; outcome::result removeNode(const CID &cid) override; @@ -31,7 +35,7 @@ namespace fc::storage::ipfs::merkledag { outcome::result select( gsl::span root_cid, gsl::span selector, - std::function node)> handler) + std::function node)> handler) const override; outcome::result> fetchGraph( @@ -55,7 +59,7 @@ namespace fc::storage::ipfs::merkledag { */ outcome::result buildGraph( const std::shared_ptr &root, - const std::vector> &links, + const std::vector> &links, bool depth_limit, size_t max_depth, size_t current_depth = 0) const; diff --git a/core/storage/ipfs/merkledag/impl/node_impl.cpp b/core/storage/ipfs/merkledag/impl/node_impl.cpp deleted file mode 100644 index f6165ff4a7..0000000000 --- a/core/storage/ipfs/merkledag/impl/node_impl.cpp +++ /dev/null @@ -1,133 +0,0 @@ -/** - * Copyright Soramitsu Co., Ltd. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include "storage/ipfs/merkledag/impl/node_impl.hpp" - -#include -#include -#include -#include -#include "merkledag.pb.h" -#include "storage/ipfs/merkledag/impl/pb_node_decoder.hpp" - -using libp2p::common::Hash256; -using libp2p::multi::ContentIdentifierCodec; -using libp2p::multi::HashType; -using libp2p::multi::MulticodecType; -using libp2p::multi::Multihash; -using merkledag::pb::PBLink; -using merkledag::pb::PBNode; -using Version = libp2p::multi::ContentIdentifier::Version; - -namespace fc::storage::ipfs::merkledag { - - size_t NodeImpl::size() const { - return getCachePB().size() + child_nodes_size_; - } - - void NodeImpl::assign(common::Buffer input) { - content_ = std::move(input); - pb_cache_ = boost::none; - } - - const common::Buffer &NodeImpl::content() const { - return content_; - } - - outcome::result NodeImpl::addChild(const std::string &name, - std::shared_ptr node) { - LinkImpl link{node->getCID(), name, node->size()}; - links_.emplace(name, std::move(link)); - pb_cache_ = boost::none; - child_nodes_size_ += node->size(); - return outcome::success(); - } - - outcome::result> NodeImpl::getLink( - const std::string &name) const { - if (auto index = links_.find(name); index != links_.end()) { - return index->second; - } - return NodeError::LINK_NOT_FOUND; - } - - void NodeImpl::removeLink(const std::string &link_name) { - if (auto index = links_.find(link_name); index != links_.end()) { - child_nodes_size_ -= index->second.getSize(); - links_.erase(index); - } - } - - const CID &NodeImpl::getCID() const { - if (!cid_) { - Hash256 digest = libp2p::crypto::sha256(getCachePB().toVector()); - auto multi_hash = Multihash::create(HashType::sha256, digest); - CID id{ - Version::V0, MulticodecType::DAG_PB, std::move(multi_hash.value())}; - cid_ = std::move(id); - } - return cid_.value(); - } - - const common::Buffer &NodeImpl::getRawBytes() const { - return getCachePB(); - } - - void NodeImpl::addLink(const Link &link) { - auto &link_impl = dynamic_cast(link); - links_.emplace(link.getName(), link_impl); - } - - std::vector> NodeImpl::getLinks() const { - std::vector> link_refs{}; - for (const auto &link : links_) { - link_refs.emplace_back(link.second); - } - return link_refs; - } - - std::shared_ptr NodeImpl::createFromString(const std::string &content) { - std::vector data{content.begin(), content.end()}; - auto node = std::make_shared(); - node->assign(common::Buffer{data}); - return node; - } - - outcome::result> NodeImpl::createFromRawBytes( - gsl::span input) { - PBNodeDecoder decoder; - if (auto result = decoder.decode(input); result.has_error()) { - return result.error(); - } - auto node = createFromString(decoder.getContent()); - for (size_t i = 0; i < decoder.getLinksCount(); ++i) { - std::vector link_cid_bytes{decoder.getLinkCID(i).begin(), - decoder.getLinkCID(i).end()}; - OUTCOME_TRY(link_cid, ContentIdentifierCodec::decode(link_cid_bytes)); - LinkImpl link{ - std::move(link_cid), decoder.getLinkName(i), decoder.getLinkSize(i)}; - node->addLink(link); - } - return node; - } - - const common::Buffer &NodeImpl::getCachePB() const { - if (!pb_cache_) { - pb_cache_ = PBNodeEncoder::encode(content_, links_); - } - return pb_cache_.value(); - } -} // namespace fc::storage::ipfs::merkledag - -OUTCOME_CPP_DEFINE_CATEGORY(fc::storage::ipfs::merkledag, NodeError, e) { - using fc::storage::ipfs::merkledag::NodeError; - switch (e) { - case (NodeError::LINK_NOT_FOUND): - return "MerkleDAG Node: link not exist"; - case (NodeError::INVALID_RAW_DATA): - return "MerkleDAG Node: failed to deserialize from incorrect raw bytes"; - } - return "MerkleDAG Node: unknown error"; -} diff --git a/core/storage/ipfs/merkledag/impl/node_impl.hpp b/core/storage/ipfs/merkledag/impl/node_impl.hpp deleted file mode 100644 index d1014d97b2..0000000000 --- a/core/storage/ipfs/merkledag/impl/node_impl.hpp +++ /dev/null @@ -1,67 +0,0 @@ -/** - * Copyright Soramitsu Co., Ltd. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -#ifndef FILECOIN_STORAGE_IPFS_MERKLEDAG_NODE_IMPL_HPP -#define FILECOIN_STORAGE_IPFS_MERKLEDAG_NODE_IMPL_HPP - -#include -#include -#include -#include - -#include -#include "primitives/cid/cid.hpp" -#include "storage/ipfs/merkledag/impl/link_impl.hpp" -#include "storage/ipfs/merkledag/impl/pb_node_encoder.hpp" -#include "storage/ipfs/merkledag/node.hpp" - -namespace fc::storage::ipfs::merkledag { - class NodeImpl : public Node { - public: - - size_t size() const override; - - void assign(common::Buffer input) override; - - const common::Buffer &content() const override; - - outcome::result addChild(const std::string &name, - std::shared_ptr node) override; - - outcome::result> getLink( - const std::string &name) const override; - - void removeLink(const std::string &name) override; - - const CID &getCID() const override; - - const common::Buffer &getRawBytes() const override; - - void addLink(const Link &link) override; - - std::vector> getLinks() const override; - - static std::shared_ptr createFromString(const std::string &content); - - static outcome::result> createFromRawBytes( - gsl::span input); - - private: - mutable boost::optional cid_{}; - common::Buffer content_; - std::map links_; - PBNodeEncoder pb_node_codec_; - size_t child_nodes_size_{}; - mutable boost::optional pb_cache_{boost::none}; - - /** - * @brief Check Protobuf-data cache status and generate new if needed - * @return Actual protobuf-cache - */ - const common::Buffer &getCachePB() const; - }; -} // namespace fc::storage::ipfs::merkledag - -#endif diff --git a/core/storage/ipfs/merkledag/impl/pb_node_decoder.cpp b/core/storage/ipfs/merkledag/impl/pb_node_decoder.cpp deleted file mode 100644 index c0363ebef9..0000000000 --- a/core/storage/ipfs/merkledag/impl/pb_node_decoder.cpp +++ /dev/null @@ -1,49 +0,0 @@ -/** - * Copyright Soramitsu Co., Ltd. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include "storage/ipfs/merkledag/impl/pb_node_decoder.hpp" - -namespace fc::storage::ipfs::merkledag { - outcome::result PBNodeDecoder::decode(gsl::span input) { - const uint8_t *raw_bytes = input.data(); - if (pb_node_.ParseFromArray(raw_bytes, input.size())) { - return outcome::success(); - } - return PBNodeDecodeError::INVALID_RAW_BYTES; - } - - const std::string &PBNodeDecoder::getContent() const { - return pb_node_.data(); - } - - size_t PBNodeDecoder::getLinksCount() const { - return pb_node_.links_size(); - } - - const std::string &PBNodeDecoder::getLinkName(size_t index) const { - return pb_node_.links(index).name(); - } - - const std::string &PBNodeDecoder::getLinkCID(size_t index) const { - return pb_node_.links(index).hash(); - } - - size_t PBNodeDecoder::getLinkSize(size_t index) const { - int size = pb_node_.links(index).tsize(); - return size < 0 ? 0 : static_cast(size); - } -} // namespace fc::storage::ipfs::merkledag - -OUTCOME_CPP_DEFINE_CATEGORY(fc::storage::ipfs::merkledag, - PBNodeDecodeError, - error) { - using fc::storage::ipfs::merkledag::PBNodeDecodeError; - switch (error) { - case (PBNodeDecodeError::INVALID_RAW_BYTES): - return "IPLD node Protobuf decoder: failed to deserialize from incorrect " - "raw bytes"; - } - return "IPLD node protobuf decoder: unknown error"; -} diff --git a/core/storage/ipfs/merkledag/merkledag_service.hpp b/core/storage/ipfs/merkledag/merkledag_service.hpp index 13b54f510a..2d89dd6da4 100644 --- a/core/storage/ipfs/merkledag/merkledag_service.hpp +++ b/core/storage/ipfs/merkledag/merkledag_service.hpp @@ -10,9 +10,11 @@ #include "common/outcome.hpp" #include "storage/ipfs/merkledag/leaf.hpp" -#include "storage/ipfs/merkledag/node.hpp" +#include "storage/ipld/ipld_node.hpp" namespace fc::storage::ipfs::merkledag { + using ipld::IPLDNode; + class MerkleDagService { public: /** @@ -25,14 +27,14 @@ namespace fc::storage::ipfs::merkledag { * @param node - entity to add * @return operation result */ - virtual outcome::result addNode(std::shared_ptr node) = 0; + virtual outcome::result addNode(std::shared_ptr node) = 0; /** * @brief Get node by id * @param cid - node id * @return operation result */ - virtual outcome::result> getNode( + virtual outcome::result> getNode( const CID &cid) const = 0; /** @@ -53,7 +55,7 @@ namespace fc::storage::ipfs::merkledag { virtual outcome::result select( gsl::span root_cid, gsl::span selector, - std::function node)> handler) + std::function node)> handler) const = 0; /** diff --git a/core/storage/ipld/CMakeLists.txt b/core/storage/ipld/CMakeLists.txt new file mode 100644 index 0000000000..0bba57f0f9 --- /dev/null +++ b/core/storage/ipld/CMakeLists.txt @@ -0,0 +1,52 @@ +# +#Copyright Soramitsu Co., Ltd.All Rights Reserved. +#SPDX - License - Identifier : Apache - 2.0 +# + +find_package(Protobuf REQUIRED) +set(PB_SCHEME "ipld_node.proto") +set(PB_BUILD_DIR ${CMAKE_CURRENT_BINARY_DIR}) + +execute_process(COMMAND ${PROTOBUF_PROTOC_EXECUTABLE} --cpp_out=${PB_BUILD_DIR} ${PB_SCHEME} + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/impl/protobuf + RESULT_VARIABLE CMD_OUTPUT + ) + +add_library(ipld_node_protobuf + ${PB_BUILD_DIR}/ipld_node.pb.h + ${PB_BUILD_DIR}/ipld_node.pb.cc + ) +target_include_directories(ipld_node_protobuf PUBLIC ${PB_BUILD_DIR}) +target_link_libraries(ipld_node_protobuf + protobuf::libprotobuf + ) +disable_clang_tidy(ipld_node_protobuf) + +add_library(ipld_block + impl/ipld_block_impl.cpp + ) +target_link_libraries(ipld_block + cid + filecoin_hasher + ) + +add_library(ipld_link + impl/ipld_link_impl.cpp + ) +target_link_libraries(ipld_link + cid + ) + + +add_library(ipld_node + impl/ipld_node_impl.cpp + impl/ipld_node_encoder_pb.cpp + impl/ipld_node_decoder_pb.cpp + ) +target_link_libraries(ipld_node + ipld_node_protobuf + ipld_link + ipld_block + Boost::boost + cid + ) diff --git a/core/storage/ipld/impl/ipld_block_impl.cpp b/core/storage/ipld/impl/ipld_block_impl.cpp new file mode 100644 index 0000000000..63b0a5db36 --- /dev/null +++ b/core/storage/ipld/impl/ipld_block_impl.cpp @@ -0,0 +1,40 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "storage/ipld/impl/ipld_block_impl.hpp" + +#include "crypto/hasher/hasher.hpp" + +namespace fc::storage::ipld { + + IPLDBlockImpl::IPLDBlockImpl(CID::Version version, + HashType hash_type, + ContentType content_type) + : cid_version_{version}, + cid_hash_type_{hash_type}, + content_type_{content_type} {} + + const CID &IPLDBlockImpl::getCID() const { + if (!cid_) { + auto multi_hash = + crypto::Hasher::calculate(cid_hash_type_, getRawBytes()); + CID id{cid_version_, content_type_, std::move(multi_hash)}; + cid_ = std::move(id); + } + return cid_.value(); + } + + const common::Buffer &IPLDBlockImpl::getRawBytes() const { + if (!raw_bytes_) { + raw_bytes_ = serialize(); + } + return raw_bytes_.value(); + } + + void IPLDBlockImpl::clearCache() const { + cid_ = boost::none; + raw_bytes_ = boost::none; + } +} // namespace fc::storage::ipld diff --git a/core/storage/ipld/impl/ipld_block_impl.hpp b/core/storage/ipld/impl/ipld_block_impl.hpp new file mode 100644 index 0000000000..7bf61c74b6 --- /dev/null +++ b/core/storage/ipld/impl/ipld_block_impl.hpp @@ -0,0 +1,58 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef FILECOIN_STORAGE_IPLD_BLOCK_IMPL_HPP +#define FILECOIN_STORAGE_IPLD_BLOCK_IMPL_HPP + +#include +#include + +#include +#include +#include +#include "common/buffer.hpp" +#include "primitives/cid/cid.hpp" +#include "storage/ipld/ipld_block.hpp" + +namespace fc::storage::ipld { + using libp2p::multi::HashType; + using ContentType = libp2p::multi::MulticodecType::Code; + + /** + * @class IPLD data structure implementation + */ + class IPLDBlockImpl : public virtual IPLDBlock { + public: + /** + * @brief Construct IPLD block + * @param version - CID version + * @param hash_type - CID hash type + * @param content_type - block content type + */ + IPLDBlockImpl(CID::Version version, + HashType hash_type, + ContentType content_type); + + const CID &getCID() const override; + + const common::Buffer &getRawBytes() const override; + + protected: + /** + * @brief Clear cached CID and serialized values + */ + void clearCache() const; + + private: + CID::Version cid_version_; + HashType cid_hash_type_; + ContentType content_type_; + + mutable boost::optional cid_; + mutable boost::optional raw_bytes_; + }; +} // namespace fc::storage::ipld + +#endif diff --git a/core/storage/ipld/impl/ipld_link_impl.cpp b/core/storage/ipld/impl/ipld_link_impl.cpp new file mode 100644 index 0000000000..6cacf10309 --- /dev/null +++ b/core/storage/ipld/impl/ipld_link_impl.cpp @@ -0,0 +1,25 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "storage/ipld/impl/ipld_link_impl.hpp" + +namespace fc::storage::ipld { + IPLDLinkImpl::IPLDLinkImpl(libp2p::multi::ContentIdentifier id, + std::string name, + size_t size) + : cid_{std::move(id)}, name_{std::move(name)}, size_{size} {} + + const std::string &IPLDLinkImpl::getName() const { + return name_; + } + + const CID &IPLDLinkImpl::getCID() const { + return cid_; + } + + size_t IPLDLinkImpl::getSize() const { + return size_; + } +} // namespace fc::storage::ipld diff --git a/core/storage/ipfs/merkledag/impl/link_impl.hpp b/core/storage/ipld/impl/ipld_link_impl.hpp similarity index 58% rename from core/storage/ipfs/merkledag/impl/link_impl.hpp rename to core/storage/ipld/impl/ipld_link_impl.hpp index 2b6da3c661..345f613f99 100644 --- a/core/storage/ipfs/merkledag/impl/link_impl.hpp +++ b/core/storage/ipld/impl/ipld_link_impl.hpp @@ -3,17 +3,17 @@ * SPDX-License-Identifier: Apache-2.0 */ -#ifndef FILECOIN_STORAGE_IPFS_MERKLEDAG_LINK_IMPL_HPP -#define FILECOIN_STORAGE_IPFS_MERKLEDAG_LINK_IMPL_HPP +#ifndef FILECOIN_STORAGE_IPLD_LINK_IMPL_HPP +#define FILECOIN_STORAGE_IPLD_LINK_IMPL_HPP #include #include #include -#include "storage/ipfs/merkledag/link.hpp" +#include "storage/ipld/ipld_link.hpp" -namespace fc::storage::ipfs::merkledag { - class LinkImpl : public Link { +namespace fc::storage::ipld { + class IPLDLinkImpl : public IPLDLink { public: /** * @brief Construct Link @@ -21,11 +21,11 @@ namespace fc::storage::ipfs::merkledag { * @param name - name of the target object * @param size - total size of the target object */ - LinkImpl(libp2p::multi::ContentIdentifier id, - std::string name, - size_t size); + IPLDLinkImpl(libp2p::multi::ContentIdentifier id, + std::string name, + size_t size); - LinkImpl() = default; + IPLDLinkImpl() = default; const std::string &getName() const override; @@ -38,6 +38,6 @@ namespace fc::storage::ipfs::merkledag { std::string name_; size_t size_{}; }; -} // namespace fc::storage::ipfs::merkledag +} // namespace fc::storage::ipld #endif diff --git a/core/storage/ipld/impl/ipld_node_decoder_pb.cpp b/core/storage/ipld/impl/ipld_node_decoder_pb.cpp new file mode 100644 index 0000000000..b6c72445c4 --- /dev/null +++ b/core/storage/ipld/impl/ipld_node_decoder_pb.cpp @@ -0,0 +1,48 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "storage/ipld/impl/ipld_node_decoder_pb.hpp" + +namespace fc::storage::ipld { + outcome::result IPLDNodeDecoderPB::decode( + gsl::span input) { + const uint8_t *raw_bytes = input.data(); + if (pb_node_.ParseFromArray(raw_bytes, input.size())) { + return outcome::success(); + } + return IPLDNodeDecoderPBError::INVALID_RAW_BYTES; + } + + const std::string &IPLDNodeDecoderPB::getContent() const { + return pb_node_.data(); + } + + size_t IPLDNodeDecoderPB::getLinksCount() const { + return pb_node_.links_size(); + } + + const std::string &IPLDNodeDecoderPB::getLinkName(size_t index) const { + return pb_node_.links(index).name(); + } + + const std::string &IPLDNodeDecoderPB::getLinkCID(size_t index) const { + return pb_node_.links(index).hash(); + } + + size_t IPLDNodeDecoderPB::getLinkSize(size_t index) const { + int size = pb_node_.links(index).tsize(); + return size < 0 ? 0 : static_cast(size); + } +} // namespace fc::storage::ipld + +OUTCOME_CPP_DEFINE_CATEGORY(fc::storage::ipld, IPLDNodeDecoderPBError, error) { + using fc::storage::ipld::IPLDNodeDecoderPBError; + switch (error) { + case (IPLDNodeDecoderPBError::INVALID_RAW_BYTES): + return "IPLD node Protobuf decoder: failed to deserialize from incorrect " + "raw bytes"; + } + return "IPLD node protobuf decoder: unknown error"; +} diff --git a/core/storage/ipfs/merkledag/impl/pb_node_decoder.hpp b/core/storage/ipld/impl/ipld_node_decoder_pb.hpp similarity index 78% rename from core/storage/ipfs/merkledag/impl/pb_node_decoder.hpp rename to core/storage/ipld/impl/ipld_node_decoder_pb.hpp index 9a36ce7b89..4577782ce1 100644 --- a/core/storage/ipfs/merkledag/impl/pb_node_decoder.hpp +++ b/core/storage/ipld/impl/ipld_node_decoder_pb.hpp @@ -3,21 +3,21 @@ * SPDX-License-Identifier: Apache-2.0 */ -#ifndef FILECOIN_STORAGE_IPFS_MERKLEDAG_PB_NODE_DECODER -#define FILECOIN_STORAGE_IPFS_MERKLEDAG_PB_NODE_DECODER +#ifndef FILECOIN_STORAGE_IPLD_NODE_PB_DECODER +#define FILECOIN_STORAGE_IPLD_NODE_PB_DECODER #include #include #include "common/buffer.hpp" #include "common/outcome.hpp" -#include "merkledag.pb.h" +#include "ipld_node.pb.h" //TODO: Sergey Kaprovich FIL-150 -namespace fc::storage::ipfs::merkledag { +namespace fc::storage::ipld { /** * @class Protobuf Node decoder */ - class PBNodeDecoder { + class IPLDNodeDecoderPB { public: /** * @brief Try to decode input bytes as Protobuf-encoded Node @@ -60,15 +60,15 @@ namespace fc::storage::ipfs::merkledag { size_t getLinkSize(size_t index) const; private: - ::merkledag::pb::PBNode pb_node_; + ::protobuf::ipld::node::PBNode pb_node_; }; /** * @enum Possible PBNodeDecoder errors */ - enum class PBNodeDecodeError { INVALID_RAW_BYTES = 1 }; + enum class IPLDNodeDecoderPBError { INVALID_RAW_BYTES = 1 }; } // namespace fc::storage::ipfs::merkledag -OUTCOME_HPP_DECLARE_ERROR(fc::storage::ipfs::merkledag, PBNodeDecodeError) +OUTCOME_HPP_DECLARE_ERROR(fc::storage::ipld, IPLDNodeDecoderPBError) #endif diff --git a/core/storage/ipfs/merkledag/impl/pb_node_encoder.cpp b/core/storage/ipld/impl/ipld_node_encoder_pb.cpp similarity index 84% rename from core/storage/ipfs/merkledag/impl/pb_node_encoder.cpp rename to core/storage/ipld/impl/ipld_node_encoder_pb.cpp index da9ec332c1..c27a96c5be 100644 --- a/core/storage/ipfs/merkledag/impl/pb_node_encoder.cpp +++ b/core/storage/ipld/impl/ipld_node_encoder_pb.cpp @@ -3,21 +3,21 @@ * SPDX-License-Identifier: Apache-2.0 */ -#include "storage/ipfs/merkledag/impl/pb_node_encoder.hpp" +#include "storage/ipld/impl/ipld_node_encoder_pb.hpp" #include #include -#include "merkledag.pb.h" +#include "ipld_node.pb.h" using google::protobuf::io::ArrayOutputStream; using google::protobuf::io::CodedOutputStream; -using merkledag::pb::PBLink; -using merkledag::pb::PBNode; +using protobuf::ipld::node::PBLink; +using protobuf::ipld::node::PBNode; -namespace fc::storage::ipfs::merkledag { - common::Buffer PBNodeEncoder::encode( +namespace fc::storage::ipld { + common::Buffer IPLDNodeEncoderPB::encode( const common::Buffer &content, - const std::map &links) { + const std::map &links) { common::Buffer data; std::vector links_pb = serializeLinks(links); std::vector content_pb = serializeContent(content); @@ -26,8 +26,8 @@ namespace fc::storage::ipfs::merkledag { return data; } - size_t PBNodeEncoder::getLinkLengthPB(const std::string &name, - const Link &link) { + size_t IPLDNodeEncoderPB::getLinkLengthPB(const std::string &name, + const IPLDLinkImpl &link) { size_t length{}; size_t cid_bytes_size = link.getCID().content_address.toBuffer().size(); length += cid_bytes_size; @@ -39,8 +39,8 @@ namespace fc::storage::ipfs::merkledag { return length; } - std::vector PBNodeEncoder::serializeLinks( - const std::map &links) { + std::vector IPLDNodeEncoderPB::serializeLinks( + const std::map &links) { // Calculate links size: size_t links_content_size{}; size_t links_headers_size{}; @@ -86,7 +86,7 @@ namespace fc::storage::ipfs::merkledag { return {}; } - std::vector PBNodeEncoder::serializeContent( + std::vector IPLDNodeEncoderPB::serializeContent( const common::Buffer &content) { size_t pb_length = getContentLengthPB(content); std::vector buffer(pb_length); @@ -102,7 +102,7 @@ namespace fc::storage::ipfs::merkledag { return buffer; } - size_t PBNodeEncoder::getContentLengthPB(const common::Buffer &content) { + size_t IPLDNodeEncoderPB::getContentLengthPB(const common::Buffer &content) { size_t length{}; if (!content.empty()) { length += sizeof(PBTag); @@ -112,7 +112,7 @@ namespace fc::storage::ipfs::merkledag { return length; } - PBNodeEncoder::PBTag PBNodeEncoder::createTag(PBFieldType type, + IPLDNodeEncoderPB::PBTag IPLDNodeEncoderPB::createTag(PBFieldType type, uint8_t order) { constexpr size_t pb_type_length = 3; uint8_t tag = (order << pb_type_length); diff --git a/core/storage/ipfs/merkledag/impl/pb_node_encoder.hpp b/core/storage/ipld/impl/ipld_node_encoder_pb.hpp similarity index 84% rename from core/storage/ipfs/merkledag/impl/pb_node_encoder.hpp rename to core/storage/ipld/impl/ipld_node_encoder_pb.hpp index 58fb97ac8e..3474e15b26 100644 --- a/core/storage/ipfs/merkledag/impl/pb_node_encoder.hpp +++ b/core/storage/ipld/impl/ipld_node_encoder_pb.hpp @@ -3,8 +3,8 @@ * SPDX-License-Identifier: Apache-2.0 */ -#ifndef FILECOIN_STORAGE_IPFS_MERKLEDAG_PB_NODE_ENCODER -#define FILECOIN_STORAGE_IPFS_MERKLEDAG_PB_NODE_ENCODER +#ifndef FILECOIN_STORAGE_IPLD_NODE_ENCODER_PB +#define FILECOIN_STORAGE_IPLD_NODE_ENCODER_PB #include #include @@ -13,10 +13,10 @@ #include #include "common/buffer.hpp" #include "common/outcome.hpp" -#include "storage/ipfs/merkledag/impl/link_impl.hpp" -#include "storage/ipfs/merkledag/node.hpp" +#include "storage/ipld/impl/ipld_link_impl.hpp" +#include "storage/ipld/ipld_node.hpp" -namespace fc::storage::ipfs::merkledag { +namespace fc::storage::ipld { /** * @class Protobuf-serializer for MerkleDAG Nodes * @details Order of parts of the Protobuf-serialized data is forced @@ -24,7 +24,7 @@ namespace fc::storage::ipfs::merkledag { * implementation * @warning Need to update serialization algorithm on Protobuf-scheme change */ - class PBNodeEncoder { + class IPLDNodeEncoderPB { public: /** * @brief Serialize Node @@ -33,7 +33,7 @@ namespace fc::storage::ipfs::merkledag { * @return Protobuf-encoded data */ static common::Buffer encode(const common::Buffer &content, - const std::map &links); + const std::map &links); private: using PBTag = uint8_t; @@ -61,7 +61,7 @@ namespace fc::storage::ipfs::merkledag { * @param link - child link * @return Number of bytes */ - static size_t getLinkLengthPB(const std::string &name, const Link &link); + static size_t getLinkLengthPB(const std::string &name, const IPLDLinkImpl &link); /** * @brief Calculate length of the serialized content @@ -76,7 +76,7 @@ namespace fc::storage::ipfs::merkledag { * @return Raw bytes */ static std::vector serializeLinks( - const std::map &links); + const std::map &links); /** * @brief Serialized Node's content diff --git a/core/storage/ipld/impl/ipld_node_impl.cpp b/core/storage/ipld/impl/ipld_node_impl.cpp new file mode 100644 index 0000000000..e84e865b2b --- /dev/null +++ b/core/storage/ipld/impl/ipld_node_impl.cpp @@ -0,0 +1,121 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "storage/ipld/impl/ipld_node_impl.hpp" + +#include +#include +#include +#include +#include "ipld_node.pb.h" +#include "storage/ipld/impl/ipld_node_decoder_pb.hpp" + +using libp2p::common::Hash256; +using libp2p::multi::ContentIdentifierCodec; +using libp2p::multi::HashType; +using libp2p::multi::MulticodecType; +using libp2p::multi::Multihash; +using protobuf::ipld::node::PBLink; +using protobuf::ipld::node::PBNode; +using Version = libp2p::multi::ContentIdentifier::Version; + +namespace fc::storage::ipld { + + IPLDNodeImpl::IPLDNodeImpl() + : IPLDBlockImpl{CID::Version::V0, HashType::sha256, ContentType::DAG_PB} { + } + + size_t IPLDNodeImpl::size() const { + return getRawBytes().size() + child_nodes_size_; + } + + void IPLDNodeImpl::assign(common::Buffer input) { + content_ = std::move(input); + clearCache(); + } + + const common::Buffer &IPLDNodeImpl::content() const { + return content_; + } + + outcome::result IPLDNodeImpl::addChild( + const std::string &name, std::shared_ptr node) { + IPLDLinkImpl link{node->getCID(), name, node->size()}; + links_.emplace(name, std::move(link)); + clearCache(); + child_nodes_size_ += node->size(); + return outcome::success(); + } + + outcome::result> IPLDNodeImpl::getLink( + const std::string &name) const { + if (auto index = links_.find(name); index != links_.end()) { + return index->second; + } + return IPLDNodeError::LINK_NOT_FOUND; + } + + void IPLDNodeImpl::removeLink(const std::string &link_name) { + if (auto index = links_.find(link_name); index != links_.end()) { + child_nodes_size_ -= index->second.getSize(); + links_.erase(index); + } + } + + void IPLDNodeImpl::addLink(const IPLDLink &link) { + auto &link_impl = dynamic_cast(link); + links_.emplace(link.getName(), link_impl); + } + + std::vector> IPLDNodeImpl::getLinks() + const { + std::vector> link_refs{}; + for (const auto &link : links_) { + link_refs.emplace_back(link.second); + } + return link_refs; + } + + std::shared_ptr IPLDNodeImpl::createFromString( + const std::string &content) { + std::vector data{content.begin(), content.end()}; + auto node = std::make_shared(); + node->assign(common::Buffer{data}); + return node; + } + + outcome::result> IPLDNodeImpl::createFromRawBytes( + gsl::span input) { + IPLDNodeDecoderPB decoder; + if (auto result = decoder.decode(input); result.has_error()) { + return result.error(); + } + auto node = createFromString(decoder.getContent()); + for (size_t i = 0; i < decoder.getLinksCount(); ++i) { + std::vector link_cid_bytes{decoder.getLinkCID(i).begin(), + decoder.getLinkCID(i).end()}; + OUTCOME_TRY(link_cid, ContentIdentifierCodec::decode(link_cid_bytes)); + IPLDLinkImpl link{ + std::move(link_cid), decoder.getLinkName(i), decoder.getLinkSize(i)}; + node->addLink(link); + } + return node; + } + + common::Buffer IPLDNodeImpl::serialize() const { + return IPLDNodeEncoderPB::encode(content_, links_); + } +} // namespace fc::storage::ipld + +OUTCOME_CPP_DEFINE_CATEGORY(fc::storage::ipld, IPLDNodeError, e) { + using fc::storage::ipld::IPLDNodeError; + switch (e) { + case (IPLDNodeError::LINK_NOT_FOUND): + return "MerkleDAG Node: link not exist"; + case (IPLDNodeError::INVALID_RAW_DATA): + return "MerkleDAG Node: failed to deserialize from incorrect raw bytes"; + } + return "MerkleDAG Node: unknown error"; +} diff --git a/core/storage/ipld/impl/ipld_node_impl.hpp b/core/storage/ipld/impl/ipld_node_impl.hpp new file mode 100644 index 0000000000..eaf8595ae6 --- /dev/null +++ b/core/storage/ipld/impl/ipld_node_impl.hpp @@ -0,0 +1,60 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef FILECOIN_STORAGE_IPLD_NODE_IMPL_HPP +#define FILECOIN_STORAGE_IPLD_NODE_IMPL_HPP + +#include +#include +#include +#include + +#include +#include "storage/ipld/impl/ipld_block_impl.hpp" +#include "storage/ipld/impl/ipld_link_impl.hpp" +#include "storage/ipld/impl/ipld_node_encoder_pb.hpp" +#include "storage/ipld/ipld_node.hpp" + +namespace fc::storage::ipld { + class IPLDNodeImpl : public IPLDNode, IPLDBlockImpl { + public: + IPLDNodeImpl(); + + size_t size() const override; + + void assign(common::Buffer input) override; + + const common::Buffer &content() const override; + + outcome::result addChild( + const std::string &name, std::shared_ptr node) override; + + outcome::result> getLink( + const std::string &name) const override; + + void removeLink(const std::string &name) override; + + void addLink(const IPLDLink &link) override; + + std::vector> getLinks() + const override; + + static std::shared_ptr createFromString( + const std::string &content); + + static outcome::result> createFromRawBytes( + gsl::span input); + + private: + common::Buffer content_; + std::map links_; + IPLDNodeEncoderPB pb_node_codec_; + size_t child_nodes_size_{}; + + common::Buffer serialize() const override; + }; +} // namespace fc::storage::ipld + +#endif diff --git a/core/storage/ipfs/merkledag/impl/protobuf/merkledag.proto b/core/storage/ipld/impl/protobuf/ipld_node.proto similarity index 83% rename from core/storage/ipfs/merkledag/impl/protobuf/merkledag.proto rename to core/storage/ipld/impl/protobuf/ipld_node.proto index 6fa949ca15..207d0b7ee6 100644 --- a/core/storage/ipfs/merkledag/impl/protobuf/merkledag.proto +++ b/core/storage/ipld/impl/protobuf/ipld_node.proto @@ -1,8 +1,8 @@ syntax = "proto2"; -package merkledag.pb; +package protobuf.ipld.node; -// An IPFS MerkleDAG Link +// An IPLD Node Link message PBLink { // multihash of the target object @@ -15,7 +15,7 @@ message PBLink { optional uint64 Tsize = 3; } -// An IPFS MerkleDAG Node +// An IPLD Node message PBNode { // refs to other objects diff --git a/core/storage/ipld/ipld_block.hpp b/core/storage/ipld/ipld_block.hpp new file mode 100644 index 0000000000..276d0159b6 --- /dev/null +++ b/core/storage/ipld/ipld_block.hpp @@ -0,0 +1,47 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef FILECOIN_STORAGE_IPLD_BLOCK_HPP +#define FILECOIN_STORAGE_IPLD_BLOCK_HPP + +#include "common/buffer.hpp" +#include "primitives/cid/cid.hpp" + +namespace fc::storage::ipld { + /** + * @class Interface for various data structure, linked with CID + */ + class IPLDBlock { + public: + /** + * @brief Destructor + */ + virtual ~IPLDBlock() = default; + + /** + * @brief Get CID of the data structure + * @return IPLD identificator + */ + virtual const CID &getCID() const = 0; + + /** + * @brief Get data structure bytes from cache or generate new + * @return Serialized value + */ + virtual const common::Buffer &getRawBytes() const = 0; + + protected: + /** + * @brief Optionally used for generating CID + * Children class must overload it to support structure serialization + * @return Serialized value + */ + virtual common::Buffer serialize() const { + return {}; + } + }; +} // namespace fc::storage::ipld + +#endif diff --git a/core/storage/ipfs/merkledag/link.hpp b/core/storage/ipld/ipld_link.hpp similarity index 79% rename from core/storage/ipfs/merkledag/link.hpp rename to core/storage/ipld/ipld_link.hpp index 4640d7f14f..3442266e1f 100644 --- a/core/storage/ipfs/merkledag/link.hpp +++ b/core/storage/ipld/ipld_link.hpp @@ -3,21 +3,21 @@ * SPDX-License-Identifier: Apache-2.0 */ -#ifndef FILECOIN_STORAGE_IPFS_MERKLEDAG_LINK_HPP -#define FILECOIN_STORAGE_IPFS_MERKLEDAG_LINK_HPP +#ifndef FILECOIN_STORAGE_IPLD_LINK_HPP +#define FILECOIN_STORAGE_IPLD_LINK_HPP #include #include #include "primitives/cid/cid.hpp" -namespace fc::storage::ipfs::merkledag { - class Link { +namespace fc::storage::ipld { + class IPLDLink { public: /** * @brief Destructor */ - virtual ~Link() = default; + virtual ~IPLDLink() = default; /** * @brief Get name of the link diff --git a/core/storage/ipfs/merkledag/node.hpp b/core/storage/ipld/ipld_node.hpp similarity index 66% rename from core/storage/ipfs/merkledag/node.hpp rename to core/storage/ipld/ipld_node.hpp index 776581e6bb..0b2f717504 100644 --- a/core/storage/ipfs/merkledag/node.hpp +++ b/core/storage/ipld/ipld_node.hpp @@ -3,25 +3,26 @@ * SPDX-License-Identifier: Apache-2.0 */ -#ifndef FILECOIN_STORAGE_IPFS_MERKLEDAG_NODE_HPP -#define FILECOIN_STORAGE_IPFS_MERKLEDAG_NODE_HPP +#ifndef FILECOIN_STORAGE_IPLD_NODE_HPP +#define FILECOIN_STORAGE_IPLD_NODE_HPP #include #include -#include #include +#include #include "common/buffer.hpp" #include "common/outcome.hpp" -#include "storage/ipfs/ipfs_block.hpp" -#include "storage/ipfs/merkledag/link.hpp" +#include "storage/ipld/ipld_block.hpp" +#include "storage/ipld/ipld_link.hpp" -namespace fc::storage::ipfs::merkledag { +namespace fc::storage::ipld { /** * @interface MerkleDAG service node */ - class Node : public IpfsBlock { + class IPLDNode : public virtual IPLDBlock { public: + virtual ~IPLDNode() = default; /** * @brief Total size of the data including the total sizes of references * @return Cumulative size in bytes @@ -47,15 +48,15 @@ namespace fc::storage::ipfs::merkledag { * @param node - child object to link * @return operation result */ - virtual outcome::result addChild(const std::string &name, - std::shared_ptr node) = 0; + virtual outcome::result addChild( + const std::string &name, std::shared_ptr node) = 0; /** * @brief Get particular link to the child node * @param name - id of the link * @return Requested link of error, if link not found */ - virtual outcome::result> getLink( + virtual outcome::result> getLink( const std::string &name) const = 0; /** @@ -69,18 +70,18 @@ namespace fc::storage::ipfs::merkledag { * @brief Insert link to the child node * @param link - object to add */ - virtual void addLink(const Link &link) = 0; + virtual void addLink(const IPLDLink &link) = 0; - virtual std::vector> getLinks() + virtual std::vector> getLinks() const = 0; }; /** * @class Possible Node errors */ - enum class NodeError : int { LINK_NOT_FOUND, INVALID_RAW_DATA }; -} // namespace fc::storage::ipfs::merkledag + enum class IPLDNodeError { LINK_NOT_FOUND = 1, INVALID_RAW_DATA }; +} // namespace fc::storage::ipld -OUTCOME_HPP_DECLARE_ERROR(fc::storage::ipfs::merkledag, NodeError) +OUTCOME_HPP_DECLARE_ERROR(fc::storage::ipld, IPLDNodeError) #endif diff --git a/test/core/primitives/CMakeLists.txt b/test/core/primitives/CMakeLists.txt index f67a04e80e..38a64aeb88 100644 --- a/test/core/primitives/CMakeLists.txt +++ b/test/core/primitives/CMakeLists.txt @@ -7,7 +7,6 @@ add_subdirectory(address) add_subdirectory(block) add_subdirectory(chain_epoch) add_subdirectory(cid) -add_subdirectory(persistent_block) add_subdirectory(rle_bitset) add_subdirectory(ticket) add_subdirectory(tipset) diff --git a/test/core/primitives/persistent_block/CMakeLists.txt b/test/core/primitives/persistent_block/CMakeLists.txt deleted file mode 100644 index 302faf1398..0000000000 --- a/test/core/primitives/persistent_block/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright Soramitsu Co., Ltd. All Rights Reserved. -# SPDX-License-Identifier: Apache-2.0 - -addtest(persistent_block_test - persistent_block_test.cpp - ) -target_link_libraries(persistent_block_test - persistent_block - ) diff --git a/test/core/primitives/persistent_block/persistent_block_test.cpp b/test/core/primitives/persistent_block/persistent_block_test.cpp deleted file mode 100644 index ee25455f52..0000000000 --- a/test/core/primitives/persistent_block/persistent_block_test.cpp +++ /dev/null @@ -1,41 +0,0 @@ -/** - * Copyright Soramitsu Co., Ltd. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include "primitives/persistent_block/persistent_block.hpp" - -#include -#include "common/buffer.hpp" -#include "primitives/cid/cid_of_cbor.hpp" -#include "testutil/outcome.hpp" - -using namespace fc::primitives; - -struct PersistentBlockTest : public ::testing::Test { - using PersistentBlock = blockchain::block::PersistentBlock; - void SetUp() override { - EXPECT_OUTCOME_TRUE(c1, cid::getCidOfCbor(std::string("cid1"))); - EXPECT_OUTCOME_TRUE(c2, cid::getCidOfCbor(std::string("cid2"))); - cid1 = c1; - cid2 = c2; - content1.put("value1"); - content2.put("value2"); - } - - fc::CID cid1; - fc::CID cid2; - - fc::common::Buffer content1; - fc::common::Buffer content2; -}; - -/** - * @brief simple test, ensuring that created block contains data - * it was created with - */ -TEST_F(PersistentBlockTest, CreatePersistentBlockSuccess) { - PersistentBlock pb(cid1, content1); - ASSERT_EQ(pb.getCID(), cid1); - ASSERT_EQ(pb.getRawBytes(), content1); -} diff --git a/test/core/storage/ipfs/ipfs_block_service_test.cpp b/test/core/storage/ipfs/ipfs_block_service_test.cpp index 758af0735f..3353f126e2 100644 --- a/test/core/storage/ipfs/ipfs_block_service_test.cpp +++ b/test/core/storage/ipfs/ipfs_block_service_test.cpp @@ -10,14 +10,14 @@ #include #include "common/outcome.hpp" #include "storage/ipfs/impl/in_memory_datastore.hpp" -#include "storage/ipfs/ipfs_block.hpp" +#include "storage/ipld/ipld_block.hpp" #include "testutil/outcome.hpp" using fc::CID; using fc::common::Buffer; using fc::common::getCidOf; using fc::storage::ipfs::InMemoryDatastore; -using fc::storage::ipfs::IpfsBlock; +using fc::storage::ipld::IPLDBlock; using fc::storage::ipfs::IpfsBlockService; /** @@ -25,7 +25,7 @@ using fc::storage::ipfs::IpfsBlockService; * This interface can be used by any data structure (like Node from * MerkleDAG service) and there are no single universal implementation */ -struct BlockTestImpl : IpfsBlock { +struct BlockTestImpl : IPLDBlock { Buffer content; /**< Raw data */ CID cid; /**< Block identifier */ diff --git a/test/core/storage/ipfs/merkledag/ipfs_merkledag_dataset.hpp b/test/core/storage/ipfs/merkledag/ipfs_merkledag_dataset.hpp index 5a6c8b398a..233cf2959f 100644 --- a/test/core/storage/ipfs/merkledag/ipfs_merkledag_dataset.hpp +++ b/test/core/storage/ipfs/merkledag/ipfs_merkledag_dataset.hpp @@ -7,12 +7,11 @@ #define CORE_STORAGE_IPFS_MERKLEDAG_NODE_LIBRARY #include -#include "storage/ipfs/merkledag/impl/node_impl.hpp" -#include "storage/ipfs/merkledag/node.hpp" +#include "storage/ipld/impl/ipld_node_impl.hpp" namespace dataset { - using Node = fc::storage::ipfs::merkledag::Node; - using NodeImpl = fc::storage::ipfs::merkledag::NodeImpl; + using fc::storage::ipld::IPLDNode; + using fc::storage::ipld::IPLDNodeImpl; using CIDCodec = libp2p::multi::ContentIdentifierCodec; /** @@ -21,8 +20,8 @@ namespace dataset { * @param from - child node * @return operation result */ - fc::outcome::result link(const std::shared_ptr &to, - const std::shared_ptr &from) { + fc::outcome::result link(const std::shared_ptr &to, + const std::shared_ptr &from) { OUTCOME_TRY(from_id, CIDCodec::toString(from->getCID())); EXPECT_OUTCOME_TRUE_1(to->addChild(from_id, from)); return fc::outcome::success(); @@ -32,16 +31,16 @@ namespace dataset { * @brief Generate node suite type A * @return Null node */ - std::vector> getTypeA() { - return {NodeImpl::createFromString("")}; + std::vector> getTypeA() { + return {IPLDNodeImpl::createFromString("")}; } /** * @brief Generate node suite type B * @return Node without child */ - std::vector> getTypeB() { - return {NodeImpl::createFromString("leve1_node1")}; + std::vector> getTypeB() { + return {IPLDNodeImpl::createFromString("leve1_node1")}; } /** @@ -51,9 +50,9 @@ namespace dataset { * | * [leve1_node1] */ - std::vector> getTypeC() { - auto root = NodeImpl::createFromString(""); - auto child_1 = NodeImpl::createFromString("leve1_node1"); + std::vector> getTypeC() { + auto root = IPLDNodeImpl::createFromString(""); + auto child_1 = IPLDNodeImpl::createFromString("leve1_node1"); EXPECT_OUTCOME_TRUE_1(link(root, child_1)); return {root, child_1}; } @@ -66,13 +65,13 @@ namespace dataset { * [leve2_node1] [leve2_node2] [leve2_node3] * */ - std::vector> getTypeD() { - auto root = NodeImpl::createFromString("leve1_node1"); - auto child_1 = NodeImpl::createFromString("leve2_node1"); + std::vector> getTypeD() { + auto root = IPLDNodeImpl::createFromString("leve1_node1"); + auto child_1 = IPLDNodeImpl::createFromString("leve2_node1"); EXPECT_OUTCOME_TRUE_1(link(root, child_1)); - auto child_2 = NodeImpl::createFromString("leve2_node2"); + auto child_2 = IPLDNodeImpl::createFromString("leve2_node2"); EXPECT_OUTCOME_TRUE_1(link(root, child_2)); - auto child_3 = NodeImpl::createFromString("leve2_node3"); + auto child_3 = IPLDNodeImpl::createFromString("leve2_node3"); EXPECT_OUTCOME_TRUE_1(link(root, child_3)); return {root, child_1, child_2, child_3}; } @@ -87,14 +86,14 @@ namespace dataset { * / | \ | * [leve2_node1] [leve2_node2] [leve2_node3]----- */ - std::vector> getTypeE() { - auto root = NodeImpl::createFromString(""); + std::vector> getTypeE() { + auto root = IPLDNodeImpl::createFromString(""); auto suite_D = getTypeD(); EXPECT_OUTCOME_TRUE_1(link(root, suite_D.front())); - auto child_1 = NodeImpl::createFromString("leve1_node2"); + auto child_1 = IPLDNodeImpl::createFromString("leve1_node2"); EXPECT_OUTCOME_TRUE_1(link(root, child_1)); EXPECT_OUTCOME_TRUE_1(link(root, suite_D.back())); - std::vector> suite_E{root, child_1}; + std::vector> suite_E{root, child_1}; suite_E.insert(suite_E.end(), suite_D.begin(), suite_D.end()); return suite_E; } diff --git a/test/core/storage/ipfs/merkledag/ipfs_merkledag_service_test.cpp b/test/core/storage/ipfs/merkledag/ipfs_merkledag_service_test.cpp index b63cfe9f11..5227713484 100644 --- a/test/core/storage/ipfs/merkledag/ipfs_merkledag_service_test.cpp +++ b/test/core/storage/ipfs/merkledag/ipfs_merkledag_service_test.cpp @@ -3,17 +3,17 @@ * SPDX-License-Identifier: Apache-2.0 */ -#include "storage/ipfs/merkledag/impl/node_impl.hpp" +#include "storage/ipfs/merkledag/impl/merkledag_service_impl.hpp" #include #include -#include #include #include "core/storage/ipfs/merkledag/ipfs_merkledag_dataset.hpp" -#include "storage/ipfs/impl/ipfs_block_service.hpp" #include "storage/ipfs/impl/in_memory_datastore.hpp" +#include "storage/ipfs/impl/ipfs_block_service.hpp" using namespace fc::storage::ipfs; +using namespace fc::storage::ipld; using namespace fc::storage::ipfs::merkledag; using libp2p::multi::ContentIdentifierCodec; @@ -22,7 +22,7 @@ using libp2p::multi::ContentIdentifierCodec; */ struct DataSample { // Various-linked MerkleDAG nodes [root, node_1, node_2 ... ] - std::vector> nodes; + std::vector> nodes; // Base58-encoded CID of the root node from the Lotus implementation std::string sample_cid; @@ -46,7 +46,8 @@ struct CommonFeaturesTest : public testing::TestWithParam { */ void SetUp() override { std::shared_ptr datastore{new InMemoryDatastore{}}; - std::shared_ptr blockservice{new IpfsBlockService{datastore}}; + std::shared_ptr blockservice{ + new IpfsBlockService{datastore}}; merkledag_service_ = std::make_shared(blockservice); data = GetParam(); EXPECT_OUTCOME_TRUE_1(this->saveToBlockService(data.nodes)); @@ -58,7 +59,7 @@ struct CommonFeaturesTest : public testing::TestWithParam { * @return operation result */ fc::outcome::result saveToBlockService( - const std::vector> &nodes) { + const std::vector> &nodes) { for (const auto &node : nodes) { EXPECT_OUTCOME_TRUE_1(merkledag_service_->addNode(node)); } @@ -70,7 +71,7 @@ struct CommonFeaturesTest : public testing::TestWithParam { * @param node - target for retrieving id * @return Base58-encoded CID v0 */ - static std::string cidToString(const std::shared_ptr &node) { + static std::string cidToString(const std::shared_ptr &node) { std::string value{}; auto result = ContentIdentifierCodec::toString(node->getCID()); if (!result.has_error()) { @@ -140,10 +141,10 @@ TEST_P(CommonFeaturesTest, CheckCidAlgorithmSuccess) { */ TEST_P(CommonFeaturesTest, LinkOperationsConsistency) { for (const auto &node : data.nodes) { - std::vector> links = node->getLinks(); + std::vector> links = node->getLinks(); std::string primary_cid = cidToString(node); for (const auto &link : links) { - LinkImpl link_impl{ + IPLDLinkImpl link_impl{ link.get().getCID(), link.get().getName(), link.get().getSize()}; EXPECT_OUTCOME_TRUE(received_link, node->getLink(link.get().getName())); std::ignore = received_link; @@ -204,9 +205,9 @@ TEST_P(CommonFeaturesTest, GraphSyncSelect) { const size_t nodes_count = data.nodes.front()->getLinks().size() + 1; EXPECT_OUTCOME_TRUE( root_cid, ContentIdentifierCodec::encode(data.nodes.front()->getCID())); - std::vector> selected_nodes; - std::function)> handler = - [&selected_nodes](std::shared_ptr node) -> bool { + std::vector> selected_nodes; + std::function)> handler = + [&selected_nodes](std::shared_ptr node) -> bool { selected_nodes.emplace_back(std::move(node)); return true; };