Add 'ipld-resolver/' from commit '4888d016b639ad6519c17cd3d8ff3ea8cda94122'

git-subtree-dir: ipld-resolver
git-subtree-mainline: e1c5fa2
git-subtree-split: 4888d01

Author: adlrocha

ipld-resolver/.github/workflows/ci.yaml

@@ -0,0 +1,90 @@
+name: CI
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - '**'
+
+jobs:
+  # Check code formatting; anything that doesn't require compilation.
+  pre-compile-checks:
+    name: Pre-compile checks
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the project
+        uses: actions/checkout@v3
+      - name: Install Rust
+        uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: nightly
+          components: rustfmt
+      - name: Check code formatting
+        run: make check-fmt
+      - name: Check license headers
+        run: make license
+      # - name: Check diagrams
+      #   run: make check-diagrams
+
+  # Test matrix, running tasks from the Makefile.
+  tests:
+    needs: [pre-compile-checks]
+    name: ${{ matrix.make.name }} (${{ matrix.os }}, ${{ matrix.rust }})
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+        rust: [nightly]
+        make:
+          - name: Lint
+            task: lint
+          - name: Test
+            task: test
+        exclude:
+          - rust: stable
+            make:
+              name: Lint
+
+    env:
+      RUST_BACKTRACE: full
+      RUSTFLAGS: -Dwarnings
+      CARGO_INCREMENTAL: '0'
+      SCCACHE_CACHE_SIZE: 10G
+      CC: "sccache clang"
+      CXX: "sccache clang++"
+
+    steps:
+      - name: Check out the project
+        uses: actions/checkout@v3
+
+      - name: Install Rust
+        uses: dtolnay/rust-toolchain@master
+        with:
+          targets: wasm32-unknown-unknown
+          toolchain: ${{ matrix.rust }}
+          components: rustfmt,clippy
+
+      # Protobuf compiler required by libp2p-core
+      - name: Install Protoc
+        uses: arduino/setup-protoc@v1
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Setup sccache
+        uses: hanabi1224/[email protected] # https://github.com/hanabi1224/sccache-action used by Forest.
+        timeout-minutes: 5
+        continue-on-error: true
+        with:
+          release-name: v0.3.1
+          # Caching everything separately, in case they don't ask for the same things to be compiled.
+          cache-key: ${{ matrix.make.name }}-${{ matrix.os }}-${{matrix.rust}}-${{ hashFiles('Cargo.lock', 'rust-toolchain', 'rust-toolchain.toml') }}
+          # Not sure why we should ever update a cache that has the hash of the lock file in it.
+          # In Forest it only contains the rust-toolchain, so it makes sense to update because dependencies could have changed.
+          cache-update: false
+
+      - name: ${{ matrix.make.name }}
+        run: make ${{ matrix.make.task }}

ipld-resolver/.gitignore

@@ -0,0 +1,5 @@
+.idea/
+*.iml
+/target
+docs/diagrams/plantuml.jar
+Cargo.lock

ipld-resolver/Cargo.toml

@@ -0,0 +1,67 @@
+[package]
+name = "ipc_ipld_resolver"
+version = "0.1.0"
+description = "P2P library to resolve IPLD content across IPC subnets."
+authors = ["Protocol Labs"]
+edition = "2021"
+license-file = "LICENSE"
+
+[dependencies]
+anyhow = "1.0"
+base64 = "0.21.0"
+blake2b_simd = "1.0"
+bloom = "0.3"
+gcra = "0.4"
+lazy_static = "1.4"
+libipld = { version = "0.14", default-features = false, features = ["dag-cbor"] }
+libp2p = { version = "0.50", default-features = false, features = [
+  "gossipsub",
+  "kad",
+  "identify",
+  "ping",
+  "noise",
+  "yamux",
+  "tcp",
+  "dns",
+  "mplex",
+  "request-response",
+  "metrics",
+  "tokio",
+  "macros",
+  "serde",
+  "secp256k1",
+  "plaintext",
+] }
+libp2p-bitswap = "0.25.1"
+log = "0.4"
+prometheus = "0.13"
+quickcheck = { version = "1", optional = true }
+rand = "0.8"
+serde = { version = "1.0", features = ["derive"] }
+serde_json = { version = "1.0.91", features = ["raw_value"] }
+thiserror = "1.0.38"
+tokio = { version = "1.16", features = ["full"] }
+
+fvm_ipld_encoding = "0.3"
+fvm_shared = { version = "~3.2", default-features = false, features = ["crypto"], optional = true }
+fvm_ipld_blockstore = { version = "0.1", optional = true }
+
+# Using the IPC SDK without the `fil-actor` feature so as not to depend on the actor `Runtime`.
+# Using the `main` branch instead of the highest available tag `v0.3.0` because the latter doesn't have a feature flag for the `Runtime`.
+ipc-sdk = { git = "https://github.com/consensus-shipyard/ipc.git", default-features = false, branch = "dev" }
+
+[dev-dependencies]
+quickcheck_macros = "1"
+env_logger = "0.10"
+fvm_ipld_hamt = "0.6"
+
+ipc_ipld_resolver = { path = ".", features = ["arb"] }
+
+[features]
+default = ["arb", "missing_blocks"]
+arb = ["quickcheck", "fvm_shared/arb"]
+missing_blocks = ["fvm_ipld_blockstore"]
+
+[patch.crates-io]
+# Use stable-only features.
+gcra = { git = "https://github.com/consensus-shipyard/gcra-rs.git", branch = "main" }

ipld-resolver/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 ConsensusLab
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

ipld-resolver/Makefile

@@ -0,0 +1,35 @@
+.PHONY: all build test lint license check-fmt check-clippy diagrams
+
+all: test build
+
+build:
+	cargo build -Z unstable-options --release
+
+test:
+	cargo test --release --workspace
+
+clean:
+	cargo clean
+
+lint: \
+	license \
+	check-fmt \
+	check-clippy
+
+license:
+	./scripts/add_license.sh
+
+check-fmt:
+	cargo fmt --all --check
+
+check-clippy:
+	cargo clippy --all --tests -- -D clippy::all
+
+diagrams:
+	$(MAKE) -C docs/diagrams
+
+check-diagrams: diagrams
+	if git diff --name-only docs/diagrams | grep .png; then \
+		echo "There are uncommitted changes to the diagrams"; \
+		exit 1; \
+	fi

ipld-resolver/README.md

@@ -0,0 +1,60 @@
+# IPLD Resolver
+
+The IPLD Resolver is a Peer-to-Peer library which can be used to resolve arbitrary CIDs from subnets in InterPlanetary Consensus.
+
+See the [docs](./docs/) for a conceptual overview.
+
+## Usage
+
+Please have a look at the [smoke test](./tests/smoke.rs) for an example of using the library.
+
+The following snippet demonstrates how one would create a resolver instance and use it:
+
+```rust
+async fn main() {
+  let config = Config {
+      connection: ConnectionConfig {
+          listen_addr: "/ip4/127.0.0.1/tcp/0".parse().unwrap(),
+          expected_peer_count: 1000,
+          max_incoming: 25,
+          max_peers_per_query: 10,
+          event_buffer_capacity: 100,
+      },
+      network: NetworkConfig {
+          local_key: Keypair::generate_secp256k1(),
+          network_name: "example".to_owned(),
+      },
+      discovery: DiscoveryConfig {
+          static_addresses: vec!["/ip4/95.217.194.97/tcp/8008/p2p/12D3KooWC1EaEEpghwnPdd89LaPTKEweD1PRLz4aRBkJEA9UiUuS".parse().unwrap()]
+          target_connections: 50,
+          enable_kademlia: true,
+      },
+      membership: MembershipConfig {
+          static_subnets: vec![],
+          max_subnets: 10,
+          publish_interval: Duration::from_secs(300),
+          min_time_between_publish: Duration::from_secs(5),
+          max_provider_age: Duration::from_secs(60),
+      },
+  };
+
+  let store = todo!("implement BitswapStore and a Blockstore");
+
+  let service = Service::new(config, store.clone());
+  let client = service.client();
+
+  tokio::task::spawn(async move { service.run().await });
+
+  let cid: Cid = todo!("the CID we want to resolve");
+  let subnet_id: SubnetID = todo!("the SubnetID from where the CID can be resolved");
+
+  match client.resolve(cid, subnet_id).await.unwrap() {
+    Ok(()) => {
+      let _content: MyContent = store.get(cid).unwrap();
+    }
+    Err(e) => {
+      println!("{cid} could not be resolved from {subnet_id}: {e}")
+    }
+  }
+}
+```

ipld-resolver/docs/README.md

@@ -0,0 +1,82 @@
+# IPLD Resolver
+
+The IPLD Resolver is a library that [IPC Agents](https://github.com/consensus-shipyard/ipc/) can use to exchange data between subnets in IPLD format.
+
+## Checkpointing
+
+The most typical use case would be the propagation of checkpoints from child subnets to the parent subnet.
+
+### Checkpoint Schema
+
+One possible conceptual model of checkpointing is depicted by the following Entity Relationship diagram:
+
+![Checkpoint Schema](diagrams/checkpoint_schema.png)
+
+It shows that the Subnet Actor in the parent subnet governs the power of validators in the child subnet by proposing _Configurations_, which the child subnet is free to adopt in its _Epochs_ when the time is right, communicating back the next adopted config via _Checkpoints_.
+
+At the end of an epoch, the validators in the child subnet produce a checkpoint over some contents, notably the cross-messages they want to propagate towards the parent subnet. Through the cross-messages, the checkpoint indirectly points to individual messages that users or actors wanted to send.
+
+Once enough signatures are collected to form a Quorum Certificate over the checkpoint (the specific rules are in the jurisdiction of the Subnet Actor), the checkpoint is submitted to the parent ledger.
+
+However, the submitted checkpoint does not contain the raw messages, only the meta-data. The content needs to be resolved using the IPC Resolver, as indicated by the dotted line.
+
+### Checkpoint Submission and Resolution
+
+The following sequence diagram shows one possible way how checkpoints can be submitted from the child to the parent subnet.
+
+It depicts two validators: one only participating on the parent subnet, and the other on the child subnet; the latter has to also run at least a full node on the parent subnet. Both validators run one IPC Agent each.
+
+The diagram shows that at the end of the epoch the child subnet validators produce a Quorum Certificate over the checkpoint, which some of their agents submit to the parent subnet.
+
+After that, the parent subnet nodes reach out to their associated IPC Agent to resolve the messages referenced by the checkpoint, which the Agent does by communicating with some of its child-subnet peers.
+
+![Checkpoint Submission](diagrams/checkpoint_submission.png)
+
+This is just a high level view of what happens during message resolution. In the next section we will delve deeper into the internals of the IPLD Resolver.
+
+
+## IPLD Resolver Sub-components
+
+The IPLD Resolver uses libp2p to form a Peer-to-Peer network, using the following protocols:
+* [Ping](https://github.com/libp2p/rust-libp2p/tree/v0.50.1/protocols/ping)
+* [Identify](https://github.com/libp2p/rust-libp2p/tree/v0.50.1/protocols/ping) is used to learn the listening address of the remote peers
+* [Kademlia](https://github.com/libp2p/rust-libp2p/tree/v0.50.1/protocols/kad) is used for peer discovery
+* [Gossipsub](https://github.com/libp2p/rust-libp2p/tree/v0.50.1/protocols/gossipsub) is used to announce information about subnets the peers provide data for
+* [Bitswap](https://github.com/ipfs-rust/libp2p-bitswap) is used to resolve CIDs to content
+
+See the libp2p [specs](https://github.com/libp2p/specs) and [docs](https://docs.libp2p.io/concepts/fundamentals/protocols/) for details on each protocol, and look [here](https://docs.ipfs.tech/concepts/bitswap/) for Bitswap.
+
+The Resolver is completely agnostic over what content it can resolve, as long as it's based on CIDs; it's not aware of the checkpointing use case above.
+
+The interface with the host system is through a host-provided implementation of the [BitswapStore](https://github.com/ipfs-rust/libp2p-bitswap/blob/7dd9cececda3e4a8f6e14c200a4b457159d8db33/src/behaviour.rs#L55) which the library uses to retrieve and store content. Implementors can make use of the [missing_blocks](../src/missing_blocks.rs) helper method which recursively collects all CIDs from an IPLD `Blockstore`, starting from the root CID we are looking for.
+
+Internally the protocols are wrapped into behaviours that interpret their events and manage their associated state:
+* `Discovery` wraps `Kademlia`
+* `Membership` wraps `Gossipsub`
+* `Content` wraps `Bitswap`
+
+The following diagram shows a typical sequence of events within the IPLD Resolver. For brevity, only one peer is shown in detail; it's counterpart is represented as a single boundary.
+
+![IPLD Resolver](diagrams/ipld_resolver.png)
+
+# Diagram Automation
+
+The diagrams in this directory can be rendered with `make diagrams`.
+
+Adding the following script to `.git/hooks/pre-commit` automatically renders and checks in the images when we commit changes to the them. CI should also check that there are no uncommitted changes.
+
+```bash
+#!/usr/bin/env bash
+
+# If any command fails, exit immediately with that command's exit status
+set -eo pipefail
+
+# Redirect output to stderr.
+exec 1>&2
+
+if git diff --cached --name-only  --diff-filter=d | grep .puml
+then
+  make diagrams
+  git add docs/diagrams/*.png
+fi
+```

ipld-resolver/docs/diagrams/Makefile

@@ -0,0 +1,16 @@
+PUMLS = $(shell find . -type f -name "*.puml")
+PNGS  = $(PUMLS:.puml=.png)
+PUML_VER=1.2023.2
+
+.PHONY: all
+all: diagrams
+
+.PHONY: diagrams
+diagrams: $(PNGS)
+
+plantuml.jar:
+	wget -O $@ https://github.com/plantuml/plantuml/releases/download/v$(PUML_VER)/plantuml-$(PUML_VER).jar --no-check-certificate --quiet
+
+%.png: plantuml.jar %.puml
+	@# Using pipelining to preserve file names.
+	cat $*.puml | java -jar plantuml.jar -pipe > $*.png