bug fixes

Author: cryptoquick

.cargo/config.toml

@@ -0,0 +1,15 @@
+# TEMPORARY — remove on or after 2026-07-18.
+#
+# `bitcoinpqc` 0.4.0 (published 2026-07-08) may not be on mirrored registries yet.
+# This patch satisfies `version = "0.4"` until mirrors sync (target lift: 2026-07-18).
+#
+# Lift checklist (on or after 2026-07-18):
+#   1. Delete this file.
+#   2. `cargo update -p bitcoinpqc`
+#   3. `cargo build --features pqc` (should resolve from crates.io with no patch).
+#
+# Environments that already see `bitcoinpqc` 0.4 on the default index may delete early.
+
+[patch.crates-io]
+bitcoinpqc = { git = "https://github.com/cryptoquick/libbitcoinpqc-bindings.git", rev = "7936b56" }
+# Monorepo local iteration (optional): `bitcoinpqc = { path = "../libbitcoinpqc-bindings" }`

.cargo/config.toml.example

@@ -7,4 +7,23 @@
 #   cp .cargo/config.toml.example .cargo/config.toml
 
 [patch."https://github.com/SurmountSystems/bao-tree.git"]
-bao-tree = { path = "../bao-tree" }
+bao-tree = { path = "../bao-tree" }
+
+# `bitcoinpqc` 0.4 mirror lag: the repo ships `.cargo/config.toml` with a temporary
+# `[patch.crates-io]` until 2026-07-18. Delete that file once your index has 0.4.
+
+# Optional: wasm32 tooling notes
+#
+# CI / `just lint-wasm`: `--no-default-features` (no pqc) — green without a C wasm toolchain.
+#
+# wasm32 + `pqc`: bitcoinpqc's clang shim lives at
+# `../libbitcoinpqc-bindings/wasm/clang-wasm32.sh`. Do **not** set global
+# `CC_wasm32-unknown-unknown` to that shim for a full Carbonado build: `zstd-sys`
+# then tries to assemble host amd64 `.S` files with the wasm clang and fails.
+# Isolate CC for bitcoinpqc only (or wait for dep/build fixes) before enabling
+# wasm+pqc in CI.
+#
+# [target.wasm32-unknown-unknown]
+# rustflags = [
+#   "--cfg", "getrandom_backend=\"wasm_js\"",
+# ]

.github/workflows/rust.yaml

@@ -77,8 +77,18 @@ jobs:
 
       - uses: Swatinem/rust-cache@v2
 
-      - name: Test (native)
-        run: just test
+      - name: Test (default features, includes parallel)
+        run: cargo test
+        env:
+          RUST_BACKTRACE: 1
+
+      - name: Test (serial FEC path without parallel)
+        run: cargo test --no-default-features --features "pqc,ots,cli" --test serial_fec_path
+        env:
+          RUST_BACKTRACE: 1
+
+      - name: Test (all features)
+        run: cargo test --all-features
         env:
           RUST_BACKTRACE: 1
 

AGENTS.md

@@ -6,17 +6,25 @@ All notable changes to the Carbonado crate and `carbonado` CLI are documented he
 
 ### Added
 
+- **`bitcoinpqc` 0.4:** migrate from crates.io `libbitcoinpqc` 0.1 to `bitcoinpqc` 0.4 (WASM-capable bindings). SLH-DSA parameter set is now **SHA2-128s** (`SLH_DSA_SHA2_128S`); SHAKE-128s sidecars from dev builds are incompatible — re-sign if needed. Temporary `.cargo/config.toml` `[patch.crates-io]` — delete on or after **2026-07-18** once registry mirrors sync.
+- **Phase 3 CPU parallelism (`parallel` feature, default on):** scoped fork-join RS parity generation in `FecInboardEncoder::take_stripe` (`src/stream/parallel.rs`); `ParallelConfig::max_threads` caps per-wave workers; output bit-identical to serial `rs.encode`. WASM compiles with `parallel` but stays serial at runtime. Disable with `--no-default-features` for serial-only builds. Tests: `tests/parallel_determinism.rs` (default), `tests/serial_fec_path.rs` (no-`parallel` CI gate); bench: `benches/parallel_bench.rs`. Documented in `doc/STREAMING_PARALLELISM.md` § Phase 3.
+- **Phase 2 async I/O adapter (optional `async` feature):** `stream_decode_async` (`AsyncRead` → `AsyncWrite` inboard decode), `stream::io` pipeline traits (`PipelineSource`/`PipelineSink`, `AsyncPipelineSource`/`AsyncPipelineSink`), `BoundedCopyTruncation` for sync-aligned truncation errors. Disk spool bridge preserves MAC-before-decrypt; `async-tokio` offloads sync pipeline via `spawn_blocking`. WASM returns `NotImplemented`. Tests: `tests/streaming_async.rs` (`cargo test --features async`).
 - **Adamantine 1.0 directory envelope:** magic `ADAMANTINE10\n` (version in magic); 19-byte header with u8 flags (`REQUIRE_OTS` only). New module `adamantine_payload` (rkyv + centralized Bao bundle). Dev `ADAMANTINE2\n` rejected.
-- **Heterogeneous segment formats:** `directory/format_policy` with `infer` heuristic (compressible → c6/c7, incompressible → c4/c5); `SegmentFormatPolicy` Auto/Force*; `FilepackEntry.segment_format`; `SegmentRef.bao_outboard_offset/len`.
+- **Heterogeneous FEC segment formats:** `directory/format_policy` with `infer` heuristic (compressible → c14/c15, incompressible → c12/c13); `SegmentFormatPolicy` Auto/Force*; `FilepackEntry.segment_format`; `SegmentRef` verification + `fec_parity` bundle indices.
 - **Catalog OTS trailer:** optional `[COTS][u32 len][proof]` after inboard catalog bytes (stable Bao root).
 - **Filepack CBOR interop:** `FilepackManifest::from_filepack_cbor`, `from_packed`, `to_filepack_cbor`; `FilepackSegmentMap`; DoS limits on CBOR flatten. Error: `InvalidFilepackCbor`.
 - **New errors:** `InvalidAdamantineFlags`, `SegmentFormatMismatch`.
 
 ### Changed
 
+- **M1 pipeline memory (hard break):** non-FEC verification decode (c6) uses `SeekWriteAt` over the post-preprocess spool (O(chunk) RAM; no full logical `Vec`). FEC verification uses `FecInboardWriteAt::finish_into` (stream logical bytes without a second full logical buffer; shard buffers remain O(FEC body) under segment-wide RS geometry). See `doc/STREAMING_PARALLELISM.md`.
+- **M2 outboard verify memory:** `stream_verification_outboard_verify` uses `PostOrderOutboard` + `ReadAt` (on-demand hash pairs) instead of copying the full sidecar into `PostOrderMemOutboard`. Streaming outboard decode keeps the sidecar on a disk spool.
+- **S5 scrub verify oracle:** `scrub` pre-check uses `verify_inboard_keyed` (`DiscardWriteAt` sink) instead of buffer `verification()` full-body staging; `scrub_outboard` pre-check uses `stream_verification_outboard_verify` with `io::sink()`. Memory tiers in `doc/STREAMING_PARALLELISM.md`.
+- **S4 streaming inboard decode:** `stream_decode` / `stream_decode_buffer` stream keyed Bao verify into `WriteAt` sinks without staging the encoded body; `decode_stream` and `file::decode` share the same Bao/FEC path. Memory tiers documented in `doc/STREAMING_PARALLELISM.md`.
 - **Directory archive layout (clean break):** inboard catalog c14/c15 only; bare segment mains; no directory `.out`/`.par`/`.ots` sidecars; Bao outboard centralized in Adam payload. Removed `DirectoryLayout`, homogeneous segment format, directory `--inboard`/`--outboard`/`--format` CLI flags.
 - **CLI directory output:** defaults to `{input}-archive/`; never `.`.
-- **Directory format generalization superseded:** catalog locked to c14/c15; segments heterogeneous c4–c7.
+- **Directory segment FEC (clean break):** catalog locked to c14/c15; segments heterogeneous c12–c15 (Verification+FEC); legacy c4–c7 rejected. Centralized bundle holds verification outboard + FEC parity per segment.
+- **Keyed verification KDF domain:** `blake3::derive_key("carbonado-v2/verification", &[format])` replaces `"carbonado-v2/bao"` (breaks keyed roots vs pre-2.1.0). Public API: `crypto::carbonado_verification_key`.
 - **Directory manifest API rename (Phase 1):** `PackIndex` → `FilepackManifest`, `PackEntry` → `FilepackEntry`, `PackSegmentRef` → `SegmentRef`, module `pack_index` → `filepack_manifest`. rkyv wire layout unchanged; on-disk archives remain compatible.
 - **Error variant rename (breaking):** `InvalidPackIndex` → `InvalidFilepackManifest`. No enum alias is provided.
 - **Narrowed error taxonomy:** OTS proof size failures → `InvalidOtsProof`; Adamantine oversized `payload_len` → `InvalidAdamantinePayloadTooLarge`; directory decode integrity failures → `SegmentMainLenMismatch`, `ContentBlake3Mismatch`, `OutputPathEscape`, `OtsFeatureRequired`, `OtsProofRequired`.

CHANGELOG.md

@@ -3,7 +3,7 @@ name = "carbonado"
 edition = "2021"
 version = "2.0.0"
 license = "MIT"
-description = "An apocalypse-resistant data storage format for the truly paranoid. Fully symmetric AES-256-CTR + HMAC-SHA512 (EtM) container. Callers supply high-entropy master keys; passphrase KDF (e.g. Argon2id) is caller responsibility. SLH-DSA signatures are provided only as sidecars (via libbitcoinpqc). Clean cryptographic break from ECIES — v1 encrypted archives require external migration."
+description = "An apocalypse-resistant data storage format for the truly paranoid. Fully symmetric AES-256-CTR + HMAC-SHA512 (EtM) container. Callers supply high-entropy master keys; passphrase KDF (e.g. Argon2id) is caller responsibility. SLH-DSA signatures are provided only as sidecars (via `bitcoinpqc`). Clean cryptographic break from ECIES — v1 encrypted archives require external migration."
 documentation = "https://docs.rs/carbonado"
 homepage = "https://github.com/bitmask-stack/carbonado"
 repository = "https://github.com/bitmask-stack/carbonado.git"
@@ -32,6 +32,8 @@ bao = "0.13"
 # We set default-features = false because the fork's defaults pull in tokio/fs/etc
 # which are not needed (we only use sync keyed) and break some cross/wasm targets.
 bao_tree = { package = "bao-tree", git = "https://github.com/SurmountSystems/bao-tree.git", branch = "76-keyed-bao", default-features = false, features = ["validate"] }
+futures-lite = { version = "2", optional = true, default-features = false, features = ["std"] }
+tokio = { version = "1", features = ["rt"], optional = true }
 bitmask-enum = "2.1.0"
 log = "0.4.19"
 serde = { version = "1.0.164", features = ["derive"] }
@@ -74,34 +76,52 @@ ctr = "0.9"
 hmac = "0.12"
 sha2 = "0.10"
 getrandom = { version = "0.2", features = ["js"] }
-libbitcoinpqc = { version = "0.1", optional = true }
+# bitcoinpqc 0.4 (WASM-capable build.rs); SLH-DSA-SHA2-128s sidecars.
+# Local mirror lag: `.cargo/config.toml` patches 0.4 until 2026-07-18 (see lift checklist there).
+bitcoinpqc = { version = "0.4", optional = true }
 
 # Note: ecies + libsecp256k1-core + nostr + secp256k1 removed (clean break from old ECIES design and Nostr bech32/npub helpers).
 # Future quantum-resistant key formats (qpub etc.) will be handled separately.
 
 [features]
-default = ["pqc", "ots", "cli"]
-pqc = ["dep:libbitcoinpqc"]
+default = ["pqc", "ots", "cli", "parallel"]
+pqc = ["dep:bitcoinpqc"]
 # OpenTimestamps stub stamping (offline/testable; no network calendar in default build).
 ots = []
 # `carbonado` CLI binary (encode/decode/key). Disable for library-only dependency trees.
 cli = ["dep:clap", "dep:bip39", "dep:directories"]
 # `gen-carbonado-man` — regenerate roff pages under doc/man/ (`just gen-man`).
 man-gen = ["cli", "dep:clap_mangen"]
+# Async I/O adapter layer (Phase 2): `stream_decode_async` and pipeline I/O traits.
+# `bao_tree/tokio_fsm` is reserved for future async Bao FSM paths — no crate code references it yet.
+async = ["dep:futures-lite", "bao_tree/tokio_fsm"]
+# Optional Tokio: `stream_decode_async` offloads sync pipeline to `spawn_blocking` (not required for trait defs).
+async-tokio = ["async", "dep:tokio"]
+# CPU parallelism (Phase 3): fork-join RS parity encode via `std::thread::scope` (std only; on by default).
+# Disable with `--no-default-features` for library-only trees that want serial `rs.encode` only.
+parallel = []
 
 [dev-dependencies]
 anyhow = "1.0.71"
+sha2 = "0.10"
+serde_json = "1"
 pretty_env_logger = "0.5.0"
 rand = "0.8.5"
 wasm-bindgen-test = "0.3.37"
 criterion = { version = "0.5", features = ["html_reports"] }
 proptest = "1"
 tiny_http = "0.12"
+tokio = { version = "1", features = ["rt", "macros"] }
 
 [[bench]]
 name = "crypto_bench"
 harness = false
 
+[[bench]]
+name = "parallel_bench"
+harness = false
+required-features = ["parallel"]
+
 # Primary CLI: `cargo install carbonado` or `cargo install --path . --bin carbonado`
 [[bin]]
 name = "carbonado"