Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 39 additions & 8 deletions rsema1d/go/cmd/bench/bench_runner.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import (
"runtime"
"time"

rsema1d "github.com/celestiaorg/celestia-app/v8/pkg/rsema1d"
rsema1d "github.com/celestiaorg/celestia-app/v10/pkg/rsema1d"
)

func makeTestData(k, rowSize int) [][]byte {
Expand All @@ -20,26 +20,50 @@ func makeTestData(k, rowSize int) [][]byte {
return data
}

// makeEncodeRows returns the K+N row slice Coder.Encode expects: original
// data in rows[:K], allocated parity rows in rows[K:].
func makeEncodeRows(data [][]byte, k, n, rowSize int) [][]byte {
rows := make([][]byte, k+n)
copy(rows, data)
for i := k; i < k+n; i++ {
rows[i] = make([]byte, rowSize)
}
return rows
}

// zeroParity re-zeroes rows[K:], required by Coder.Encode before each call.
func zeroParity(rows [][]byte, k int) {
for _, row := range rows[k:] {
clear(row)
}
}

func benchmarkEncode(name string, k, n, rowSize int) {
workers := runtime.GOMAXPROCS(0)
data := makeTestData(k, rowSize)
config := &rsema1d.Config{
K: k,
N: n,
RowSize: rowSize,
WorkerCount: workers,
}
coder, err := rsema1d.NewCoder(config)
if err != nil {
panic(err)
}
rows := makeEncodeRows(data, k, n, rowSize)

// Warmup
for i := 0; i < 10; i++ {
rsema1d.Encode(data, config)
zeroParity(rows, k)
coder.Encode(rows)
}

// Measure
iterations := 100
start := time.Now()
for i := 0; i < iterations; i++ {
_, _, _, err := rsema1d.Encode(data, config)
zeroParity(rows, k)
_, err := coder.Encode(rows)
if err != nil {
panic(err)
}
Expand All @@ -59,11 +83,14 @@ func benchmarkProofGen(name string, k, n, rowSize int) {
config := &rsema1d.Config{
K: k,
N: n,
RowSize: rowSize,
WorkerCount: workers,
}

extData, _, _, err := rsema1d.Encode(data, config)
coder, err := rsema1d.NewCoder(config)
if err != nil {
panic(err)
}
extData, err := coder.Encode(makeEncodeRows(data, k, n, rowSize))
if err != nil {
panic(err)
}
Expand Down Expand Up @@ -94,14 +121,18 @@ func benchmarkVerification(name string, k, n, rowSize int) {
config := &rsema1d.Config{
K: k,
N: n,
RowSize: rowSize,
WorkerCount: workers,
}

extData, commitment, _, err := rsema1d.Encode(data, config)
coder, err := rsema1d.NewCoder(config)
if err != nil {
panic(err)
}
extData, err := coder.Encode(makeEncodeRows(data, k, n, rowSize))
if err != nil {
panic(err)
}
commitment := extData.Commitment()

proof, err := extData.GenerateStandaloneProof(0)
if err != nil {
Expand Down
115 changes: 46 additions & 69 deletions rsema1d/go/cmd/generate_fuzzy_vectors/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,14 @@ import (
"encoding/hex"
"encoding/json"
"fmt"
"math/bits"
"math/rand"
"os"
"time"

"github.com/celestiaorg/celestia-app/v8/pkg/rsema1d"
"github.com/celestiaorg/celestia-app/v8/pkg/rsema1d/field"
"github.com/celestiaorg/celestia-app/v10/pkg/rsema1d"
"github.com/celestiaorg/celestia-app/v10/pkg/rsema1d/field"
"github.com/celestiaorg/celestia-app/v10/pkg/rsema1d/merkle"
)

type TestVector struct {
Expand Down Expand Up @@ -95,64 +97,19 @@ func encodeProofNodes(proof [][]byte) []string {
return encoded
}

func isPowerOfTwo(n int) bool {
return n > 0 && (n&(n-1)) == 0
}

func nextPowerOfTwo(n int) int {
if n <= 1 {
return 1
}
p := 1
for p < n {
p <<= 1
}
return p
}

func randomN(rng *rand.Rand) int {
// Heavily sample small/medium values for fast default tests, but keep some large values.
p := rng.Float64()
switch {
case p < 0.70:
return rng.Intn(256) + 1
case p < 0.95:
return rng.Intn(2048-256) + 257
default:
return rng.Intn(32768-2048) + 2049
}
}

// randomValidKN returns k and n satisfying celestia-app config validation:
// k and k+n are powers of two, and k+n <= 65536.
func randomValidKN(rng *rand.Rand, maxTotalRows int) (int, int) {
if maxTotalRows < 2 {
maxTotalRows = 2
maxTotal := min(maxTotalRows, 65536)
if maxTotal < 2 {
maxTotal = 2
}
maxExp := bits.Len(uint(maxTotal)) - 1 // largest b with 2^b <= maxTotal

for {
n := randomN(rng)
np2 := nextPowerOfTwo(n)

// Constraints:
// 1) k + n <= 65536 (Go config validation)
// 2) k + next_pow_2(n) < 65536 (cross-validation target)
// 3) k + n <= maxTotalRows (runtime cap for default test speed)
kMaxByKN := 65536 - n
kMaxByNp2 := 65535 - np2
kMaxByTotal := maxTotalRows - n
kMax := min(kMaxByKN, min(kMaxByNp2, kMaxByTotal))
if kMax < 1 {
continue
}

k := rng.Intn(kMax) + 1

// Bias towards unequal pairs.
if k == n && rng.Float64() < 0.90 {
continue
}

return k, n
}
// k = 2^a, k+n = 2^b with a < b, so n = 2^b - 2^a > 0.
b := rng.Intn(maxExp) + 1
a := rng.Intn(b)
return 1 << a, (1 << b) - (1 << a)
}

func generateTestVector(name string, k, n, rowSize int, seed int64, proofIndices []int, inclusionIndices []int) TestVector {
Expand All @@ -168,18 +125,30 @@ func generateTestVector(name string, k, n, rowSize int, seed int64, proofIndices
config := &rsema1d.Config{
K: k,
N: n,
RowSize: rowSize,
WorkerCount: 1,
}

// Use only public API
extData, commitment, rlcOrig, err := rsema1d.Encode(originalData, config)
coder, err := rsema1d.NewCoder(config)
if err != nil {
panic(fmt.Sprintf("Failed to create coder: %v", err))
}
// Encode takes all K+N rows, with parity rows allocated and zeroed.
rows := make([][]byte, k+n)
copy(rows, originalData)
for i := k; i < k+n; i++ {
rows[i] = make([]byte, rowSize)
}
extData, err := coder.Encode(rows)
if err != nil {
panic(fmt.Sprintf("Failed to encode: %v", err))
}
commitment := extData.Commitment()
rlcOrig := extData.RLC()
rlcOrigHex := make([]string, len(rlcOrig))
for i, rlc := range rlcOrig {
b := field.ToBytes128(rlc)
for i, v := range rlcOrig {
var b [field.GF128Size]byte
field.EncodeGF128(b[:], v)
rlcOrigHex[i] = hex.EncodeToString(b[:])
}

Expand Down Expand Up @@ -219,20 +188,28 @@ func generateTestVector(name string, k, n, rowSize int, seed int64, proofIndices
})
}
}
// celestia-app dropped GenerateRowInclusionProof; rebuild it from a row
// proof plus the rlcOrig root (mirrors unexported computeRLCRoot).
var leaf [field.GF128Size]byte
rlcRoot := merkle.RootFromFunc(make([]byte, k*merkle.NodeSize), func(i int, _ []byte) []byte {
field.EncodeGF128(leaf[:], rlcOrig[i])
return leaf[:]
})

var inclusionTests []RowInclusionTest
for _, idx := range inclusionIndices {
inclusionProof, err := extData.GenerateRowInclusionProof(idx)
rowProof, err := extData.GenerateRowProof(idx)
if err != nil {
panic(fmt.Sprintf("Failed to generate inclusion proof for index %d: %v", idx, err))
}
inclusionTests = append(inclusionTests, RowInclusionTest{
Index: inclusionProof.Index,
IsOriginal: inclusionProof.Index < k,
RowHash: hashBytes(inclusionProof.Row),
RowProofHash: hashMerkleProof(inclusionProof.RowProof.RowProof),
RowData: hex.EncodeToString(inclusionProof.Row),
RowProof: encodeProofNodes(inclusionProof.RowProof.RowProof),
RLCRoot: hex.EncodeToString(inclusionProof.RLCRoot[:]),
Index: rowProof.Index,
IsOriginal: rowProof.Index < k,
RowHash: hashBytes(rowProof.Row),
RowProofHash: hashMerkleProof(rowProof.RowProof),
RowData: hex.EncodeToString(rowProof.Row),
RowProof: encodeProofNodes(rowProof.RowProof),
RLCRoot: hex.EncodeToString(rlcRoot[:]),
})
}

Expand Down
8 changes: 4 additions & 4 deletions rsema1d/go/go.mod
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
module github.com/celestiaorg/lumina/rsema1d/go

go 1.25.7
go 1.26.5

require github.com/celestiaorg/celestia-app/v8 v8.0.0-20260306035632-961e399b4dc4
require github.com/celestiaorg/celestia-app/v10 v10.0.0-20260827124947-2b18389af773

require (
github.com/celestiaorg/reedsolomon v1.12.6-0.20250824224240-8b66bda83fd0 // indirect
github.com/cometbft/cometbft/api v1.0.0 // indirect
github.com/klauspost/cpuid/v2 v2.3.0 // indirect
golang.org/x/sys v0.41.0 // indirect
github.com/klauspost/reedsolomon v1.14.2 // indirect
golang.org/x/sys v0.47.0 // indirect
)
30 changes: 12 additions & 18 deletions rsema1d/go/go.sum
Original file line number Diff line number Diff line change
@@ -1,28 +1,22 @@
github.com/celestiaorg/celestia-app/v8 v8.0.0-20260306035632-961e399b4dc4 h1:mvfToPQaISmSnHJ7+bkTpXbsnvp5pIn4JkA1IKFxzSI=
github.com/celestiaorg/celestia-app/v8 v8.0.0-20260306035632-961e399b4dc4/go.mod h1:xjVAOJOAFQIEGHh0ZR3pTnPc0eUACozBnftjLCtAOI0=
github.com/celestiaorg/reedsolomon v1.12.6-0.20250824224240-8b66bda83fd0 h1:nM6tHBLW1Uej4VKe6Zm3+kYnOrvcQ1Q4I1qFBIbC+EU=
github.com/celestiaorg/reedsolomon v1.12.6-0.20250824224240-8b66bda83fd0/go.mod h1:39naxIQYdj/oxjNZVvWvhRreh3uguAq4v6Ke7mI9cDI=
github.com/celestiaorg/celestia-app/v10 v10.0.0-20260827124947-2b18389af773 h1:MtD5oaGkh9gE5xDVl0QJt90LKKHF3GG2WGATyP1rTbE=
github.com/celestiaorg/celestia-app/v10 v10.0.0-20260827124947-2b18389af773/go.mod h1:/WokPBTX4iwQHD9eM4Vcfj/OtR+rsu2WChMju76Fv0I=
github.com/cometbft/cometbft v1.0.1 h1:JNVgbpL76sA4kXmBnyZ7iPjFAxi6HVp2l+rdT2RXVUs=
github.com/cometbft/cometbft v1.0.1/go.mod h1:r9fEwrbU6Oxs11I2bLsfAiG37OMn0Vip0w9arYU0Nw0=
github.com/cometbft/cometbft/api v1.0.0 h1:gGBwvsJi/gnHJEtwYfjPIGs2AKg/Vfa1ZuKCPD1/Ko4=
github.com/cometbft/cometbft/api v1.0.0/go.mod h1:EkQiqVSu/p2ebrZEnB2z6Re7r8XNe//M7ylR0qEwWm0=
github.com/cosmos/gogoproto v1.7.2 h1:5G25McIraOC0mRFv9TVO139Uh3OklV2hczr13KKVHCA=
github.com/cosmos/gogoproto v1.7.2/go.mod h1:8S7w53P1Y1cHwND64o0BnArT6RmdgIvsBuco6uTllsk=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
github.com/klauspost/reedsolomon v1.13.2 h1:9qtQy2tKEVpVB8Pfq87ZljHZb60/LbeTQ1OxV8EGzdE=
github.com/klauspost/reedsolomon v1.13.2/go.mod h1:ggJT9lc71Vu+cSOPBlxGvBN6TfAS77qB4fp8vJ05NSA=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k=
golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
github.com/klauspost/reedsolomon v1.14.2 h1:SafJYwpBBQBI6amHUygcjxZjXeN2HpiENHQDwuPWCCQ=
github.com/klauspost/reedsolomon v1.14.2/go.mod h1:yjqqjgMTQkBUHSG97/rm4zipffCNbCiZcB3kTqr++sQ=
github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE=
github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg=
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc=
google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
Loading