Skip to content

Commit cf35bb4

Browse files
dignifiedquiredignifiedquire
authored
Merge pull request #959 from filecoin-project/simpler-windows
refactor: unify window size handling
2 parents 4084eed + 546b7a9 commit cf35bb4

File tree

11 files changed

+32
-81
lines changed

11 files changed

+32
-81
lines changed

‎.circleci/config.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -181,7 +181,7 @@ jobs:
181181
- run:
182182
name: Ensure existence of Groth parameters and keys on remote host
183183
command: |
184-
./fil-proofs-tooling/scripts/paramcache-remote.sh "${CIRCLE_BRANCH}" "${BENCHMARK_SERVER_SSH_USERNAME}@${BENCHMARK_SERVER_IP_ADDR}" "-z=$((1024*1024*1024)):$(((128*1024*1024)/32))"
184+
./fil-proofs-tooling/scripts/paramcache-remote.sh "${CIRCLE_BRANCH}" "${BENCHMARK_SERVER_SSH_USERNAME}@${BENCHMARK_SERVER_IP_ADDR}" "-z=$((1024*1024*1024))"
185185
no_output_timeout: 60m
186186
- run:
187187
name: Run hash-constraints benchmarks on remote host

‎fil-proofs-tooling/src/bin/benchy/election_post.rs

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -52,12 +52,8 @@ impl Report {
5252
}
5353
}
5454

55-
pub fn run(sector_size: usize, window_size_nodes: usize) -> anyhow::Result<()> {
56-
info!(
57-
"Benchy Election PoSt: sector-size={} - window_size={}",
58-
sector_size,
59-
window_size_nodes * 32
60-
);
55+
pub fn run(sector_size: usize) -> anyhow::Result<()> {
56+
info!("Benchy Election PoSt: sector-size={}", sector_size,);
6157

6258
let sector_size_unpadded_bytes_ammount =
6359
UnpaddedBytesAmount::from(PaddedBytesAmount(sector_size as u64));
@@ -102,7 +98,6 @@ pub fn run(sector_size: usize, window_size_nodes: usize) -> anyhow::Result<()> {
10298
let porep_config = PoRepConfig {
10399
sector_size: SectorSize(sector_size as u64),
104100
partitions: N_PARTITIONS,
105-
window_size_nodes,
106101
};
107102
let cache_dir = tempfile::tempdir().unwrap();
108103
let sector_id = SectorId::from(SECTOR_ID);
@@ -147,7 +142,6 @@ pub fn run(sector_size: usize, window_size_nodes: usize) -> anyhow::Result<()> {
147142
// Measure PoSt generation and verification.
148143
let post_config = PoStConfig {
149144
sector_size: SectorSize(sector_size as u64),
150-
window_size_nodes,
151145
};
152146

153147
let challenge_count = 1u64;

‎fil-proofs-tooling/src/bin/benchy/main.rs

Lines changed: 1 addition & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -112,13 +112,6 @@ fn main() {
112112
.required(true)
113113
.help("The data size in KiB")
114114
.takes_value(true),
115-
)
116-
.arg(
117-
Arg::with_name("window-size")
118-
.long("window-size")
119-
.help("The window size in bytes")
120-
.default_value("4096")
121-
.takes_value(true),
122115
);
123116

124117
let hash_cmd = SubCommand::with_name("hash-constraints")
@@ -164,10 +157,7 @@ fn main() {
164157
let sector_size_kibs = value_t!(m, "size", usize)
165158
.expect("could not convert `size` CLI argument to `usize`");
166159
let sector_size = sector_size_kibs * 1024;
167-
let window_size_bytes = value_t!(m, "window-size", usize)
168-
.expect("could not convert `window-size` CLI argument to `usize`");
169-
let window_size_nodes = window_size_bytes / 32;
170-
election_post::run(sector_size, window_size_nodes).expect("election-post failed");
160+
election_post::run(sector_size).expect("election-post failed");
171161
}
172162
("hash-constraints", Some(_m)) => {
173163
hash_fns::run().expect("hash-constraints failed");

‎filecoin-proofs/src/api/mod.rs

Lines changed: 1 addition & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,6 @@ pub fn get_unsealed_range<T: Into<PathBuf> + AsRef<Path>>(
7373
let pp = public_params(
7474
PaddedBytesAmount::from(porep_config),
7575
usize::from(PoRepProofPartitions::from(porep_config)),
76-
porep_config.window_size_nodes,
7776
);
7877

7978
let offset_padded: PaddedBytesAmount = UnpaddedBytesAmount::from(offset).into();
@@ -289,9 +288,7 @@ mod tests {
289288
use storage_proofs::fr32::bytes_into_fr;
290289
use tempfile::NamedTempFile;
291290

292-
use crate::constants::{
293-
SECTOR_SIZE_ONE_KIB, SINGLE_PARTITION_PROOF_LEN, WINDOW_SIZE_NODES_ONE_KIB,
294-
};
291+
use crate::constants::{SECTOR_SIZE_ONE_KIB, SINGLE_PARTITION_PROOF_LEN};
295292
use crate::types::{PoStConfig, SectorSize};
296293

297294
static INIT_LOGGER: Once = Once::new();
@@ -316,7 +313,6 @@ mod tests {
316313
PoRepConfig {
317314
sector_size: SectorSize(SECTOR_SIZE_ONE_KIB),
318315
partitions: PoRepProofPartitions(2),
319-
window_size_nodes: WINDOW_SIZE_NODES_ONE_KIB,
320316
},
321317
not_convertible_to_fr_bytes,
322318
convertible_to_fr_bytes,
@@ -345,7 +341,6 @@ mod tests {
345341
PoRepConfig {
346342
sector_size: SectorSize(SECTOR_SIZE_ONE_KIB),
347343
partitions: PoRepProofPartitions(2),
348-
window_size_nodes: WINDOW_SIZE_NODES_ONE_KIB,
349344
},
350345
convertible_to_fr_bytes,
351346
not_convertible_to_fr_bytes,
@@ -393,7 +388,6 @@ mod tests {
393388
let result = verify_post(
394389
PoStConfig {
395390
sector_size: SectorSize(SECTOR_SIZE_ONE_KIB),
396-
window_size_nodes: WINDOW_SIZE_NODES_ONE_KIB,
397391
},
398392
&[0; 32],
399393
1,
@@ -424,7 +418,6 @@ mod tests {
424418
let rng = &mut XorShiftRng::from_seed(crate::TEST_SEED);
425419

426420
let sector_size = SECTOR_SIZE_ONE_KIB;
427-
let window_size_nodes = WINDOW_SIZE_NODES_ONE_KIB;
428421

429422
let number_of_bytes_in_piece =
430423
UnpaddedBytesAmount::from(PaddedBytesAmount(sector_size.clone()));
@@ -456,7 +449,6 @@ mod tests {
456449
let config = PoRepConfig {
457450
sector_size: SectorSize(sector_size.clone()),
458451
partitions: PoRepProofPartitions(2),
459-
window_size_nodes,
460452
};
461453

462454
let cache_dir = tempfile::tempdir().unwrap();

‎filecoin-proofs/src/api/seal.rs

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,6 @@ pub fn seal_pre_commit<R: AsRef<Path>, T: AsRef<Path>, S: AsRef<Path>>(
7777
vanilla_params: setup_params(
7878
PaddedBytesAmount::from(porep_config),
7979
usize::from(PoRepProofPartitions::from(porep_config)),
80-
porep_config.window_size_nodes,
8180
),
8281
partitions: Some(usize::from(PoRepProofPartitions::from(porep_config))),
8382
};
@@ -220,7 +219,6 @@ pub fn seal_commit<T: AsRef<Path>>(
220219
vanilla_params: setup_params(
221220
PaddedBytesAmount::from(porep_config),
222221
usize::from(PoRepProofPartitions::from(porep_config)),
223-
porep_config.window_size_nodes,
224222
),
225223
partitions: Some(usize::from(PoRepProofPartitions::from(porep_config))),
226224
};
@@ -292,7 +290,6 @@ pub fn verify_seal(
292290
vanilla_params: setup_params(
293291
PaddedBytesAmount::from(porep_config),
294292
usize::from(PoRepProofPartitions::from(porep_config)),
295-
porep_config.window_size_nodes,
296293
),
297294
partitions: Some(usize::from(PoRepProofPartitions::from(porep_config))),
298295
};

‎filecoin-proofs/src/bin/paramcache.rs

Lines changed: 8 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,11 @@ use storage_proofs::stacked::StackedDrg;
1818

1919
const POREP_PROOF_PARTITION_CHOICES: [PoRepProofPartitions; 1] = [PoRepProofPartitions(2)];
2020

21-
const PUBLISHED_SECTOR_SIZES: [(u64, usize); 4] = [
22-
(SECTOR_SIZE_ONE_KIB, WINDOW_SIZE_NODES_ONE_KIB),
23-
(SECTOR_SIZE_16_MIB, WINDOW_SIZE_NODES_16_MIB),
24-
(SECTOR_SIZE_256_MIB, WINDOW_SIZE_NODES_256_MIB),
25-
(SECTOR_SIZE_1_GIB, WINDOW_SIZE_NODES_1_GIB),
21+
const PUBLISHED_SECTOR_SIZES: [u64; 4] = [
22+
SECTOR_SIZE_ONE_KIB,
23+
SECTOR_SIZE_16_MIB,
24+
SECTOR_SIZE_256_MIB,
25+
SECTOR_SIZE_1_GIB,
2626
];
2727

2828
fn cache_porep_params(porep_config: PoRepConfig) {
@@ -35,7 +35,6 @@ fn cache_porep_params(porep_config: PoRepConfig) {
3535
let public_params = public_params(
3636
PaddedBytesAmount::from(porep_config),
3737
usize::from(PoRepProofPartitions::from(porep_config)),
38-
porep_config.window_size_nodes,
3938
);
4039

4140
{
@@ -134,35 +133,24 @@ pub fn main() {
134133
)
135134
.get_matches();
136135

137-
let sizes: HashSet<(u64, usize)> = if matches.is_present("params-for-sector-sizes") {
138-
values_t!(matches.values_of("params-for-sector-sizes"), String)
136+
let sizes: HashSet<u64> = if matches.is_present("params-for-sector-sizes") {
137+
values_t!(matches.values_of("params-for-sector-sizes"), u64)
139138
.unwrap()
140139
.into_iter()
141-
.map(|v| {
142-
let parts = v.split(':').take(2).collect::<Vec<_>>();
143-
assert_eq!(
144-
parts.len(),
145-
2,
146-
"invalid param, expecting sector_size:window_size_nodes"
147-
);
148-
(parts[0].parse().unwrap(), parts[1].parse().unwrap())
149-
})
150140
.collect()
151141
} else {
152142
PUBLISHED_SECTOR_SIZES.iter().cloned().collect()
153143
};
154144

155-
for (sector_size, window_size_nodes) in sizes {
145+
for sector_size in sizes {
156146
cache_post_params(PoStConfig {
157147
sector_size: SectorSize(sector_size),
158-
window_size_nodes,
159148
});
160149

161150
for p in &POREP_PROOF_PARTITION_CHOICES {
162151
cache_porep_params(PoRepConfig {
163152
sector_size: SectorSize(sector_size),
164153
partitions: *p,
165-
window_size_nodes,
166154
});
167155
}
168156
}

‎filecoin-proofs/src/caches.rs

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,6 @@ pub fn get_stacked_params(porep_config: PoRepConfig) -> Result<Arc<groth16::Para
8181
let public_params = public_params(
8282
PaddedBytesAmount::from(porep_config),
8383
usize::from(PoRepProofPartitions::from(porep_config)),
84-
porep_config.window_size_nodes,
8584
);
8685

8786
let parameters_generator = || {
@@ -127,7 +126,6 @@ pub fn get_stacked_verifying_key(porep_config: PoRepConfig) -> Result<Arc<Bls12V
127126
let public_params = public_params(
128127
PaddedBytesAmount::from(porep_config),
129128
usize::from(PoRepProofPartitions::from(porep_config)),
130-
porep_config.window_size_nodes,
131129
);
132130

133131
let vk_generator = || {

‎filecoin-proofs/src/parameters.rs

Lines changed: 16 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
use anyhow::Result;
12
use storage_proofs::drgraph::{DefaultTreeHasher, BASE_DEGREE};
23
use storage_proofs::election_post::{self, ElectionPoSt};
34
use storage_proofs::proof::ProofScheme;
@@ -21,16 +22,26 @@ pub type PostPublicParams = election_post::PublicParams;
2122
pub fn public_params(
2223
sector_bytes: PaddedBytesAmount,
2324
partitions: usize,
24-
window_size_nodes: usize,
2525
) -> stacked::PublicParams<DefaultTreeHasher> {
2626
StackedDrg::<DefaultTreeHasher, DefaultPieceHasher>::setup(&setup_params(
2727
sector_bytes,
2828
partitions,
29-
window_size_nodes,
3029
))
3130
.unwrap()
3231
}
3332

33+
pub fn window_size_nodes_for_sector_bytes(sector_size: PaddedBytesAmount) -> Result<usize> {
34+
use crate::constants::*;
35+
36+
match u64::from(sector_size) {
37+
SECTOR_SIZE_ONE_KIB => Ok(WINDOW_SIZE_NODES_ONE_KIB),
38+
SECTOR_SIZE_16_MIB => Ok(WINDOW_SIZE_NODES_16_MIB),
39+
SECTOR_SIZE_256_MIB => Ok(WINDOW_SIZE_NODES_256_MIB),
40+
SECTOR_SIZE_1_GIB => Ok(WINDOW_SIZE_NODES_1_GIB),
41+
_ => Err(anyhow!("Unknown sector size {:?}", sector_size)),
42+
}
43+
}
44+
3445
pub fn post_public_params(post_config: PoStConfig) -> PostPublicParams {
3546
ElectionPoSt::<DefaultTreeHasher>::setup(&post_setup_params(post_config)).unwrap()
3647
}
@@ -43,16 +54,12 @@ pub fn post_setup_params(post_config: PoStConfig) -> PostSetupParams {
4354
}
4455
}
4556

46-
pub fn setup_params(
47-
sector_bytes: PaddedBytesAmount,
48-
partitions: usize,
49-
window_size_nodes: usize,
50-
) -> stacked::SetupParams {
51-
let sector_bytes = usize::from(sector_bytes);
52-
57+
pub fn setup_params(sector_bytes: PaddedBytesAmount, partitions: usize) -> stacked::SetupParams {
5358
let window_challenges = select_challenges(partitions, POREP_WINDOW_MINIMUM_CHALLENGES, LAYERS);
5459
let wrapper_challenges =
5560
select_challenges(partitions, POREP_WRAPPER_MINIMUM_CHALLENGES, LAYERS);
61+
let window_size_nodes = window_size_nodes_for_sector_bytes(sector_bytes).unwrap();
62+
let sector_bytes = usize::from(sector_bytes);
5663

5764
let config = StackedConfig {
5865
window_challenges,

‎filecoin-proofs/src/types/porep_config.rs

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@ use crate::types::*;
1212
pub struct PoRepConfig {
1313
pub sector_size: SectorSize,
1414
pub partitions: PoRepProofPartitions,
15-
pub window_size_nodes: usize,
1615
}
1716

1817
impl From<PoRepConfig> for PaddedBytesAmount {
@@ -49,11 +48,8 @@ impl From<PoRepConfig> for SectorSize {
4948
impl PoRepConfig {
5049
/// Returns the cache identifier as used by `storage-proofs::paramater_cache`.
5150
pub fn get_cache_identifier(&self) -> String {
52-
let params = crate::parameters::public_params(
53-
self.sector_size.into(),
54-
self.partitions.into(),
55-
self.window_size_nodes,
56-
);
51+
let params =
52+
crate::parameters::public_params(self.sector_size.into(), self.partitions.into());
5753

5854
<StackedCompound as CacheableParameters<
5955
Bls12,

‎filecoin-proofs/src/types/post_config.rs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@ use crate::types::*;
1010
#[derive(Clone, Copy, Debug)]
1111
pub struct PoStConfig {
1212
pub sector_size: SectorSize,
13-
pub window_size_nodes: usize,
1413
}
1514

1615
impl From<PoStConfig> for PaddedBytesAmount {

0 commit comments

Comments
 (0)