From 530e5c46571676768ec7fa486ce21bbb69c8001c Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Thu, 28 Jun 2018 16:39:58 +0200 Subject: [PATCH 1/3] start implementation of faster drgporep input generation --- examples/drgporep.rs | 108 +++++++++--------------------------- src/test_helper.rs | 127 ++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 144 insertions(+), 91 deletions(-) diff --git a/examples/drgporep.rs b/examples/drgporep.rs index c7202ac2..03a8903e 100644 --- a/examples/drgporep.rs +++ b/examples/drgporep.rs @@ -7,16 +7,14 @@ extern crate sapling_crypto; use bellman::groth16::*; use bellman::{Circuit, ConstraintSystem, SynthesisError}; -use pairing::bls12_381::{Bls12, Fr}; +use pairing::bls12_381::Bls12; use proofs::example_helper::Example; use rand::Rng; use sapling_crypto::jubjub::{JubjubBls12, JubjubEngine}; -use proofs::fr32::{bytes_into_fr, fr_into_bytes}; -use proofs::porep::PoRep; -use proofs::proof::ProofScheme; -use proofs::util::data_at_node; -use proofs::{circuit, drgporep}; +use proofs::circuit; +use proofs::fr32::fr_into_bytes; +use proofs::test_helper::fake_drgpoprep_proof; struct DrgPoRepExample<'a, E: JubjubEngine> { params: &'a E::Params, @@ -56,6 +54,8 @@ impl<'a, E: JubjubEngine> Circuit for DrgPoRepExample<'a, E> { #[derive(Default)] struct DrgPoRepApp {} +const SLOTH_ROUNDS: usize = 1; + impl Example for DrgPoRepApp { fn name() -> String { "DrgPoRep".to_string() @@ -102,97 +102,41 @@ impl Example for DrgPoRepApp { rng: &mut R, engine_params: &JubjubBls12, groth_params: &Parameters, - _tree_depth: usize, + tree_depth: usize, _challenge_count: usize, - leaves: usize, + _leaves: usize, lambda: usize, m: usize, ) -> Proof { - let prover_id: Fr = rng.gen(); - let prover_id_bytes = fr_into_bytes::(&prover_id); - let mut data: Vec = (0..leaves) - .flat_map(|_| fr_into_bytes::(&rng.gen())) - .collect(); - let original_data = data.clone(); - let challenge = 2; - - let sp = drgporep::SetupParams { - lambda, - drg: drgporep::DrgParams { n: leaves, m }, - }; - - let pp = drgporep::DrgPoRep::setup(&sp).expect("failed to create drgporep setup"); - - let (tau, aux) = - drgporep::DrgPoRep::replicate(&pp, prover_id_bytes.as_slice(), data.as_mut_slice()) - .expect("failed to replicate"); - - let pub_inputs = drgporep::PublicInputs { - prover_id: &prover_id, - challenge, - tau: &tau, - }; - let priv_inputs = drgporep::PrivateInputs { - replica: data.as_slice(), - aux: &aux, - }; - - let proof_nc = - drgporep::DrgPoRep::prove(&pp, &pub_inputs, &priv_inputs).expect("failed to prove"); - - assert!( - drgporep::DrgPoRep::verify(&pp, &pub_inputs, &proof_nc).expect("failed to verify"), - "failed to verify (non circuit)" - ); - - let replica_node = Some(&proof_nc.replica_node.data); - - let replica_node_path = proof_nc.replica_node.proof.as_options(); - let replica_root = Some(proof_nc.replica_node.proof.root().into()); - let replica_parents: Vec<_> = proof_nc - .replica_parents - .iter() - .map(|(_, parent)| Some(&parent.data)) - .collect(); - let replica_parents_paths: Vec<_> = proof_nc - .replica_parents - .iter() - .map(|(_, parent)| parent.proof.as_options()) - .collect(); - let data_node = bytes_into_fr::( - data_at_node(&original_data, challenge, lambda).expect("failed to read original data"), - ).unwrap(); - - let data_node_path = proof_nc.node.as_options(); - let data_root = Some(proof_nc.node.root().into()); - let prover_id = Some(prover_id_bytes.as_slice()); + let ( + prover_id, + replica_node, + replica_node_path, + replica_root, + replica_parents, + replica_parents_paths, + data_node, + data_node_path, + data_root, + ) = fake_drgpoprep_proof(rng, tree_depth, m, SLOTH_ROUNDS); - assert!( - proof_nc.node.validate(challenge), - "failed to verify data commitment" - ); - assert!( - proof_nc.node.validate_data(&data_node), - "failed to verify data commitment with data" - ); // create an instance of our circut (with the witness) let c = DrgPoRepExample { params: engine_params, lambda: lambda * 8, - replica_node, + replica_node: Some(&replica_node), replica_node_path: &replica_node_path, - replica_root, - replica_parents, + replica_root: Some(replica_root), + replica_parents: replica_parents.iter().map(|parent| Some(parent)).collect(), replica_parents_paths: &replica_parents_paths, data_node: Some(&data_node), - data_node_path, - data_root, - prover_id, + data_node_path: data_node_path.clone(), + data_root: Some(data_root), + prover_id: Some(prover_bytes.as_slice()), m, }; - // create groth16 proof - create_random_proof(c, groth_params, rng).expect("failed to create random proof") + create_random_proof(c, groth_params, rng).expect("failed to create proof") } fn verify_proof( diff --git a/src/test_helper.rs b/src/test_helper.rs index e70cd024..da6a1c32 100644 --- a/src/test_helper.rs +++ b/src/test_helper.rs @@ -1,6 +1,9 @@ use crypto; +use drgraph::{MerkleProof, MerkleTree}; +use error; use fr32::{bytes_into_fr, fr_into_bytes}; use pairing::bls12_381::{Bls12, Fr}; +use pairing::PrimeFieldRepr; use pairing::{BitIterator, PrimeField}; use rand::Rng; use sapling_crypto::pedersen_hash; @@ -21,18 +24,113 @@ macro_rules! table_tests { } } -pub fn random_merkle_path( +pub fn fake_drgpoprep_proof( rng: &mut R, tree_depth: usize, -) -> (Vec>, Fr, Fr) { - let auth_path: Vec> = vec![Some((rng.gen(), rng.gen())); tree_depth]; + m: usize, + sloth_rounds: usize, +) -> ( + Fr, + Fr, + Vec>, + Fr, + Vec, + Vec>>, + Fr, + Vec>, + Fr, +) { + let prover_id: Fr = rng.gen(); + let challenge = m + 1; + // Part 1: original data inputs + // generate a leaf + let data_node: Fr = rng.gen(); + // generate a fake merkle tree for the leaf and get commD + let (data_node_path, data_root) = random_merkle_path_with_value(rng, tree_depth, &data_node, 0); - let value: Fr = rng.gen(); + // Part 2: replica data inputs + // generate parent nodes + let replica_parents: Vec = (0..m).map(|_| rng.gen()).collect(); + // run kdf for proverid, parent nodes + let ciphertexts = replica_parents + .iter() + .fold( + Ok(fr_into_bytes::(&prover_id)), + |acc: error::Result>, parent: &Fr| { + acc.and_then(|mut acc| { + parent.into_repr().write_le(&mut acc)?; + Ok(acc) + }) + }, + ) + .unwrap(); + let key = crypto::kdf::kdf::(ciphertexts.as_slice(), m); + // run sloth(key, node) + let replica_node: Fr = crypto::sloth::encode::(&key, &data_node, sloth_rounds); + // run fake merkle with only the first 1+m real leaves + + let mut leaves = replica_parents.clone(); + leaves.push(data_node); + // ensure we have an even number of leaves + if m + 1 % 2 != 0 { + leaves.push(rng.gen()); + } + + // get commR + let subtree = MerkleTree::from_data(leaves); + let subtree_root: Fr = subtree.root().into(); + let subtree_depth = subtree.height(); + let remaining_depth = tree_depth - subtree_depth; + let (remaining_path, replica_root) = + random_merkle_path_with_value(rng, remaining_depth, &subtree_root, remaining_depth); + + // generate merkle path for challenged node and parents + let replica_parents_paths: Vec<_> = (0..m) + .map(|i| { + let subtree_proof: MerkleProof = subtree.gen_proof(i).into(); + let mut subtree_path = subtree_proof.as_options(); + subtree_path.extend(&remaining_path); + subtree_path + }) + .collect(); + + let replica_node_path = { + let subtree_proof: MerkleProof = subtree.gen_proof(challenge).into(); + let mut subtree_path = subtree_proof.as_options(); + subtree_path.extend(&remaining_path); + subtree_path + }; + + ( + prover_id, + replica_node, + replica_node_path, + replica_root, + replica_parents, + replica_parents_paths, + data_node, + data_node_path, + data_root, + ) +} + +pub fn random_merkle_path_with_value( + rng: &mut R, + tree_depth: usize, + value: &Fr, + offset: usize, +) -> (Vec>, Fr) { + let auth_path: Vec> = vec![Some((rng.gen(), rng.gen())); tree_depth]; // TODO: cleanup - let h = - crypto::pedersen::pedersen_compression(&bytes_into_bits(&fr_into_bytes::(&value))); - let mut cur = bytes_into_fr::(&bits_to_bytes(&h)).unwrap(); + let mut cur = if offset == 0 { + let h = crypto::pedersen::pedersen_compression(&bytes_into_bits(&fr_into_bytes::( + &value, + ))); + bytes_into_fr::(&bits_to_bytes(&h)).unwrap() + } else { + *value + }; for (i, p) in auth_path.clone().into_iter().enumerate() { let (uncle, is_right) = p.unwrap(); @@ -50,7 +148,7 @@ pub fn random_merkle_path( rhs.reverse(); cur = pedersen_hash::pedersen_hash::( - pedersen_hash::Personalization::MerkleTree(i), + pedersen_hash::Personalization::MerkleTree(i + offset), lhs.into_iter() .take(Fr::NUM_BITS as usize) .chain(rhs.into_iter().take(Fr::NUM_BITS as usize)), @@ -59,5 +157,16 @@ pub fn random_merkle_path( .0; } - (auth_path, value, cur) + (auth_path, cur) +} + +pub fn random_merkle_path( + rng: &mut R, + tree_depth: usize, +) -> (Vec>, Fr, Fr) { + let value: Fr = rng.gen(); + + let (path, root) = random_merkle_path_with_value(rng, tree_depth, &value, 0); + + (path, value, root) } From d39a4b66c43dccbf943b64c3f31a3e0d9c308ca4 Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Tue, 3 Jul 2018 17:52:05 +0200 Subject: [PATCH 2/3] fix height generation --- README.md | 4 ++-- src/test_helper.rs | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 7fdbe84d..dfe73855 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ Configure to use nightly: ## Build ``` -> cargo build +> cargo build --release --features u128-support ``` ## Test @@ -28,7 +28,7 @@ Configure to use nightly: Build ``` -> cargo build --examples --release +> cargo build --examples --release --features u128-support ``` Running them diff --git a/src/test_helper.rs b/src/test_helper.rs index da6a1c32..7c4f738d 100644 --- a/src/test_helper.rs +++ b/src/test_helper.rs @@ -79,7 +79,7 @@ pub fn fake_drgpoprep_proof( // get commR let subtree = MerkleTree::from_data(leaves); let subtree_root: Fr = subtree.root().into(); - let subtree_depth = subtree.height(); + let subtree_depth = subtree.height() - 1; // .height() inludes the leave let remaining_depth = tree_depth - subtree_depth; let (remaining_path, replica_root) = random_merkle_path_with_value(rng, remaining_depth, &subtree_root, remaining_depth); @@ -101,6 +101,8 @@ pub fn fake_drgpoprep_proof( subtree_path }; + assert_eq!(data_node_path.len(), replica_node_path.len()); + ( prover_id, replica_node, From 4ec01de210fd93c6e51b073c226aea2735858f0c Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Fri, 6 Jul 2018 15:16:19 +0200 Subject: [PATCH 3/3] feat: implement bench subcommand for examples --- examples/drgporep.rs | 45 +++++++++++++ examples/merklepor.rs | 30 +++++++++ src/example_helper.rs | 143 ++++++++++++++++++++++++++++++++++++++---- 3 files changed, 206 insertions(+), 12 deletions(-) diff --git a/examples/drgporep.rs b/examples/drgporep.rs index 03a8903e..32b41017 100644 --- a/examples/drgporep.rs +++ b/examples/drgporep.rs @@ -13,6 +13,7 @@ use rand::Rng; use sapling_crypto::jubjub::{JubjubBls12, JubjubEngine}; use proofs::circuit; +use proofs::circuit::bench::BenchCS; use proofs::fr32::fr_into_bytes; use proofs::test_helper::fake_drgpoprep_proof; @@ -120,6 +121,8 @@ impl Example for DrgPoRepApp { data_root, ) = fake_drgpoprep_proof(rng, tree_depth, m, SLOTH_ROUNDS); + let prover_bytes = fr_into_bytes::(&prover_id); + // create an instance of our circut (with the witness) let c = DrgPoRepExample { params: engine_params, @@ -147,6 +150,48 @@ impl Example for DrgPoRepApp { // not implemented yet None } + fn create_bench( + &mut self, + rng: &mut R, + engine_params: &JubjubBls12, + tree_depth: usize, + _challenge_count: usize, + _leaves: usize, + lambda: usize, + m: usize, + ) { + let ( + prover_id, + replica_node, + replica_node_path, + replica_root, + replica_parents, + replica_parents_paths, + data_node, + data_node_path, + data_root, + ) = fake_drgpoprep_proof(rng, tree_depth, m, SLOTH_ROUNDS); + + let prover_bytes = fr_into_bytes::(&prover_id); + // create an instance of our circut (with the witness) + let c = DrgPoRepExample { + params: engine_params, + lambda: lambda * 8, + replica_node: Some(&replica_node), + replica_node_path: &replica_node_path, + replica_root: Some(replica_root), + replica_parents: replica_parents.iter().map(|parent| Some(parent)).collect(), + replica_parents_paths: &replica_parents_paths, + data_node: Some(&data_node), + data_node_path: data_node_path.clone(), + data_root: Some(data_root), + prover_id: Some(prover_bytes.as_slice()), + m, + }; + + let mut cs = BenchCS::::new(); + c.synthesize(&mut cs).expect("failed to synthesize circuit"); + } } fn main() { diff --git a/examples/merklepor.rs b/examples/merklepor.rs index c019ff01..9eac3de8 100644 --- a/examples/merklepor.rs +++ b/examples/merklepor.rs @@ -6,6 +6,7 @@ extern crate rand; extern crate sapling_crypto; use bellman::groth16::*; +use bellman::Circuit; use pairing::bls12_381::{Bls12, Fr}; use pairing::Field; use rand::Rng; @@ -13,6 +14,7 @@ use sapling_crypto::circuit::multipack; use sapling_crypto::jubjub::JubjubBls12; use proofs::circuit; +use proofs::circuit::bench::BenchCS; use proofs::example_helper::Example; use proofs::test_helper::random_merkle_path; @@ -130,6 +132,34 @@ impl Example for MerklePorApp { // -- verify proof with public inputs Some(verify_proof(pvk, proof, &expected_inputs).expect("failed to verify proof")) } + + fn create_bench( + &mut self, + rng: &mut R, + engine_params: &JubjubBls12, + tree_depth: usize, + challenge_count: usize, + _leaves: usize, + _lambda: usize, + _m: usize, + ) { + let (auth_path, leaf, root) = random_merkle_path(rng, tree_depth); + self.root = root; + self.leaf = leaf; + self.auth_paths = (0..challenge_count).map(|_| auth_path.clone()).collect(); + let values = (0..challenge_count).map(|_| Some(&self.leaf)).collect(); + + // create an instance of our circut (with the witness) + let c = circuit::ppor::ParallelProofOfRetrievability { + params: engine_params, + values, + auth_paths: &self.auth_paths, + root: Some(self.root), + }; + + let mut cs = BenchCS::::new(); + c.synthesize(&mut cs).expect("failed to synthesize circuit"); + } } fn main() { diff --git a/src/example_helper.rs b/src/example_helper.rs index 005ab79d..b41dc96d 100644 --- a/src/example_helper.rs +++ b/src/example_helper.rs @@ -1,5 +1,5 @@ use bellman::groth16::*; -use clap::{App, Arg}; +use clap::{self, App, Arg, SubCommand}; use colored::*; use env_logger; use indicatif::{ProgressBar, ProgressStyle}; @@ -11,6 +11,27 @@ use std::io::Write; use std::path::Path; use std::time::{Duration, Instant}; +fn prettyb(num: usize) -> String { + let num = num as f64; + let negative = if num.is_sign_positive() { "" } else { "-" }; + let num = num.abs(); + let units = ["B", "kB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"]; + if num < 1_f64 { + return format!("{}{} {}", negative, num, "B"); + } + let delimiter = 1024_f64; + let exponent = ::std::cmp::min( + (num.ln() / delimiter.ln()).floor() as i32, + (units.len() - 1) as i32, + ); + let pretty_bytes = format!("{:.2}", num / delimiter.powi(exponent)) + .parse::() + .unwrap() * 1_f64; + let unit = units[exponent as usize]; + format!("{}{} {}", negative, pretty_bytes, unit) +} + +/// Generate a unique cache path, based on the inputs. fn get_cache_path(name: &str, data_size: usize, challenge_count: usize, m: usize) -> String { format!( "/tmp/filecoin-proofs-cache-{}-{}-{}-{}", @@ -21,8 +42,17 @@ fn get_cache_path(name: &str, data_size: usize, challenge_count: usize, m: usize ) } +/// The available circuit types for benchmarking. +#[derive(Debug)] +pub enum CSType { + Groth, + Bench, +} + +/// A trait that makes it easy to implement "Examples". These are really tunable benchmarking CLI tools. pub trait Example: Default { - fn do_the_work(&mut self, data_size: usize, challenge_count: usize, m: usize) { + /// The actual work. + fn work_groth(&mut self, typ: CSType, data_size: usize, challenge_count: usize, m: usize) { let engine_params = Self::generate_engine_params(); let rng = &mut XorShiftRng::from_seed([0x3dbe6259, 0x8d313d76, 0x3237db17, 0xe5bc0654]); @@ -30,7 +60,8 @@ pub trait Example: Default { let leaves = data_size / 32; let tree_depth = (leaves as f64).log2().ceil() as usize; - info!(target: "config", "data size: {} bytes", data_size); + info!(target: "config", "constraint system: {:?}", typ); + info!(target: "config", "data size: {}", prettyb(data_size)); info!(target: "config", "m: {}", m); info!(target: "config", "tree depth: {}", tree_depth); @@ -136,10 +167,63 @@ pub trait Example: Default { info!(".") } - fn main() { - let mut instance = Self::default(); - // set default logging level to info + fn work_bench(&mut self, typ: CSType, data_size: usize, challenge_count: usize, m: usize) { + let engine_params = Self::generate_engine_params(); + let rng = &mut XorShiftRng::from_seed([0x3dbe6259, 0x8d313d76, 0x3237db17, 0xe5bc0654]); + + let lambda = 32; + let leaves = data_size / 32; + let tree_depth = (leaves as f64).log2().ceil() as usize; + + info!(target: "config", "constraint system: {:?}", typ); + info!(target: "config", "data size: {}", prettyb(data_size)); + info!(target: "config", "m: {}", m); + info!(target: "config", "tree depth: {}", tree_depth); + // need more samples as this is a faster operation + let samples = (Self::samples() * 10) as u32; + + let mut total_synth = Duration::new(0, 0); + + let pb = ProgressBar::new(u64::from(samples)); + pb.set_style( + ProgressStyle::default_bar() + .template( + "{spinner:.green} [{elapsed_precise}] [{bar:40.cyan/blue}] {pos}/{len} ({eta})", + ) + .progress_chars("#>-"), + ); + + for _ in 0..samples { + // -- create proof + + let start = Instant::now(); + self.create_bench( + rng, + &engine_params, + tree_depth, + challenge_count, + leaves, + lambda, + m, + ); + total_synth += start.elapsed(); + pb.inc(1); + } + + // -- print statistics + + let synth_avg = total_synth / samples; + let synth_avg = + f64::from(synth_avg.subsec_nanos()) / 1_000_000_000f64 + (synth_avg.as_secs() as f64); + + info!(target: "stats", "Average synthesize time: {:?} seconds", synth_avg); + + // need this, as the last item doesn't get flushed to the console sometimes + info!(".") + } + + fn init_logger(&self) { let mut builder = env_logger::Builder::new(); builder .filter_level(LevelFilter::Info) @@ -162,8 +246,10 @@ pub trait Example: Default { ) }) .init(); + } - let matches = App::new(Self::name()) + fn clap(&self) -> clap::ArgMatches { + App::new(stringify!($name)) .version("1.0") .arg( Arg::with_name("size") @@ -185,13 +271,43 @@ pub trait Example: Default { .default_value("6") .takes_value(true), ) - .get_matches(); + .subcommand( + SubCommand::with_name("groth") + .about("execute circuits using groth constraint system"), + ) + .subcommand( + SubCommand::with_name("bench") + .about("execute circuits using a minimal benchmarking constraint"), + ) + .get_matches() + } + + fn main() { + let mut instance = Self::default(); + // set default logging level to info + + instance.init_logger(); + + let (data_size, challenge_count, m, typ) = { + let matches = instance.clap(); + + let data_size = value_t!(matches, "size", usize).unwrap() * 1024; + let challenge_count = value_t!(matches, "challenges", usize).unwrap(); + let m = value_t!(matches, "m", usize).unwrap(); - let data_size = value_t!(matches, "size", usize).unwrap() * 1024; - let challenge_count = value_t!(matches, "challenges", usize).unwrap(); - let m = value_t!(matches, "m", usize).unwrap(); + let typ = match matches.subcommand_name() { + Some("groth") => CSType::Groth, + Some("bench") => CSType::Bench, + _ => panic!("please select a valid subcommand"), + }; - instance.do_the_work(data_size, challenge_count, m); + (data_size, challenge_count, m, typ) + }; + + match typ { + CSType::Groth => instance.work_groth(typ, data_size, challenge_count, m), + CSType::Bench => instance.work_bench(typ, data_size, challenge_count, m), + } } /// The name of the application. Used for identifying caches. @@ -229,4 +345,7 @@ pub trait Example: Default { /// Verify the given proof, return `None` if not implemented. fn verify_proof(&mut self, &Proof, &PreparedVerifyingKey) -> Option; + + /// Create a new bench + fn create_bench(&mut self, &mut R, &E::Params, usize, usize, usize, usize, usize); }