Merge pull request zcash#81 from filecoin-project/feat/zigzag

Implement ZigZag PoRep.

src/circuit/drgporep.rs

@@ -54,7 +54,7 @@ pub fn drgporep<E, CS>(
     data_node_path: Vec<Option<(E::Fr, bool)>>,
     data_root: Option<E::Fr>,
     prover_id: Option<&[u8]>,
-    m: usize,
+    degree: usize,
 ) -> Result<(), SynthesisError>
 where
     E: JubjubEngine,
@@ -132,7 +132,7 @@ where
         &params,
         prover_id_bits,
         parents_bits,
-        m,
+        degree,
     )?;
 
     let decoded = sloth::decode(
@@ -166,6 +166,7 @@ mod tests {
     use super::*;
     use circuit::test::*;
     use drgporep;
+    use drgraph::BucketGraph;
     use fr32::{bytes_into_fr, fr_into_bytes};
     use pairing::bls12_381::*;
     use pairing::Field;
@@ -181,16 +182,16 @@ mod tests {
         let rng = &mut XorShiftRng::from_seed([0x3dbe6259, 0x8d313d76, 0x3237db17, 0xe5bc0654]);
 
         let lambda = 32;
-        let n = 12;
-        let m = 6;
+        let nodes = 12;
+        let degree = 6;
         let challenge = 2;
 
         let prover_id: Vec<u8> = fr_into_bytes::<Bls12>(&rng.gen());
-        let mut data: Vec<u8> = (0..n)
+        let mut data: Vec<u8> = (0..nodes)
             .flat_map(|_| fr_into_bytes::<Bls12>(&rng.gen()))
             .collect();
 
-        // TODO: don't clone evertything
+        // TODO: don't clone everything
         let original_data = data.clone();
         let dn = bytes_into_fr::<Bls12>(
             data_at_node(&original_data, challenge, lambda).expect("failed to read original data"),
@@ -200,10 +201,14 @@ mod tests {
 
         let sp = drgporep::SetupParams {
             lambda,
-            drg: drgporep::DrgParams { n, m },
+            drg: drgporep::DrgParams {
+                nodes: nodes,
+                degree,
+            },
         };
 
-        let pp = drgporep::DrgPoRep::setup(&sp).expect("failed to create drgporep setup");
+        let pp =
+            drgporep::DrgPoRep::<BucketGraph>::setup(&sp).expect("failed to create drgporep setup");
         let (tau, aux) =
             drgporep::DrgPoRep::replicate(&pp, prover_id.as_slice(), data.as_mut_slice())
                 .expect("failed to replicate");
@@ -269,7 +274,7 @@ mod tests {
             data_node_path,
             data_root,
             prover_id,
-            m,
+            degree,
         ).expect("failed to synthesize circuit");
 
         if !cs.is_satisfied() {

src/crypto/feistel.rs

@@ -18,7 +18,8 @@ pub fn invert_permute(num_elements: u32, index: u32, keys: &[u32]) -> u32 {
     u
 }
 
-fn encode(num_elements: u32, index: u32, keys: &[u32]) -> u32 {
+/// common_setup performs common calculations on inputs shared by encode and decode.
+fn common_setup(num_elements: u32, index: u32) -> (u32, u32, u32, u32) {
     let mut next_pow4 = 4;
     let mut log4 = 1;
 
@@ -31,8 +32,14 @@ fn encode(num_elements: u32, index: u32, keys: &[u32]) -> u32 {
     let right_mask = (1 << log4) - 1;
     let half_bits = log4;
 
-    let mut left = (index & left_mask) >> half_bits;
-    let mut right = index & right_mask;
+    let left = (index & left_mask) >> half_bits;
+    let right = index & right_mask;
+
+    (left, right, right_mask, half_bits)
+}
+
+fn encode(num_elements: u32, index: u32, keys: &[u32]) -> u32 {
+    let (mut left, mut right, right_mask, half_bits) = common_setup(num_elements, index);
 
     for key in keys.iter().take(4) {
         let (l, r) = (right, left ^ feistel(right, *key, right_mask));
@@ -44,18 +51,7 @@ fn encode(num_elements: u32, index: u32, keys: &[u32]) -> u32 {
 }
 
 fn decode(num_elements: u32, index: u32, keys: &[u32]) -> u32 {
-    let mut next_pow4 = 4;
-    let mut log4 = 1;
-    while next_pow4 < num_elements {
-        next_pow4 *= 4;
-        log4 += 1;
-    }
-    let left_mask = ((1 << log4) - 1) << log4;
-    let right_mask = (1 << log4) - 1;
-    let half_bits = log4;
-
-    let mut left = (index & left_mask) >> half_bits;
-    let mut right = index & right_mask;
+    let (mut left, mut right, right_mask, half_bits) = common_setup(num_elements, index);
 
     for i in (0..4).rev() {
         let (l, r) = ((right ^ feistel(left, keys[i], right_mask)), left);
@@ -94,24 +90,65 @@ fn sha256_digest(data: &[u8]) -> Vec<u8> {
     context.finish().as_ref().into()
 }
 
-#[test]
-fn test_feistel_multiple_of_4() {
-    let n = 16;
-    for i in 0..n {
-        let p = encode(n, i, &[1, 2, 3, 4]);
-        let v = decode(n, p, &[1, 2, 3, 4]);
-        assert_eq!(i, v, "failed to permute");
-        assert!(p <= n, "output number is too big");
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    // Some sample n-values which are not powers of four and also don't coincidentally happen to
+    // encode/decode correctly.
+    const BAD_NS: &[u32] = &[5, 6, 8, 12, 17];
+
+    fn encode_decode(n: u32, expect_success: bool) {
+        let mut failed = false;
+        for i in 0..n {
+            let p = encode(n, i, &[1, 2, 3, 4]);
+            let v = decode(n, p, &[1, 2, 3, 4]);
+            let equal = i == v;
+            let in_range = p <= n;
+            if expect_success {
+                assert!(equal, "failed to permute (n = {})", n);
+                assert!(in_range, "output number is too big (n = {})", n);
+            } else {
+                if !equal || !in_range {
+                    failed = true;
+                }
+            }
+        }
+        if !expect_success {
+            assert!(failed, "expected failure (n = {})", n);
+        }
+    }
+
+    #[test]
+    fn test_feistel_power_of_4() {
+        // Our implementation is guaranteed to produce a permutation when input size (number of elements)
+        // is a power of our.
+        let mut n = 1;
+
+        // Powers of 4 always succeed.
+        for _ in 0..4 {
+            n *= 4;
+            encode_decode(n, true);
+        }
+
+        // Some non-power-of 4 also succeed, but here is a selection of examples values showing
+        // that this is not guaranteed.
+        for i in BAD_NS.iter() {
+            encode_decode(*i, false);
+        }
     }
-}
 
-#[test]
-fn test_feistel_on_arbitrary_set() {
-    let n = 12;
-    for i in 0..n {
-        let p = permute(n, i, &[1, 2, 3, 4]);
-        let v = invert_permute(n, p, &[1, 2, 3, 4]);
-        assert_eq!(i, v, "failed to permute");
-        assert!(p <= n, "output number is too big");
+    #[test]
+    fn test_feistel_on_arbitrary_set() {
+        for n in BAD_NS.iter() {
+            for i in 0..*n {
+                let p = permute(*n, i, &[1, 2, 3, 4]);
+                let v = invert_permute(*n, p, &[1, 2, 3, 4]);
+                // Since every element in the set is reversibly mapped to another element also in the set,
+                // this is indeed a permutation.
+                assert_eq!(i, v, "failed to permute");
+                assert!(p <= *n, "output number is too big");
+            }
+        }
     }
 }

src/drgporep.rs

@@ -1,9 +1,11 @@
 use crypto::{kdf, sloth};
-use drgraph::{BucketGraph, Graph, MerkleProof};
+use drgraph::{Graph, MerkleProof};
+
 use fr32::{bytes_into_fr, fr_into_bytes};
 use pairing::bls12_381::{Bls12, Fr};
 use pairing::{PrimeField, PrimeFieldRepr};
 use porep::{self, PoRep};
+use std::marker::PhantomData;
 use util::data_at_node;
 use vde::{self, decode_block};
 
@@ -31,8 +33,11 @@ pub struct SetupParams {
 
 #[derive(Debug, Clone)]
 pub struct DrgParams {
-    pub n: usize,
-    pub m: usize,
+    // Number of nodes
+    pub nodes: usize,
+
+    // Base degree of DRG
+    pub degree: usize,
 }
 
 #[derive(Debug, Clone)]
@@ -80,17 +85,19 @@ impl Proof {
 }
 
 #[derive(Default)]
-pub struct DrgPoRep {}
+pub struct DrgPoRep<G: Graph> {
+    phantom: PhantomData<G>,
+}
 
-impl<'a> ProofScheme<'a> for DrgPoRep {
-    type PublicParams = PublicParams<BucketGraph>;
+impl<'a, G: Graph> ProofScheme<'a> for DrgPoRep<G> {
+    type PublicParams = PublicParams<G>;
     type SetupParams = SetupParams;
     type PublicInputs = PublicInputs<'a>;
     type PrivateInputs = PrivateInputs<'a>;
     type Proof = Proof;
 
     fn setup(sp: &Self::SetupParams) -> Result<Self::PublicParams> {
-        let graph = BucketGraph::new(sp.drg.n, sp.drg.m);
+        let graph = G::new(sp.drg.nodes, sp.drg.degree);
 
         Ok(PublicParams {
             lambda: sp.lambda,
@@ -149,13 +156,13 @@ impl<'a> ProofScheme<'a> for DrgPoRep {
         }
 
         let expected_parents = pub_params.graph.parents(challenge);
-        let actual_parents = proof
+        let parents_as_expected = proof
             .replica_parents
             .iter()
-            .map(|x| x.0)
-            .collect::<Vec<_>>();
+            .zip(expected_parents)
+            .all(|(actual, expected)| actual.0 == expected);
 
-        if expected_parents != actual_parents {
+        if !parents_as_expected {
             println!("proof parents were not those provided in public parameters");
             return Ok(false);
         }
@@ -187,12 +194,12 @@ impl<'a> ProofScheme<'a> for DrgPoRep {
     }
 }
 
-impl<'a> PoRep<'a> for DrgPoRep {
+impl<'a, G: Graph> PoRep<'a> for DrgPoRep<G> {
     type Tau = porep::Tau;
     type ProverAux = porep::ProverAux;
 
     fn replicate(
-        pp: &PublicParams<BucketGraph>,
+        pp: &Self::PublicParams,
         prover_id: &[u8],
         data: &mut [u8],
     ) -> Result<(porep::Tau, porep::ProverAux)> {
@@ -216,7 +223,7 @@ impl<'a> PoRep<'a> for DrgPoRep {
     }
 
     fn extract_all<'b>(
-        pp: &'b PublicParams<BucketGraph>,
+        pp: &'b Self::PublicParams,
         prover_id: &'b [u8],
         data: &'b [u8],
     ) -> Result<Vec<u8>> {
@@ -229,7 +236,7 @@ impl<'a> PoRep<'a> for DrgPoRep {
     }
 
     fn extract(
-        pp: &PublicParams<BucketGraph>,
+        pp: &Self::PublicParams,
         prover_id: &[u8],
         data: &[u8],
         node: usize,
@@ -247,6 +254,7 @@ impl<'a> PoRep<'a> for DrgPoRep {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use drgraph::BucketGraph;
     use rand::{Rng, SeedableRng, XorShiftRng};
 
     #[test]
@@ -260,12 +268,12 @@ mod tests {
         let sp = SetupParams {
             lambda: lambda,
             drg: DrgParams {
-                n: data.len() / lambda,
-                m: 10,
+                nodes: data.len() / lambda,
+                degree: 10,
             },
         };
 
-        let pp = DrgPoRep::setup(&sp).unwrap();
+        let pp = DrgPoRep::<BucketGraph>::setup(&sp).unwrap();
 
         DrgPoRep::replicate(&pp, prover_id.as_slice(), data_copy.as_mut_slice()).unwrap();
 
@@ -290,12 +298,12 @@ mod tests {
         let sp = SetupParams {
             lambda: lambda,
             drg: DrgParams {
-                n: data.len() / lambda,
-                m: 10,
+                nodes: data.len() / lambda,
+                degree: 10,
             },
         };
 
-        let pp = DrgPoRep::setup(&sp).unwrap();
+        let pp = DrgPoRep::<BucketGraph>::setup(&sp).unwrap();
 
         DrgPoRep::replicate(&pp, prover_id.as_slice(), data_copy.as_mut_slice()).unwrap();
 
@@ -317,18 +325,18 @@ mod tests {
 
     fn prove_verify_aux(
         lambda: usize,
-        n: usize,
+        nodes: usize,
         i: usize,
         use_wrong_challenge: bool,
         use_wrong_parents: bool,
     ) {
         let rng = &mut XorShiftRng::from_seed([0x3dbe6259, 0x8d313d76, 0x3237db17, 0xe5bc0654]);
 
-        let m = i * 10;
+        let degree = 5;
         let lambda = lambda;
 
         let prover_id = fr_into_bytes::<Bls12>(&rng.gen());
-        let data: Vec<u8> = (0..n)
+        let data: Vec<u8> = (0..nodes)
             .flat_map(|_| fr_into_bytes::<Bls12>(&rng.gen()))
             .collect();
 
@@ -338,10 +346,10 @@ mod tests {
 
         let sp = SetupParams {
             lambda,
-            drg: DrgParams { n, m },
+            drg: DrgParams { nodes, degree },
         };
 
-        let pp = DrgPoRep::setup(&sp).unwrap();
+        let pp = DrgPoRep::<BucketGraph>::setup(&sp).unwrap();
 
         let (tau, aux) =
             DrgPoRep::replicate(&pp, prover_id.as_slice(), data_copy.as_mut_slice()).unwrap();
@@ -365,10 +373,11 @@ mod tests {
             // Only one 'wrong' option will be tested at a time.
             assert!(!use_wrong_challenge);
             let real_parents = real_proof.replica_parents;
-            // A real node will never have all parents equal to 1.
             let fake_parents = real_parents
                 .iter()
-                .map(|(p, data_proof)| (1, data_proof.clone()))
+                // Incrementing each parent node will give us a different parent set.
+                // It's fine to be out of range, since this only needs to fail.
+                .map(|(i, data_proof)| (i + 1, data_proof.clone()))
                 .collect::<Vec<_>>();
 
             let proof = Proof::new(