conditional encode (#1394)

* feat: conditional encoding of batch to blob

* chore: add worked example

* fix: construction of blob field elements

* fix: use actual vs cooked len in appropriate assertions

* use batch instead of blob (no encoding) and load seqexconf correct

* refactor get_blob_data_bytes to avoid inconsistency

* get blob from BatchProvingTask | related changes

* lower num advices (but only used for tests)

* b64 serde

* refactor(prover): move params out of prover/verifier

* clean

* remove redundant function 'degrees'

* add sanity check (decoded blob == batch bytes)

* test: decode_blob

* make some methods public to work with scroll-prover

---------

Co-authored-by: Zhang Zhuo <mycinbrin@gmail.com>

.gitignore

@@ -4,8 +4,7 @@
 .vscode
 .idea
 *.log
-*.json
 *.sh
 *.txt
 *.srs
-tmp
+tmp

aggregator/data/worked-example

@@ -0,0 +1,34 @@
+Romeo and Juliet
+Excerpt from Act 2, Scene 2
+
+JULIET
+O Romeo, Romeo! wherefore art thou Romeo?
+Deny thy father and refuse thy name;
+Or, if thou wilt not, be but sworn my love,
+And I'll no longer be a Capulet.
+
+ROMEO
+[Aside] Shall I hear more, or shall I speak at this?
+
+JULIET
+'Tis but thy name that is my enemy;
+Thou art thyself, though not a Montague.
+What's Montague? it is nor hand, nor foot,
+Nor arm, nor face, nor any other part
+Belonging to a man. O, be some other name!
+What's in a name? that which we call a rose
+By any other name would smell as sweet;
+So Romeo would, were he not Romeo call'd,
+Retain that dear perfection which he owes
+Without that title. Romeo, doff thy name,
+And for that name which is no part of thee
+Take all myself.
+
+ROMEO
+I take thee at thy word:
+Call me but love, and I'll be new baptized;
+Henceforth I never will be Romeo.
+
+JULIET
+What man art thou that thus bescreen'd in night
+So stumblest on my counsel?

aggregator/src/aggregation/barycentric.rs

@@ -351,6 +351,7 @@ mod tests {
     use super::*;
     use crate::{
         blob::{BatchData, KZG_TRUSTED_SETUP},
+        eip4844::{get_blob_bytes, get_coefficients},
         MAX_AGG_SNARKS,
     };
     use c_kzg::{Blob as RethBlob, KzgProof};
@@ -395,12 +396,15 @@ mod tests {
             vec![0; 340],
             vec![10; 23],
         ]);
+        let batch_bytes = batch.get_batch_data_bytes();
+        let blob_bytes = get_blob_bytes(&batch_bytes);
+        let coeffs = get_coefficients(&blob_bytes);
 
         for z in 0..10 {
             let z = Scalar::from(u64::try_from(13241234 + z).unwrap());
             assert_eq!(
-                reth_point_evaluation(z, &batch.get_coefficients().map(|c| Scalar::from_raw(c.0))),
-                interpolate(z, &batch.get_coefficients().map(|c| Scalar::from_raw(c.0)))
+                reth_point_evaluation(z, &coeffs.map(|c| Scalar::from_raw(c.0))),
+                interpolate(z, &coeffs.map(|c| Scalar::from_raw(c.0)))
             );
         }
     }

aggregator/src/aggregation/batch_data.rs

@@ -1,3 +1,4 @@
+use eth_types::H256;
 use ethers_core::utils::keccak256;
 use halo2_ecc::bigint::CRTInteger;
 use halo2_proofs::{
@@ -372,6 +373,7 @@ impl<const N_SNARKS: usize> BatchDataConfig<N_SNARKS> {
         config
     }
 
+    #[allow(clippy::too_many_arguments)]
     pub fn assign(
         &self,
         layouter: &mut impl Layouter<Fr>,
@@ -381,13 +383,14 @@ impl<const N_SNARKS: usize> BatchDataConfig<N_SNARKS> {
         // `core.rs`. Since these are already constrained, we can just use them as is.
         chunks_are_padding: &[AssignedCell<Fr, Fr>],
         batch_data: &BatchData<N_SNARKS>,
+        versioned_hash: H256,
         barycentric_assignments: &[CRTInteger<Fr>],
     ) -> Result<AssignedBatchDataExport, Error> {
         self.load_range_tables(layouter)?;
 
         let assigned_rows = layouter.assign_region(
             || "BatchData rows",
-            |mut region| self.assign_rows(&mut region, challenge_value, batch_data),
+            |mut region| self.assign_rows(&mut region, challenge_value, batch_data, versioned_hash),
         )?;
 
         layouter.assign_region(
@@ -415,13 +418,14 @@ impl<const N_SNARKS: usize> BatchDataConfig<N_SNARKS> {
         region: &mut Region<Fr>,
         challenge_value: Challenges<Value<Fr>>,
         batch_data: &BatchData<N_SNARKS>,
+        versioned_hash: H256,
     ) -> Result<Vec<AssignedBatchDataConfig>, Error> {
         let n_rows_data = BatchData::<N_SNARKS>::n_rows_data();
         let n_rows_metadata = BatchData::<N_SNARKS>::n_rows_metadata();
         let n_rows_digest_rlc = BatchData::<N_SNARKS>::n_rows_digest_rlc();
         let n_rows_total = BatchData::<N_SNARKS>::n_rows();
 
-        let rows = batch_data.to_rows(challenge_value);
+        let rows = batch_data.to_rows(versioned_hash, challenge_value);
         assert_eq!(rows.len(), n_rows_total);
 
         // enable data selector

aggregator/src/aggregation/blob_data.rs

@@ -1,19 +1,17 @@
-use std::io::Write;
-
 use gadgets::util::Expr;
 use halo2_ecc::bigint::CRTInteger;
 use halo2_proofs::{
     circuit::{AssignedCell, Layouter, Region, Value},
     halo2curves::bn256::Fr,
-    plonk::{Advice, Column, ConstraintSystem, Error, Expression, SecondPhase, Selector},
+    plonk::{Advice, Column, ConstraintSystem, Error, Expression, Fixed, SecondPhase, Selector},
     poly::Rotation,
 };
 use itertools::Itertools;
 use zkevm_circuits::{table::U8Table, util::Challenges};
 
 use crate::{
-    aggregation::{decoder::witgen::init_zstd_encoder, rlc::POWS_OF_256},
-    blob::{BatchData, BLOB_WIDTH, N_BLOB_BYTES, N_DATA_BYTES_PER_COEFFICIENT},
+    aggregation::rlc::POWS_OF_256,
+    blob::{BLOB_WIDTH, N_BLOB_BYTES, N_DATA_BYTES_PER_COEFFICIENT},
     RlcConfig,
 };
 
@@ -30,7 +28,7 @@ use crate::{
 #[derive(Clone, Debug)]
 pub struct BlobDataConfig<const N_SNARKS: usize> {
     /// Selector to mark the first row in the layout, enabled at offset=0.
-    q_first: Selector,
+    q_first: Column<Fixed>,
     /// Whether the row is enabled or not. We need exactly N_BLOB_BYTES rows, enabled from offset=1
     /// to offset=N_BLOB_BYTES.
     q_enabled: Selector,
@@ -47,8 +45,11 @@ pub struct BlobDataConfig<const N_SNARKS: usize> {
 }
 
 pub struct AssignedBlobDataExport {
+    pub enable_encoding_bool: bool,
+    pub enable_encoding: AssignedCell<Fr, Fr>,
     pub bytes_rlc: AssignedCell<Fr, Fr>,
     pub bytes_len: AssignedCell<Fr, Fr>,
+    pub cooked_len: AssignedCell<Fr, Fr>,
 }
 
 impl<const N_SNARKS: usize> BlobDataConfig<N_SNARKS> {
@@ -58,8 +59,8 @@ impl<const N_SNARKS: usize> BlobDataConfig<N_SNARKS> {
         u8_table: U8Table,
     ) -> Self {
         let config = Self {
+            q_first: meta.fixed_column(),
             q_enabled: meta.selector(),
-            q_first: meta.complex_selector(),
             byte: meta.advice_column(),
             is_padding: meta.advice_column(),
             bytes_rlc: meta.advice_column_in(SecondPhase),
@@ -76,23 +77,30 @@ impl<const N_SNARKS: usize> BlobDataConfig<N_SNARKS> {
         });
 
         meta.create_gate("BlobDataConfig: first row", |meta| {
-            let is_first = meta.query_selector(config.q_first);
+            let is_first = meta.query_fixed(config.q_first, Rotation::cur());
 
             let byte = meta.query_advice(config.byte, Rotation::cur());
             let bytes_rlc = meta.query_advice(config.bytes_rlc, Rotation::cur());
             let bytes_len = meta.query_advice(config.bytes_len, Rotation::cur());
             let is_padding_next = meta.query_advice(config.is_padding, Rotation::next());
 
+            let bytes_rlc_next = meta.query_advice(config.bytes_rlc, Rotation::next());
+            let bytes_len_next = meta.query_advice(config.bytes_len, Rotation::next());
+
             vec![
                 is_first.expr() * byte,
                 is_first.expr() * bytes_rlc,
+                is_first.expr() * bytes_rlc_next,
                 is_first.expr() * bytes_len,
+                is_first.expr() * bytes_len_next,
                 is_first.expr() * is_padding_next,
             ]
         });
 
         meta.create_gate("BlobDataConfig: main gate", |meta| {
             let is_enabled = meta.query_selector(config.q_enabled);
+            let is_skip_rlc = meta.query_fixed(config.q_first, Rotation::prev());
+            let trigger_rlc = 1.expr() - is_skip_rlc;
 
             let is_padding_curr = meta.query_advice(config.is_padding, Rotation::cur());
             let is_padding_prev = meta.query_advice(config.is_padding, Rotation::prev());
@@ -116,6 +124,7 @@ impl<const N_SNARKS: usize> BlobDataConfig<N_SNARKS> {
                 // bytes_rlc updates in the non-padded territory
                 is_enabled.expr()
                     * (1.expr() - is_padding_curr.expr())
+                    * trigger_rlc.expr()
                     * (bytes_rlc_prev.expr() * challenges.keccak_input() + byte.expr()
                         - bytes_rlc_curr.expr()),
                 // bytes_rlc remains unchanged in padded territory
@@ -125,6 +134,7 @@ impl<const N_SNARKS: usize> BlobDataConfig<N_SNARKS> {
                 // bytes_len increments in the non-padded territory
                 is_enabled.expr()
                     * (1.expr() - is_padding_curr.expr())
+                    * trigger_rlc.expr()
                     * (bytes_len_prev.expr() + 1.expr() - bytes_len_curr.expr()),
                 // bytes_len remains unchanged in padded territory
                 is_enabled.expr()
@@ -143,15 +153,16 @@ impl<const N_SNARKS: usize> BlobDataConfig<N_SNARKS> {
         layouter: &mut impl Layouter<Fr>,
         challenge_value: Challenges<Value<Fr>>,
         rlc_config: &RlcConfig,
-        batch_data: &BatchData<N_SNARKS>,
+        blob_bytes: &[u8],
         barycentric_assignments: &[CRTInteger<Fr>],
     ) -> Result<AssignedBlobDataExport, Error> {
-        let (assigned_bytes, bytes_rlc, bytes_len) = layouter.assign_region(
+        let (assigned_bytes, bytes_rlc, bytes_len, enable_encoding_bool) = layouter.assign_region(
             || "BlobData bytes",
-            |mut region| self.assign_rows(&mut region, batch_data, &challenge_value),
+            |mut region| self.assign_rows(&mut region, blob_bytes, &challenge_value),
         )?;
+        let enable_encoding = assigned_bytes[0].clone();
 
-        let cooked_bytes_len = layouter.assign_region(
+        let cooked_len = layouter.assign_region(
             || "BlobData internal checks",
             |mut region| {
                 self.assign_internal_checks(
@@ -165,41 +176,38 @@ impl<const N_SNARKS: usize> BlobDataConfig<N_SNARKS> {
         )?;
 
         Ok(AssignedBlobDataExport {
+            enable_encoding_bool,
+            enable_encoding,
             bytes_rlc,
-            bytes_len: cooked_bytes_len,
+            bytes_len,
+            cooked_len,
         })
     }
 
     #[allow(clippy::type_complexity)]
     pub fn assign_rows(
         &self,
         region: &mut Region<Fr>,
-        batch_data: &BatchData<N_SNARKS>,
+        blob_bytes: &[u8],
         challenges: &Challenges<Value<Fr>>,
     ) -> Result<
         (
             Vec<AssignedCell<Fr, Fr>>,
             AssignedCell<Fr, Fr>,
             AssignedCell<Fr, Fr>,
+            bool,
         ),
         Error,
     > {
-        let batch_bytes = batch_data.get_batch_data_bytes();
-        let blob_bytes = {
-            let mut encoder = init_zstd_encoder(None);
-            encoder
-                .set_pledged_src_size(Some(batch_bytes.len() as u64))
-                .map_err(|_| Error::Synthesis)?;
-            encoder
-                .write_all(&batch_bytes)
-                .map_err(|_| Error::Synthesis)?;
-            encoder.finish().map_err(|_| Error::Synthesis)?
-        };
+        let enable_encoding = blob_bytes[0].eq(&1);
+
         assert!(blob_bytes.len() <= N_BLOB_BYTES, "too many blob bytes");
 
-        self.q_first.enable(region, 0)?;
+        // Assign fixed column and selector.
+        region.assign_fixed(|| "q_first", self.q_first, 0, || Value::known(Fr::one()))?;
         for i in 1..=N_BLOB_BYTES {
             self.q_enabled.enable(region, i)?;
+            region.assign_fixed(|| "q_first", self.q_first, i, || Value::known(Fr::zero()))?;
         }
 
         for col in [self.byte, self.bytes_rlc, self.bytes_len, self.is_padding] {
@@ -217,60 +225,69 @@ impl<const N_SNARKS: usize> BlobDataConfig<N_SNARKS> {
         let mut last_bytes_len = None;
         for (i, &byte) in blob_bytes.iter().enumerate() {
             let byte_value = Value::known(Fr::from(byte as u64));
-            bytes_rlc = bytes_rlc * challenges.keccak_input() + byte_value;
+            if i > 0 {
+                bytes_rlc = bytes_rlc * challenges.keccak_input() + byte_value;
+            }
 
+            let offset = i + 1;
             assigned_bytes.push(region.assign_advice(
                 || "byte",
                 self.byte,
-                i + 1,
+                offset,
                 || byte_value,
             )?);
             region.assign_advice(
                 || "is_padding",
                 self.is_padding,
-                i + 1,
+                offset,
                 || Value::known(Fr::zero()),
             )?;
             last_bytes_rlc =
-                Some(region.assign_advice(|| "bytes_rlc", self.bytes_rlc, i + 1, || bytes_rlc)?);
+                Some(region.assign_advice(|| "bytes_rlc", self.bytes_rlc, offset, || bytes_rlc)?);
             last_bytes_len = Some(region.assign_advice(
                 || "bytes_len",
                 self.bytes_len,
-                i + 1,
-                || Value::known(Fr::from(i as u64 + 1)),
+                offset,
+                || Value::known(Fr::from(i as u64)),
             )?);
         }
 
         let mut last_bytes_rlc = last_bytes_rlc.expect("at least 1 byte guaranteed");
         let mut last_bytes_len = last_bytes_len.expect("at least 1 byte guaranteed");
         for i in blob_bytes.len()..N_BLOB_BYTES {
+            let offset = i + 1;
             assigned_bytes.push(region.assign_advice(
                 || "byte",
                 self.byte,
-                i + 1,
+                offset,
                 || Value::known(Fr::zero()),
             )?);
             region.assign_advice(
                 || "is_padding",
                 self.is_padding,
-                i + 1,
+                offset,
                 || Value::known(Fr::one()),
             )?;
             last_bytes_rlc = region.assign_advice(
                 || "bytes_rlc",
                 self.bytes_rlc,
-                i + 1,
+                offset,
                 || last_bytes_rlc.value().cloned(),
             )?;
             last_bytes_len = region.assign_advice(
                 || "bytes_len",
                 self.bytes_len,
-                i + 1,
+                offset,
                 || last_bytes_len.value().cloned(),
             )?;
         }
 
-        Ok((assigned_bytes, last_bytes_rlc, last_bytes_len))
+        Ok((
+            assigned_bytes,
+            last_bytes_rlc,
+            last_bytes_len,
+            enable_encoding,
+        ))
     }
 
     pub fn assign_internal_checks(
@@ -307,6 +324,10 @@ impl<const N_SNARKS: usize> BlobDataConfig<N_SNARKS> {
             pows_of_256
         };
 
+        // The first byte in the blob is a boolean indicating whether or not blob is an encoded
+        // form of the batch.
+        rlc_config.enforce_binary(region, &assigned_bytes[0], &mut rlc_config_offset)?;
+
         ////////////////////////////////////////////////////////////////////////////////
         //////////////////////////////////// LINKING ///////////////////////////////////
         ////////////////////////////////////////////////////////////////////////////////