Skip to content

Commit

Permalink
feat: hash-based JSON circuit (#41)
Browse files Browse the repository at this point in the history
* feat: hash based JSON verification

* WIP: save

* resetting for clearer approach

* good save state

* feat: working hash version

Though this will be too expensive, the idea works!

* WIP: need to clear after comma

* WIP: good progress

* WIP: getting keys also now

* feat: (mostly?) working tree hasher

* seems to be correct for spotify

* perf: first optimization

* wip: brain hurty

left a note to myself

* fix: tree hasher seems correct now

* TODO: note to self

* feat: hash based JSON verification

* WIP: save

* resetting for clearer approach

* good save state

* feat: working hash version

Though this will be too expensive, the idea works!

* WIP: need to clear after comma

* WIP: good progress

* WIP: getting keys also now

* feat: (mostly?) working tree hasher

* seems to be correct for spotify

* perf: first optimization

* wip: brain hurty

left a note to myself

* fix: tree hasher seems correct now

* TODO: note to self

* cleanup from rebase

* cleanup

* WIP: seems to monomial correctly

* rename

* add in value to eval at

* WIP: start looking for matches

* made some fixes

* it may be working!

* now i can write tests!

* more tests

* more JSON hasher tests

* cleanup
  • Loading branch information
Autoparallel authored Dec 11, 2024
1 parent da99172 commit 06c6920
Show file tree
Hide file tree
Showing 18 changed files with 793 additions and 81 deletions.
23 changes: 3 additions & 20 deletions circuits.json
Original file line number Diff line number Diff line change
Expand Up @@ -14,29 +14,12 @@
25
]
},
"json_mask_object_1024b": {
"file": "json/nivc/masker",
"template": "JsonMaskObjectNIVC",
"json_extraction_1024b": {
"file": "json/parser/hash_parser",
"template": "ParserHasher",
"params": [
1024,
10,
10
]
},
"json_mask_array_index_1024b": {
"file": "json/nivc/masker",
"template": "JsonMaskArrayIndexNIVC",
"params": [
1024,
10
]
},
"json_extract_value_1024b": {
"file": "json/nivc/extractor",
"template": "MaskExtractFinal",
"params": [
1024,
50
]
}
}
431 changes: 431 additions & 0 deletions circuits/json/parser/hash_machine.circom

Large diffs are not rendered by default.

95 changes: 95 additions & 0 deletions circuits/json/parser/hash_parser.circom
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
pragma circom 2.1.9;

include "../../utils/bits.circom";
include "hash_machine.circom";

template ParserHasher(DATA_BYTES, MAX_STACK_HEIGHT) {
signal input data[DATA_BYTES];
signal input polynomial_input;
signal input sequence_digest;

//--------------------------------------------------------------------------------------------//
// Initialze the parser
component State[DATA_BYTES];
State[0] = StateUpdateHasher(MAX_STACK_HEIGHT);
for(var i = 0; i < MAX_STACK_HEIGHT; i++) {
State[0].stack[i] <== [0,0];
State[0].tree_hash[i] <== [0,0];
}
State[0].byte <== data[0];
State[0].polynomial_input <== polynomial_input;
State[0].monomial <== 0;
State[0].parsing_string <== 0;
State[0].parsing_number <== 0;

// Set up monomials for stack/tree digesting
signal monomials[4 * MAX_STACK_HEIGHT];
monomials[0] <== 1;
for(var i = 1 ; i < 4 * MAX_STACK_HEIGHT ; i++) {
monomials[i] <== monomials[i - 1] * polynomial_input;
}
signal intermediate_digest[DATA_BYTES][4 * MAX_STACK_HEIGHT];
signal state_digest[DATA_BYTES];

// Debugging
// for(var i = 0; i<MAX_STACK_HEIGHT; i++) {
// log("State[", 0, "].next_stack[", i,"] = [",State[0].next_stack[i][0], "][", State[0].next_stack[i][1],"]" );
// }
// for(var i = 0; i<MAX_STACK_HEIGHT; i++) {
// log("State[", 0, "].next_tree_hash[", i,"] = [",State[0].next_tree_hash[i][0], "][", State[0].next_tree_hash[i][1],"]" );
// }
// log("State[", 0, "].next_monomial =", State[0].next_monomial);
// log("State[", 0, "].next_parsing_string =", State[0].next_parsing_string);
// log("State[", 0, "].next_parsing_number =", State[0].next_parsing_number);
// log("xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx");

var total_matches = 0;
signal is_matched[DATA_BYTES];
for(var data_idx = 1; data_idx < DATA_BYTES; data_idx++) {
State[data_idx] = StateUpdateHasher(MAX_STACK_HEIGHT);
State[data_idx].byte <== data[data_idx];
State[data_idx].polynomial_input <== polynomial_input;
State[data_idx].stack <== State[data_idx - 1].next_stack;
State[data_idx].parsing_string <== State[data_idx - 1].next_parsing_string;
State[data_idx].parsing_number <== State[data_idx - 1].next_parsing_number;
State[data_idx].monomial <== State[data_idx - 1].next_monomial;
State[data_idx].tree_hash <== State[data_idx - 1].next_tree_hash;

// Digest the whole stack and tree hash
var accumulator = 0;
for(var i = 0 ; i < MAX_STACK_HEIGHT ; i++) {
intermediate_digest[data_idx][4 * i] <== State[data_idx].next_stack[i][0] * monomials[4 * i];
intermediate_digest[data_idx][4 * i + 1] <== State[data_idx].next_stack[i][1] * monomials[4 * i + 1];
intermediate_digest[data_idx][4 * i + 2] <== State[data_idx].next_tree_hash[i][0] * monomials[4 * i + 2];
intermediate_digest[data_idx][4 * i + 3] <== State[data_idx].next_tree_hash[i][1] * monomials[4 * i + 3];
accumulator += intermediate_digest[data_idx][4 * i] + intermediate_digest[data_idx][4 * i + 1] + intermediate_digest[data_idx][4 * i + 2] + intermediate_digest[data_idx][4 * i + 3];
}
state_digest[data_idx] <== accumulator;
is_matched[data_idx] <== IsEqual()([state_digest[data_idx], sequence_digest]);
total_matches += is_matched[data_idx];

// Debugging
// for(var i = 0; i<MAX_STACK_HEIGHT; i++) {
// log("State[", data_idx, "].next_stack[", i,"] = [",State[data_idx].next_stack[i][0], "][", State[data_idx].next_stack[i][1],"]" );
// }
// for(var i = 0; i<MAX_STACK_HEIGHT; i++) {
// log("State[", data_idx, "].next_tree_hash[", i,"] = [",State[data_idx].next_tree_hash[i][0], "][", State[data_idx].next_tree_hash[i][1],"]" );
// }
// log("State[", data_idx, "].next_monomial =", State[data_idx].next_monomial);
// log("State[", data_idx, "].next_parsing_string =", State[data_idx].next_parsing_string);
// log("State[", data_idx, "].next_parsing_number =", State[data_idx].next_parsing_number);
// log("++++++++++++++++++++++++++++++++++++++++++++++++");
// log("state_digest[", data_idx,"] = ", state_digest[data_idx]);
// log("total_matches = ", total_matches);
// log("xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx");
}

// TODO: Assert something about total matches but keep in mind we should try to output the target value hash
total_matches === 1;

// Constrain to have valid JSON
for(var i = 0; i < MAX_STACK_HEIGHT; i++) {
State[DATA_BYTES - 1].next_stack[i] === [0,0];
State[DATA_BYTES - 1].next_tree_hash[i] === [0,0];
}
}
99 changes: 97 additions & 2 deletions circuits/test/common/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ export function generateDescription(input: any): string {
}

export function readJSONInputFile(filename: string, key: any[]): [number[], number[][], number[]] {
const valueStringPath = join(__dirname, "..", "..", "..", "examples", "json", "test", filename);
const valueStringPath = join(__dirname, "..", "..", "..", "examples", "json", filename);

let input: number[] = [];
let output: number[] = [];
Expand Down Expand Up @@ -327,4 +327,99 @@ export const http_body = [
34, 84, 97, 121, 108, 111, 114, 32, 83, 119, 105, 102, 116, 34, 13, 10, 32, 32, 32, 32, 32, 32,
32, 32, 32, 32, 32, 32, 32, 32, 32, 125, 13, 10, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 125,
13, 10, 32, 32, 32, 32, 32, 32, 32, 93, 13, 10, 32, 32, 32, 125, 13, 10, 125,
];
];

export function strToBytes(str: string): number[] {
return Array.from(str.split('').map(c => c.charCodeAt(0)));
}

// Enum equivalent for JsonMaskType
export type JsonMaskType =
| { type: "Object", value: number[] } // Changed from Uint8Array to number[]
| { type: "ArrayIndex", value: number };

// Constants for the field arithmetic
const PRIME = BigInt("21888242871839275222246405745257275088548364400416034343698204186575808495617");
const ONE = BigInt(1);
const ZERO = BigInt(0);

function modAdd(a: bigint, b: bigint): bigint {
return (a + b) % PRIME;
}

function modMul(a: bigint, b: bigint): bigint {
return (a * b) % PRIME;
}

export function jsonTreeHasher(
polynomialInput: bigint,
keySequence: JsonMaskType[],
targetValue: number[], // Changed from Uint8Array to number[]
maxStackHeight: number
): [Array<[bigint, bigint]>, Array<[bigint, bigint]>] {
if (keySequence.length >= maxStackHeight) {
throw new Error("Key sequence length exceeds max stack height");
}

const stack: Array<[bigint, bigint]> = [];
const treeHashes: Array<[bigint, bigint]> = [];

for (const valType of keySequence) {
if (valType.type === "Object") {
stack.push([ONE, ONE]);
let stringHash = ZERO;
let monomial = ONE;

for (const byte of valType.value) {
stringHash = modAdd(stringHash, modMul(monomial, BigInt(byte)));
monomial = modMul(monomial, polynomialInput);
}
treeHashes.push([stringHash, ZERO]);
} else { // ArrayIndex
treeHashes.push([ZERO, ZERO]);
stack.push([BigInt(2), BigInt(valType.value)]);
}
}

let targetValueHash = ZERO;
let monomial = ONE;

for (const byte of targetValue) {
targetValueHash = modAdd(targetValueHash, modMul(monomial, BigInt(byte)));
monomial = modMul(monomial, polynomialInput);
}

treeHashes[keySequence.length - 1] = [treeHashes[keySequence.length - 1][0], targetValueHash];

return [stack, treeHashes];
}

export function compressTreeHash(
polynomialInput: bigint,
stackAndTreeHashes: [Array<[bigint, bigint]>, Array<[bigint, bigint]>]
): bigint {
const [stack, treeHashes] = stackAndTreeHashes;

if (stack.length !== treeHashes.length) {
throw new Error("Stack and tree hashes must have the same length");
}

let accumulated = ZERO;
let monomial = ONE;

for (let idx = 0; idx < stack.length; idx++) {
accumulated = modAdd(accumulated, modMul(stack[idx][0], monomial));
monomial = modMul(monomial, polynomialInput);

accumulated = modAdd(accumulated, modMul(stack[idx][1], monomial));
monomial = modMul(monomial, polynomialInput);

accumulated = modAdd(accumulated, modMul(treeHashes[idx][0], monomial));
monomial = modMul(monomial, polynomialInput);

accumulated = modAdd(accumulated, modMul(treeHashes[idx][1], monomial));
monomial = modMul(monomial, polynomialInput);
}

return accumulated;
}
Loading

0 comments on commit 06c6920

Please sign in to comment.