diff --git a/barretenberg/acir_tests/bbjs-test/src/index.ts b/barretenberg/acir_tests/bbjs-test/src/index.ts index 2bc07d7fcff6..64dbad6029d0 100644 --- a/barretenberg/acir_tests/bbjs-test/src/index.ts +++ b/barretenberg/acir_tests/bbjs-test/src/index.ts @@ -9,9 +9,7 @@ const logger = pino({ }); const proofPath = (dir: string) => path.join(dir, "proof"); -const proofAsFieldsPath = (dir: string) => path.join(dir, "proof_fields.json"); -const publicInputsAsFieldsPath = (dir: string) => - path.join(dir, "public_inputs_fields.json"); +const publicInputsPath = (dir: string) => path.join(dir, "public_inputs"); const vkeyPath = (dir: string) => path.join(dir, "vk"); async function generateProof({ @@ -27,7 +25,7 @@ async function generateProof({ oracleHash?: string; multiThreaded?: boolean; }) { - const { UltraHonkBackend, deflattenFields } = await import("@aztec/bb.js"); + const { UltraHonkBackend } = await import("@aztec/bb.js"); logger.debug(`Generating proof for ${bytecodePath}...`); const circuitArtifact = await fs.readFile(bytecodePath); @@ -45,17 +43,16 @@ async function generateProof({ await fs.writeFile(proofPath(outputDirectory), Buffer.from(proof.proof)); logger.debug("Proof written to " + proofPath(outputDirectory)); - await fs.writeFile( - publicInputsAsFieldsPath(outputDirectory), - JSON.stringify(proof.publicInputs) + // Convert public inputs from field strings to binary + const publicInputsBuffer = Buffer.concat( + proof.publicInputs.map((field: string) => { + const hex = field.startsWith('0x') ? field.slice(2) : field; + return Buffer.from(hex.padStart(64, '0'), 'hex'); + }) ); + await fs.writeFile(publicInputsPath(outputDirectory), publicInputsBuffer); logger.debug( - "Public inputs written to " + publicInputsAsFieldsPath(outputDirectory) - ); - - await fs.writeFile( - proofAsFieldsPath(outputDirectory), - JSON.stringify(deflattenFields(proof.proof)) + "Public inputs written to " + publicInputsPath(outputDirectory) ); const verificationKey = await backend.getVerificationKey({ @@ -69,21 +66,24 @@ async function generateProof({ } async function verifyProof({ directory }: { directory: string }) { - const { BarretenbergVerifier } = await import("@aztec/bb.js"); + const { UltraHonkVerifierBackend } = await import("@aztec/bb.js"); - const verifier = new BarretenbergVerifier(); + const verifier = new UltraHonkVerifierBackend(); const proof = await fs.readFile(proofPath(directory)); - const publicInputs = JSON.parse( - await fs.readFile(publicInputsAsFieldsPath(directory), "utf8") - ); + // Read binary public inputs and convert to field strings + const publicInputsBinary = await fs.readFile(publicInputsPath(directory)); + const publicInputs = []; + for (let i = 0; i < publicInputsBinary.length; i += 32) { + const chunk = publicInputsBinary.slice(i, Math.min(i + 32, publicInputsBinary.length)); + publicInputs.push('0x' + chunk.toString('hex')); + } logger.debug(`publicInputs: ${JSON.stringify(publicInputs)}`); - const vkey = await fs.readFile(vkeyPath(directory)); + const verificationKey = await fs.readFile(vkeyPath(directory)); - const verified = await verifier.verifyUltraHonkProof( - { proof: new Uint8Array(proof), publicInputs }, - new Uint8Array(vkey) + const verified = await verifier.verifyProof( + { proof: new Uint8Array(proof), publicInputs, verificationKey}, ); await verifier.destroy(); diff --git a/barretenberg/acir_tests/bootstrap.sh b/barretenberg/acir_tests/bootstrap.sh index 6b60ab44d6c5..9ea344e84195 100755 --- a/barretenberg/acir_tests/bootstrap.sh +++ b/barretenberg/acir_tests/bootstrap.sh @@ -21,6 +21,11 @@ tests_hash=$(hash_str \ ../ts/.rebuild_patterns \ ../noir/)) +function hex_to_fields_json { + # 1. split encoded hex into 64-character lines 3. encode as JSON array of hex strings + fold -w64 | jq -R -s -c 'split("\n") | map(select(length > 0)) | map("0x" + .)' +} + # Generate inputs for a given recursively verifying program. function run_proof_generation { local program=$1 @@ -28,7 +33,6 @@ function run_proof_generation { local bb=$(realpath ../cpp/$native_build_dir/bin/bb) local outdir=$(mktemp -d) trap "rm -rf $outdir" EXIT - local adjustment=16 local ipa_accumulation_flag="" cd ./acir_tests/assert_statement @@ -37,35 +41,37 @@ function run_proof_generation { # Adjust settings based on program type if [[ $program == *"rollup"* ]]; then - adjustment=26 ipa_accumulation_flag="--ipa_accumulation" fi # If the test program has zk in it's name would like to use the zk prover, so we empty the flag in this case. if [[ $program == *"zk"* ]]; then disable_zk="" fi - local prove_cmd="$bb prove --scheme ultra_honk $disable_zk $ipa_accumulation_flag --output_format fields --write_vk -o $outdir -b ./target/program.json -w ./target/witness.gz" + local prove_cmd="$bb prove --scheme ultra_honk $disable_zk $ipa_accumulation_flag --write_vk -o $outdir -b ./target/program.json -w ./target/witness.gz" echo_stderr "$prove_cmd" dump_fail "$prove_cmd" - local vk_fields=$(cat "$outdir/vk_fields.json") - local vk_hash_fields=$(cat "$outdir/vk_hash_fields.json") - local public_inputs_fields=$(cat "$outdir/public_inputs_fields.json") - local proof_fields=$(cat "$outdir/proof_fields.json") - generate_toml "$program" "$vk_fields" "$vk_hash_fields" "$proof_fields" "$public_inputs_fields" + # Split the hex-encoded vk bytes into fields boundaries (but still hex-encoded), first making 64-character lines and then encoding as JSON. + # This used to be done by barretenberg itself, but with serialization now always being in field elements we can do it outside of bb. + local vk_fields=$(cat "$outdir/vk" | xxd -p -c 0 | hex_to_fields_json) + local vk_hash_field="\"0x$(cat "$outdir/vk_hash" | xxd -p -c 0)\"" + local public_inputs_fields=$(cat "$outdir/public_inputs" | xxd -p -c 0 | hex_to_fields_json) + local proof_fields=$(cat "$outdir/proof" | xxd -p -c 0 | hex_to_fields_json) + + generate_toml "$program" "$vk_fields" "$vk_hash_field" "$proof_fields" "$public_inputs_fields" } function generate_toml { local program=$1 local vk_fields=$2 - local vk_hash_fields=$3 + local vk_hash_field=$3 local proof_fields=$4 - local num_inner_public_inputs=$5 + local public_inputs_fields=$5 local output_file="../$program/Prover.toml" jq -nr \ - --arg key_hash "$vk_hash_fields" \ + --arg key_hash "$vk_hash_field" \ --argjson vk_f "$vk_fields" \ --argjson public_inputs_f "$public_inputs_fields" \ --argjson proof_f "$proof_fields" \ @@ -79,7 +85,6 @@ function generate_toml { } function regenerate_recursive_inputs { - local program=$1 # Compile the assert_statement test as it's used for the recursive tests. cd ./acir_tests/assert_statement local nargo=$(realpath ../../../../noir/noir-repo/target/release/nargo) @@ -91,7 +96,7 @@ function regenerate_recursive_inputs { parallel 'run_proof_generation {}' ::: $(ls internal_test_programs) } -export -f regenerate_recursive_inputs run_proof_generation generate_toml +export -f hex_to_fields_json regenerate_recursive_inputs run_proof_generation generate_toml function compile { echo_header "Compiling acir_tests" @@ -160,10 +165,10 @@ function test_cmds { # bb.js browser tests. Isolate because server. local browser_prefix="$tests_hash:ISOLATE=1:NET=1:CPUS=8" - echo "$browser_prefix:NAME=chrome_verify_honk_proof $scripts/browser_prove.sh verify_honk_proof chrome" - echo "$browser_prefix:NAME=chrome_a_1_mul $scripts/browser_prove.sh a_1_mul chrome" - echo "$browser_prefix:NAME=webkit_verify_honk_proof $scripts/browser_prove.sh verify_honk_proof webkit" - echo "$browser_prefix:NAME=webkit_a_1_mul $scripts/browser_prove.sh a_1_mul webkit" + echo "$browser_prefix $scripts/browser_prove.sh verify_honk_proof chrome" + echo "$browser_prefix $scripts/browser_prove.sh a_1_mul chrome" + echo "$browser_prefix $scripts/browser_prove.sh verify_honk_proof webkit" + echo "$browser_prefix $scripts/browser_prove.sh a_1_mul webkit" # bb.js tests. # ecdsa_secp256r1_3x through bb.js on node to check 256k support. diff --git a/barretenberg/acir_tests/browser-test-app/src/index.ts b/barretenberg/acir_tests/browser-test-app/src/index.ts index d65061b85519..4fa385813707 100644 --- a/barretenberg/acir_tests/browser-test-app/src/index.ts +++ b/barretenberg/acir_tests/browser-test-app/src/index.ts @@ -30,17 +30,16 @@ function installUltraHonkGlobals() { } async function verify(proofData: ProofData, verificationKey: Uint8Array) { - const { BarretenbergVerifier } = await import("@aztec/bb.js"); + const { UltraHonkVerifierBackend } = await import("@aztec/bb.js"); logger.debug(`verifying...`); - const verifier = new BarretenbergVerifier(); - const verified = await verifier.verifyUltraHonkProof( - proofData, - verificationKey + const backend = new UltraHonkVerifierBackend(); + const verified = await backend.verifyProof( + {...proofData, verificationKey} ); logger.debug(`verified: ${verified}`); - await verifier.destroy(); + await backend.destroy(); logger.debug("test complete."); return verified; diff --git a/barretenberg/acir_tests/scripts/bb_prove_bbjs_verify.sh b/barretenberg/acir_tests/scripts/bb_prove_bbjs_verify.sh index c29b45c212cf..9b8b1eb2139d 100755 --- a/barretenberg/acir_tests/scripts/bb_prove_bbjs_verify.sh +++ b/barretenberg/acir_tests/scripts/bb_prove_bbjs_verify.sh @@ -22,7 +22,6 @@ $bb prove \ -b target/program.json \ -w target/witness.gz \ -k output-$$/vk \ - --output_format bytes_and_fields \ -o output-$$ # Verify the proof with bb.js classes diff --git a/barretenberg/acir_tests/scripts/bb_prove_sol_verify.sh b/barretenberg/acir_tests/scripts/bb_prove_sol_verify.sh index 45cd3943439e..6d68dc43ccfe 100755 --- a/barretenberg/acir_tests/scripts/bb_prove_sol_verify.sh +++ b/barretenberg/acir_tests/scripts/bb_prove_sol_verify.sh @@ -34,7 +34,7 @@ mkdir -p output-$$ trap "rm -rf output-$$" EXIT # Create a proof, write the solidity contract, write the proof as fields in order to extract the public inputs -$bb prove $flags -b target/program.json --oracle_hash keccak --output_format bytes_and_fields --write_vk -o output-$$ +$bb prove $flags -b target/program.json --oracle_hash keccak --write_vk -o output-$$ $bb verify $flags --oracle_hash keccak -i output-$$/public_inputs -k output-$$/vk -p output-$$/proof $bb write_solidity_verifier $write_contract_flags -k output-$$/vk -o output-$$/Verifier.sol @@ -42,8 +42,7 @@ $bb write_solidity_verifier $write_contract_flags -k output-$$/vk -o output-$$/V # index.js will start an anvil, on a random port # Deploy the verifier then send a test transaction PROOF="output-$$/proof" \ -PROOF_AS_FIELDS="output-$$/proof_fields.json" \ -PUBLIC_INPUTS_AS_FIELDS="output-$$/public_inputs_fields.json" \ +PUBLIC_INPUTS="output-$$/public_inputs" \ VERIFIER_PATH="output-$$/Verifier.sol" \ TEST_PATH="../../sol-test/HonkTest.sol" \ HAS_ZK="$has_zk" \ diff --git a/barretenberg/acir_tests/scripts/bbjs_prove_bb_verify.sh b/barretenberg/acir_tests/scripts/bbjs_prove_bb_verify.sh index f5cdb196f6e3..b809adf477dc 100755 --- a/barretenberg/acir_tests/scripts/bbjs_prove_bb_verify.sh +++ b/barretenberg/acir_tests/scripts/bbjs_prove_bb_verify.sh @@ -14,17 +14,7 @@ node ../../bbjs-test prove \ -w target/witness.gz \ -o output-$$ -proof_bytes=$(cat output-$$/proof | xxd -p) -public_inputs=$(cat output-$$/public_inputs_fields.json | jq -r '.[]') - -public_inputs_bytes="" -for input in $public_inputs; do - public_inputs_bytes+=$input -done - -# Combine proof header and the proof to a single file -echo -n $proof_bytes | xxd -r -p > output-$$/proof -echo -n $public_inputs_bytes | xxd -r -p > output-$$/public_inputs +# The proof and public_inputs are already in binary format from bbjs-test bb=$(../../../cpp/scripts/find-bb) # Verify the proof with bb cli diff --git a/barretenberg/acir_tests/scripts/bbjs_prove_sol_verify.sh b/barretenberg/acir_tests/scripts/bbjs_prove_sol_verify.sh index 180a5ab4518a..1574b00cbb97 100755 --- a/barretenberg/acir_tests/scripts/bbjs_prove_sol_verify.sh +++ b/barretenberg/acir_tests/scripts/bbjs_prove_sol_verify.sh @@ -35,8 +35,7 @@ $bb write_solidity_verifier --scheme ultra_honk -k output-$$/vk -o output-$$/Ver # Verify the proof using the solidity verifier PROOF="output-$$/proof" \ -PROOF_AS_FIELDS="output-$$/proof_fields.json" \ -PUBLIC_INPUTS_AS_FIELDS="output-$$/public_inputs_fields.json" \ +PUBLIC_INPUTS="output-$$/public_inputs" \ VERIFIER_PATH="output-$$/Verifier.sol" \ TEST_PATH="../../sol-test/HonkTest.sol" \ HAS_ZK="$has_zk" \ diff --git a/barretenberg/acir_tests/sol-test/src/index.js b/barretenberg/acir_tests/sol-test/src/index.js index 1da35911dcbb..6ee750c84543 100644 --- a/barretenberg/acir_tests/sol-test/src/index.js +++ b/barretenberg/acir_tests/sol-test/src/index.js @@ -165,6 +165,20 @@ const linkLibrary = (bytecode, libraryName, libraryAddress) => { return bytecode.replace(regex, address); }; +/** + * Converts binary data to array of field elements (32-byte chunks as hex strings) + * @param {Buffer} buffer - Binary data + * @return {Array} Array of hex strings with 0x prefix + */ +const binaryToFields = (buffer) => { + const fields = []; + for (let i = 0; i < buffer.length; i += 32) { + const chunk = buffer.slice(i, i + 32); + fields.push('0x' + chunk.toString('hex')); + } + return fields; +}; + /** * Takes in a proof as fields, and returns the public inputs, as well as the number of public inputs * @param {Array} proofAsFields @@ -225,26 +239,22 @@ try { const proof = readFileSync(proofPath); proofStr = proof.toString("hex"); - let publicInputsAsFieldsPath = getEnvVarCanBeUndefined( - "PUBLIC_INPUTS_AS_FIELDS" - ); // PUBLIC_INPUTS_AS_FIELDS is not defined for bb plonk, but is for bb honk and bbjs honk. - var publicInputs; - let proofAsFieldsPath = getEnvVarCanBeUndefined("PROOF_AS_FIELDS"); // PROOF_AS_FIELDS is not defined for bbjs, but is for bb plonk and bb honk. + let publicInputsPath = getEnvVarCanBeUndefined("PUBLIC_INPUTS"); + var publicInputs = []; let numExtraPublicInputs = 0; let extraPublicInputs = []; - if (proofAsFieldsPath) { - const proofAsFields = readFileSync(proofAsFieldsPath); + + // For flows that use binary proof format, extract public inputs from the proof + const proofAsFields = binaryToFields(proof); + if (proofAsFields.length > NUMBER_OF_FIELDS_IN_PROOF) { // We need to extract the public inputs from the proof. This might be empty, or just the pairing point object, or be the entire public inputs... - [numExtraPublicInputs, extraPublicInputs] = readPublicInputs( - JSON.parse(proofAsFields.toString()) - ); + [numExtraPublicInputs, extraPublicInputs] = readPublicInputs(proofAsFields); } - // We need to do this because plonk doesn't define this path - if (publicInputsAsFieldsPath) { - const innerPublicInputs = JSON.parse( - readFileSync(publicInputsAsFieldsPath).toString() - ); // assumes JSON array of PI hex strings + // Read public inputs from binary file if available + if (publicInputsPath) { + const publicInputsBinary = readFileSync(publicInputsPath); + const innerPublicInputs = binaryToFields(publicInputsBinary); publicInputs = innerPublicInputs.concat(extraPublicInputs); } else { // for plonk, the extraPublicInputs are all of the public inputs diff --git a/barretenberg/bootstrap.sh b/barretenberg/bootstrap.sh index 810c209dd464..ac48877abdb6 100755 --- a/barretenberg/bootstrap.sh +++ b/barretenberg/bootstrap.sh @@ -1,7 +1,6 @@ #!/usr/bin/env bash source $(git rev-parse --show-toplevel)/ci3/source - function bootstrap_all { # To run bb we need a crs. # Download ignition up front to ensure no race conditions at runtime. @@ -32,20 +31,6 @@ case "$cmd" in "release-preview") ./docs/bootstrap.sh release-preview ;; - bootstrap_e2e_hack) - echo "WARNING: This assumes your PR only changes barretenberg and the rest of the repository is unchanged from master." - echo "WARNING: This is only sound if you have not changed VK generation! (or noir-projects VKs will be incorrect)." - echo "WARNING: It builds up until yarn-project and allows end-to-end tests (not boxes/playground/release image etc)." - merge_base=$(git merge-base HEAD origin/master) - for project in noir barretenberg avm-transpiler noir-projects l1-contracts yarn-project ; do - if [ $project == barretenberg ]; then - ../$project/bootstrap.sh # i.e. this script - else - AZTEC_CACHE_COMMIT=$merge_base ../$project/bootstrap.sh - fi - done - ;; - *) echo "Unknown command: $cmd" exit 1 diff --git a/barretenberg/cpp/scripts/test_civc_standalone_vks_havent_changed.sh b/barretenberg/cpp/scripts/test_civc_standalone_vks_havent_changed.sh index 6b75b86bfbec..cf80d3b5c3e0 100755 --- a/barretenberg/cpp/scripts/test_civc_standalone_vks_havent_changed.sh +++ b/barretenberg/cpp/scripts/test_civc_standalone_vks_havent_changed.sh @@ -1,6 +1,8 @@ #!/bin/bash source $(git rev-parse --show-toplevel)/ci3/source +# export bb as it is needed when using exported functions +export bb=$(./find-bb) cd .. # NOTE: We pin the captured IVC inputs to a known master commit, exploiting that there won't be frequent changes. @@ -11,7 +13,8 @@ cd .. # - Generate a hash for versioning: sha256sum bb-civc-inputs.tar.gz # - Upload the compressed results: aws s3 cp bb-civc-inputs.tar.gz s3://aztec-ci-artifacts/protocol/bb-civc-inputs-[hash(0:8)].tar.gz # Note: In case of the "Test suite failed to run ... Unexpected token 'with' " error, need to run: docker pull aztecprotocol/build:3.0 -pinned_short_hash="e5081516" + +pinned_short_hash="9c83acbc" pinned_civc_inputs_url="https://aztec-ci-artifacts.s3.us-east-2.amazonaws.com/protocol/bb-civc-inputs-${pinned_short_hash}.tar.gz" function compress_and_upload { @@ -44,7 +47,7 @@ if [[ "${1:-}" == "--update_inputs" ]]; then # Generate new inputs echo "Running bootstrap to generate new IVC inputs..." - ../../bootstrap.sh # bootstrap aztec-packages from root + BOOTSTRAP_TO=yarn-project ../../bootstrap.sh # bootstrap aztec-packages from root ../../yarn-project/end-to-end/bootstrap.sh build_bench # build bench to generate IVC inputs compress_and_upload ../../yarn-project/end-to-end/example-app-ivc-inputs-out @@ -62,9 +65,9 @@ function check_circuit_vks { local flow_folder="$inputs_tmp_dir/$1" if [[ "${2:-}" == "--update_inputs" ]]; then - ./build/bin/bb check --update_inputs --scheme client_ivc --ivc_inputs_path "$flow_folder/ivc-inputs.msgpack" || { echo_stderr "Error: Likely VK change detected in $flow_folder! Updating inputs."; exit 1; } + $bb check --update_inputs --scheme client_ivc --ivc_inputs_path "$flow_folder/ivc-inputs.msgpack" || { echo_stderr "Error: Likely VK change detected in $flow_folder! Updating inputs."; exit 1; } else - ./build/bin/bb check --scheme client_ivc --ivc_inputs_path "$flow_folder/ivc-inputs.msgpack" || { echo_stderr "Error: Likely VK change detected in $flow_folder!"; exit 1; } + $bb check --scheme client_ivc --ivc_inputs_path "$flow_folder/ivc-inputs.msgpack" || { echo_stderr "Error: Likely VK change detected in $flow_folder!"; exit 1; } fi } diff --git a/barretenberg/cpp/src/barretenberg/api/api.hpp b/barretenberg/cpp/src/barretenberg/api/api.hpp index 6fc8f244d65f..a91537b8be6c 100644 --- a/barretenberg/cpp/src/barretenberg/api/api.hpp +++ b/barretenberg/cpp/src/barretenberg/api/api.hpp @@ -17,7 +17,6 @@ class API { bool ipa_accumulation{ false }; // indicate whether the command is doing IPA proof aggregation std::string scheme; // the proving system or IVC scheme std::string oracle_hash_type; // which hash function does the prover use as a random oracle? - std::string output_format; // output bytes, fields, both, or a msgpack buffer of fields std::string verifier_type; // is a verification key for use a single circuit verifier (e.g. a SNARK or folding // recursive verifier) or is it for an ivc verifier? bool write_vk{ false }; // should we addditionally write the verification key when writing the proof @@ -37,7 +36,6 @@ class API { << " ipa_accumulation: " << flags.ipa_accumulation << "\n" << " scheme: " << flags.scheme << "\n" << " oracle_hash_type: " << flags.oracle_hash_type << "\n" - << " output_format: " << flags.output_format << "\n" << " verifier_type: " << flags.verifier_type << "\n" << " write_vk " << flags.write_vk << "\n" << " include_gates_per_opcode " << flags.include_gates_per_opcode << "\n" diff --git a/barretenberg/cpp/src/barretenberg/api/api_client_ivc.cpp b/barretenberg/cpp/src/barretenberg/api/api_client_ivc.cpp index 68e40aaea69f..9ad55a4e5c06 100644 --- a/barretenberg/cpp/src/barretenberg/api/api_client_ivc.cpp +++ b/barretenberg/cpp/src/barretenberg/api/api_client_ivc.cpp @@ -28,63 +28,33 @@ namespace { // anonymous namespace * @param bytecode_path * @param witness_path */ -void write_standalone_vk(const std::string& output_format, - const std::filesystem::path& bytecode_path, - const std::filesystem::path& output_path) +void write_standalone_vk(const std::filesystem::path& bytecode_path, const std::filesystem::path& output_path) { auto bytecode = get_bytecode(bytecode_path); auto response = bbapi::ClientIvcComputeStandaloneVk{ .circuit = { .name = "standalone_circuit", .bytecode = std::move(bytecode) } }.execute(); - bool wrote_file = false; bool is_stdout = output_path == "-"; - auto write_fn = [&](const std::filesystem::path& path, const auto& data) { - if (is_stdout) { - write_bytes_to_stdout(data); - } else { - write_file(path, data); - } - }; - if (output_format == "bytes_and_fields" && is_stdout) { - throw_or_abort("Cannot write to stdout in bytes_and_fields format."); - } - if (output_format == "bytes" || output_format == "bytes_and_fields") { - write_fn(output_path / "vk", response.bytes); - wrote_file = true; - } - if (output_format == "fields" || output_format == "bytes_and_fields") { - std::string json = field_elements_to_json(response.fields); - write_fn(output_path / "vk_fields.json", std::vector(json.begin(), json.end())); - wrote_file = true; - } - if (!wrote_file) { - throw_or_abort("Unsupported output format for standalone vk: " + output_format); + if (is_stdout) { + write_bytes_to_stdout(response.bytes); + } else { + write_file(output_path / "vk", response.bytes); } } - -std::vector write_civc_vk(const std::string& output_format, - std::vector bytecode, - const std::filesystem::path& output_dir) +void write_civc_vk(std::vector bytecode, const std::filesystem::path& output_dir) { - if (output_format != "bytes") { - throw_or_abort("Unsupported output format for ClientIVC vk: " + output_format); - } // compute the hiding kernel's vk info("ClientIVC: computing IVC vk for hiding kernel circuit"); - auto response = bbapi::ClientIvcComputeIvcVk{ - .circuit{ .name = "standalone_circuit", .bytecode = std::move(bytecode) } - }.execute({ .trace_settings = {} }); - auto civc_vk_bytes = response.bytes; + auto response = + bbapi::ClientIvcComputeIvcVk{ .circuit{ .bytecode = std::move(bytecode) } }.execute({ .trace_settings = {} }); const bool output_to_stdout = output_dir == "-"; if (output_to_stdout) { - write_bytes_to_stdout(civc_vk_bytes); + write_bytes_to_stdout(response.bytes); } else { - write_file(output_dir / "vk", civc_vk_bytes); + write_file(output_dir / "vk", response.bytes); } - return civc_vk_bytes; } - } // anonymous namespace void ClientIVCAPI::prove(const Flags& flags, @@ -125,12 +95,10 @@ void ClientIVCAPI::prove(const Flags& flags, }; write_proof(); - if (flags.write_vk) { vinfo("writing ClientIVC vk in directory ", output_dir); - // we get the bytecode of the hiding circuit (the last step of the execution) - auto vk_buf = write_civc_vk("bytes", raw_steps[raw_steps.size() - 1].bytecode, output_dir); - auto vk = from_buffer(vk_buf); + // write CIVC vk using the bytecode of the hiding circuit (the last step of the execution) + write_civc_vk(raw_steps[raw_steps.size() - 1].bytecode, output_dir); } } @@ -210,10 +178,9 @@ void ClientIVCAPI::write_vk(const Flags& flags, { BB_BENCH_NAME("ClientIVCAPI::write_vk"); if (flags.verifier_type == "ivc") { - auto bytecode = get_bytecode(bytecode_path); - write_civc_vk(flags.output_format, bytecode, output_path); + write_civc_vk(get_bytecode(bytecode_path), output_path); } else if (flags.verifier_type == "standalone") { - write_standalone_vk(flags.output_format, bytecode_path, output_path); + write_standalone_vk(bytecode_path, output_path); } else { const std::string msg = std::string("Can't write vk for verifier type ") + flags.verifier_type; throw_or_abort(msg); diff --git a/barretenberg/cpp/src/barretenberg/api/api_client_ivc.test.cpp b/barretenberg/cpp/src/barretenberg/api/api_client_ivc.test.cpp index 912bcd24f9ff..05c8b18f12f7 100644 --- a/barretenberg/cpp/src/barretenberg/api/api_client_ivc.test.cpp +++ b/barretenberg/cpp/src/barretenberg/api/api_client_ivc.test.cpp @@ -51,8 +51,10 @@ void create_test_private_execution_steps(const std::filesystem::path& output_pat auto app_vk_response = bbapi::ClientIvcComputeStandaloneVk{ .circuit = { .name = "app_circuit", .bytecode = app_bytecode } }.execute(); - auto app_vk = app_vk_response.bytes; - auto app_vk_fields = from_buffer(app_vk).to_field_elements(); + + // Decode VK to get field elements + auto app_vk = from_buffer(app_vk_response.bytes); + auto app_vk_fields = app_vk.to_field_elements(); // Now create a kernel circuit that verifies the app circuit auto kernel_bytecode = acir_bincode_mocks::create_simple_kernel(app_vk_fields.size(), /*is_init_kernel=*/true); @@ -65,8 +67,10 @@ void create_test_private_execution_steps(const std::filesystem::path& output_pat // Create PrivateExecutionStepRaw for the kernel std::vector raw_steps; - raw_steps.push_back( - { .bytecode = app_bytecode, .witness = app_witness_data, .vk = app_vk, .function_name = "app_function" }); + raw_steps.push_back({ .bytecode = app_bytecode, + .witness = app_witness_data, + .vk = app_vk_response.bytes, + .function_name = "app_function" }); raw_steps.push_back({ .bytecode = kernel_bytecode, .witness = kernel_witness_data, .vk = kernel_vk, @@ -98,30 +102,33 @@ class ClientIVCAPITests : public ::testing::Test { namespace bb { std::vector compress(const std::vector& input); -} +} // namespace bb -// Used to get a mock IVC vk. +// Helper to get an IVC verification key for testing ClientIVC::MegaVerificationKey get_ivc_vk(const std::filesystem::path& test_dir) { auto [app_bytecode, app_witness_data] = acir_bincode_mocks::create_simple_circuit_bytecode(); bbapi::BBApiRequest request; auto app_vk_response = bbapi::ClientIvcComputeStandaloneVk{ .circuit = { .name = "app_circuit", .bytecode = app_bytecode } }.execute(); - auto app_vk = app_vk_response.bytes; - auto app_vk_fields = from_buffer(app_vk).to_field_elements(); - // Use this to get the size of the vk. - auto bytecode = acir_bincode_mocks::create_simple_kernel(app_vk_fields.size(), /*is_init_kernel=*/false); + + // Decode to get the field count + auto app_vk = from_buffer(app_vk_response.bytes); + size_t vk_field_count = app_vk.to_field_elements().size(); + + // Create a kernel circuit with the correct VK size + auto bytecode = acir_bincode_mocks::create_simple_kernel(vk_field_count, /*is_init_kernel=*/false); std::filesystem::path bytecode_path = test_dir / "circuit.acir"; write_file(bytecode_path, bb::compress(bytecode)); ClientIVCAPI::Flags write_vk_flags; write_vk_flags.verifier_type = "ivc"; - write_vk_flags.output_format = "bytes"; ClientIVCAPI api; api.write_vk(write_vk_flags, bytecode_path, test_dir); - return from_buffer(read_file(test_dir / "vk")); + auto buffer = read_file(test_dir / "vk"); + return from_buffer(buffer); }; // Test the ClientIVCAPI::prove flow, making sure --write_vk @@ -149,7 +156,7 @@ TEST_F(ClientIVCAPITests, DISABLED_ProveAndVerifyFileBasedFlow) auto verify_vk_equivalence = [&](const std::filesystem::path& vk1_path, const ClientIVC::MegaVerificationKey& vk2) { auto vk1_data = read_file(vk1_path); auto vk1 = from_buffer(vk1_data); - ASSERT_TRUE(msgpack::msgpack_check_eq(vk1, vk2, "VK from prove should match VK from write_vk")); + ASSERT_EQ(vk1, vk2); }; // Helper lambda to verify proof @@ -180,20 +187,15 @@ TEST_F(ClientIVCAPITests, WriteVkFieldsSmokeTest) std::filesystem::path bytecode_path = test_dir / "circuit.acir"; write_file(bytecode_path, bb::compress(bytecode)); - // Test write_vk with fields output format + // Test write_vk ClientIVCAPI::Flags flags; flags.verifier_type = "standalone"; - flags.output_format = "fields"; ClientIVCAPI api; api.write_vk(flags, bytecode_path, test_dir); - // Read and verify the fields format - auto vk_data = read_file(test_dir / "vk_fields.json"); - std::string vk_str(vk_data.begin(), vk_data.end()); - // Just check that this looks a bit like JSON. - EXPECT_NE(vk_str.find('['), std::string::npos); - EXPECT_NE(vk_str.find(']'), std::string::npos); + // Verify the binary VK file was created + EXPECT_TRUE(std::filesystem::exists(test_dir / "vk")); } TEST_F(ClientIVCAPITests, WriteIVCVkSmokeTest) @@ -208,7 +210,6 @@ TEST_F(ClientIVCAPITests, WriteIVCVkSmokeTest) // Set flags for VK generation ClientIVCAPI::Flags flags; flags.verifier_type = "ivc"; - flags.output_format = "bytes"; // Call write_vk ClientIVCAPI api; diff --git a/barretenberg/cpp/src/barretenberg/api/api_ultra_honk.cpp b/barretenberg/cpp/src/barretenberg/api/api_ultra_honk.cpp index 0107e52854a0..94a7df474a26 100644 --- a/barretenberg/cpp/src/barretenberg/api/api_ultra_honk.cpp +++ b/barretenberg/cpp/src/barretenberg/api/api_ultra_honk.cpp @@ -23,56 +23,26 @@ namespace bb { namespace { -void write_vk_outputs(const bbapi::CircuitComputeVk::Response& vk_response, - const std::string& output_format, - const std::filesystem::path& output_dir) +void write_vk_outputs(const bbapi::CircuitComputeVk::Response& vk_response, const std::filesystem::path& output_dir) { - if (output_format == "bytes" || output_format == "bytes_and_fields") { - write_file(output_dir / "vk", vk_response.bytes); - info("VK saved to ", output_dir / "vk"); - write_file(output_dir / "vk_hash", vk_response.hash); - info("VK Hash saved to ", output_dir / "vk_hash"); - } - - if (output_format == "fields" || output_format == "bytes_and_fields") { - // Use the fields directly from vk_response - std::string vk_json = field_elements_to_json(vk_response.fields); - write_file(output_dir / "vk_fields.json", { vk_json.begin(), vk_json.end() }); - info("VK fields saved to ", output_dir / "vk_fields.json"); - - // For vk_hash fields - convert the bytes to fr and then to JSON - auto vk_hash_fr = from_buffer(vk_response.hash); - std::string vk_hash_json = format("\"", vk_hash_fr, "\""); - write_file(output_dir / "vk_hash_fields.json", { vk_hash_json.begin(), vk_hash_json.end() }); - info("VK Hash fields saved to ", output_dir / "vk_hash_fields.json"); - } + write_file(output_dir / "vk", vk_response.bytes); + info("VK saved to ", output_dir / "vk"); + write_file(output_dir / "vk_hash", vk_response.hash); + info("VK Hash saved to ", output_dir / "vk_hash"); } -void write_proof_outputs(const bbapi::CircuitProve::Response& prove_response, - const std::string& output_format, - const std::filesystem::path& output_dir) +void write_proof_outputs(const bbapi::CircuitProve::Response& prove_response, const std::filesystem::path& output_dir) { - if (output_format == "bytes" || output_format == "bytes_and_fields") { - auto public_inputs_buf = to_buffer(prove_response.public_inputs); - auto proof_buf = to_buffer(prove_response.proof); - - write_file(output_dir / "public_inputs", public_inputs_buf); - write_file(output_dir / "proof", proof_buf); - info("Public inputs saved to ", output_dir / "public_inputs"); - info("Proof saved to ", output_dir / "proof"); - } - - if (output_format == "fields" || output_format == "bytes_and_fields") { - std::string public_inputs_json = field_elements_to_json(prove_response.public_inputs); - std::string proof_json = field_elements_to_json(prove_response.proof); + auto public_inputs_buf = to_buffer(prove_response.public_inputs); + auto proof_buf = to_buffer(prove_response.proof); - write_file(output_dir / "public_inputs_fields.json", { public_inputs_json.begin(), public_inputs_json.end() }); - write_file(output_dir / "proof_fields.json", { proof_json.begin(), proof_json.end() }); - info("Public inputs fields saved to ", output_dir / "public_inputs_fields.json"); - info("Proof fields saved to ", output_dir / "proof_fields.json"); - } + write_file(output_dir / "public_inputs", public_inputs_buf); + write_file(output_dir / "proof", proof_buf); + info("Public inputs saved to ", output_dir / "public_inputs"); + info("Proof saved to ", output_dir / "proof"); } -} // namespace + +} // anonymous namespace bool UltraHonkAPI::check([[maybe_unused]] const Flags& flags, [[maybe_unused]] const std::filesystem::path& bytecode_path, @@ -98,24 +68,28 @@ void UltraHonkAPI::prove(const Flags& flags, bbapi::ProofSystemSettings settings{ .ipa_accumulation = flags.ipa_accumulation, .oracle_hash_type = flags.oracle_hash_type, .disable_zk = flags.disable_zk }; + + // Read input files + auto bytecode = get_bytecode(bytecode_path); + auto witness = get_bytecode(witness_path); + // Handle VK std::vector vk_bytes; + if (!vk_path.empty() && !flags.write_vk) { vk_bytes = read_file(vk_path); } // Prove auto response = bbapi::CircuitProve{ .circuit = { .name = "circuit", - .bytecode = get_bytecode(bytecode_path), + .bytecode = std::move(bytecode), .verification_key = std::move(vk_bytes) }, - .witness = get_bytecode(witness_path), + .witness = std::move(witness), .settings = std::move(settings) } .execute(); - - // Write proof outputs (not VK - that's handled above) - write_proof_outputs(response, flags.output_format, output_dir); + write_proof_outputs(response, output_dir); if (flags.write_vk) { - write_vk_outputs(response.vk, flags.output_format, output_dir); + write_vk_outputs(response.vk, output_dir); } } @@ -171,13 +145,11 @@ void UltraHonkAPI::write_vk(const Flags& flags, .oracle_hash_type = flags.oracle_hash_type, .disable_zk = flags.disable_zk }; - // Execute compute VK command auto response = bbapi::CircuitComputeVk{ .circuit = { .name = "circuit", .bytecode = std::move(bytecode) }, .settings = settings } .execute(); - // Write VK outputs using the helper function - write_vk_outputs(response, flags.output_format, output_dir); + write_vk_outputs(response, output_dir); } void UltraHonkAPI::gates([[maybe_unused]] const Flags& flags, diff --git a/barretenberg/cpp/src/barretenberg/api/api_ultra_honk.hpp b/barretenberg/cpp/src/barretenberg/api/api_ultra_honk.hpp index ada7c462dfcf..59ef11011879 100644 --- a/barretenberg/cpp/src/barretenberg/api/api_ultra_honk.hpp +++ b/barretenberg/cpp/src/barretenberg/api/api_ultra_honk.hpp @@ -41,17 +41,4 @@ class UltraHonkAPI : public API { const std::filesystem::path& vk_path) override; }; -template -void write_recursion_inputs_ultra_honk(const std::string& bytecode_path, - const std::string& witness_path, - const std::string& output_path); - -extern template void write_recursion_inputs_ultra_honk(const std::string& bytecode_path, - const std::string& witness_path, - const std::string& output_path); - -extern template void write_recursion_inputs_ultra_honk(const std::string& bytecode_path, - const std::string& witness_path, - const std::string& output_path); - } // namespace bb diff --git a/barretenberg/cpp/src/barretenberg/api/api_ultra_honk.test.cpp b/barretenberg/cpp/src/barretenberg/api/api_ultra_honk.test.cpp index 5b2826469589..30232b96ba6a 100644 --- a/barretenberg/cpp/src/barretenberg/api/api_ultra_honk.test.cpp +++ b/barretenberg/cpp/src/barretenberg/api/api_ultra_honk.test.cpp @@ -8,6 +8,7 @@ #include "barretenberg/dsl/acir_format/proof_surgeon.hpp" #include "barretenberg/flavor/ultra_flavor.hpp" #include "barretenberg/flavor/ultra_rollup_flavor.hpp" +#include "barretenberg/ultra_honk/decider_proving_key.hpp" #include #include #include @@ -76,7 +77,6 @@ TEST_F(ApiUltraHonkTest, ProveAndVerify) auto [bytecode_path, witness_path] = create_test_circuit_files(test_dir); API::Flags flags; - flags.output_format = "bytes"; flags.oracle_hash_type = "poseidon2"; // Set default oracle hash type UltraHonkAPI api; @@ -107,7 +107,6 @@ TEST_F(ApiUltraHonkTest, ProveWithWriteVk) auto [bytecode_path, witness_path] = create_test_circuit_files(test_dir); API::Flags flags; - flags.output_format = "bytes_and_fields"; // Test both output formats flags.oracle_hash_type = "poseidon2"; flags.write_vk = true; @@ -123,8 +122,6 @@ TEST_F(ApiUltraHonkTest, ProveWithWriteVk) EXPECT_TRUE(std::filesystem::exists(proof_output_dir / "public_inputs")); EXPECT_TRUE(std::filesystem::exists(proof_output_dir / "vk")); EXPECT_TRUE(std::filesystem::exists(proof_output_dir / "vk_hash")); - EXPECT_TRUE(std::filesystem::exists(proof_output_dir / "vk_fields.json")); - EXPECT_TRUE(std::filesystem::exists(proof_output_dir / "vk_hash_fields.json")); // Verify the proof bool verified = @@ -136,9 +133,8 @@ TEST_F(ApiUltraHonkTest, ProveAndVerifyWithFields) { auto [bytecode_path, witness_path] = create_test_circuit_files(test_dir); - // First generate VK in bytes format for the prove step + // First generate VK for the prove step API::Flags vk_flags; - vk_flags.output_format = "bytes"; vk_flags.oracle_hash_type = "poseidon2"; UltraHonkAPI api; @@ -148,19 +144,18 @@ TEST_F(ApiUltraHonkTest, ProveAndVerifyWithFields) api.write_vk(vk_flags, bytecode_path, vk_output_path); EXPECT_TRUE(std::filesystem::exists(vk_output_path / "vk")); - // Now test fields format for proof generation + // Now test proof generation API::Flags flags; - flags.output_format = "fields"; flags.oracle_hash_type = "poseidon2"; - // Generate proof with fields output + // Generate proof auto proof_output_dir = test_dir / "proof"; std::filesystem::create_directories(proof_output_dir); api.prove(flags, bytecode_path, witness_path, vk_output_path / "vk", proof_output_dir); - // Check that proof field files were created - EXPECT_TRUE(std::filesystem::exists(proof_output_dir / "proof_fields.json")); - EXPECT_TRUE(std::filesystem::exists(proof_output_dir / "public_inputs_fields.json")); + // Check that proof files were created + EXPECT_TRUE(std::filesystem::exists(proof_output_dir / "proof")); + EXPECT_TRUE(std::filesystem::exists(proof_output_dir / "public_inputs")); } TEST_F(ApiUltraHonkTest, ProveWithDifferentSettings) @@ -176,7 +171,6 @@ TEST_F(ApiUltraHonkTest, ProveWithDifferentSettings) for (const auto& [oracle_hash_type, disable_zk] : test_cases) { API::Flags flags; - flags.output_format = "bytes"; flags.oracle_hash_type = oracle_hash_type; flags.disable_zk = disable_zk; flags.write_vk = true; @@ -198,43 +192,27 @@ TEST_F(ApiUltraHonkTest, ProveWithDifferentSettings) TEST_F(ApiUltraHonkTest, WriteVk) { auto [bytecode_path, witness_path] = create_test_circuit_files(test_dir); + API::Flags flags; + flags.oracle_hash_type = "poseidon2"; - // Smoke test fields format (no real verification) - { - API::Flags flags; - flags.output_format = "fields"; - flags.oracle_hash_type = "poseidon2"; - - UltraHonkAPI api; - api.write_vk(flags, bytecode_path, test_dir); - - EXPECT_TRUE(std::filesystem::exists(test_dir / "vk_fields.json")); - EXPECT_TRUE(std::filesystem::exists(test_dir / "vk_hash_fields.json")); - - EXPECT_FALSE(std::filesystem::exists(test_dir / "vk")); - EXPECT_FALSE(std::filesystem::exists(test_dir / "vk_hash")); - } - - // Test with bytes format, simple vk recalculation - { - API::Flags flags; - flags.output_format = "bytes"; - flags.oracle_hash_type = "poseidon2"; - - UltraHonkAPI api; - api.write_vk(flags, bytecode_path, test_dir); - - // Test against bbapi::CircuitComputeVk - auto bytecode = read_file(bytecode_path); - auto expected_vk = - bbapi::CircuitComputeVk({ .circuit = { .bytecode = bb::decompress(bytecode.data(), bytecode.size()) }, - .settings = { .oracle_hash_type = flags.oracle_hash_type } }) - .execute(); - - info("after write_vk, expected_vk size: {}", expected_vk.bytes.size()); - EXPECT_EQ(expected_vk.bytes, read_file(test_dir / "vk")); - EXPECT_EQ(expected_vk.hash, read_file(test_dir / "vk_hash")); - } + UltraHonkAPI api; + api.write_vk(flags, bytecode_path, test_dir); + + // Test against bbapi::CircuitComputeVk + auto bytecode = read_file(bytecode_path); + auto expected_vk = + bbapi::CircuitComputeVk({ .circuit = { .bytecode = bb::decompress(bytecode.data(), bytecode.size()) }, + .settings = { .oracle_hash_type = flags.oracle_hash_type } }) + .execute(); + + info("after write_vk, expected_vk size: {}", expected_vk.bytes.size()); + EXPECT_EQ(expected_vk.bytes, read_file(test_dir / "vk")); + EXPECT_EQ(expected_vk.hash, read_file(test_dir / "vk_hash")); + + // Verify round-trip: decode the VK and check that to_field_elements() matches + auto vk_from_bytes = from_buffer(expected_vk.bytes); + auto vk_from_file = from_buffer(read_file(test_dir / "vk")); + EXPECT_EQ(vk_from_bytes.to_field_elements(), vk_from_file.to_field_elements()); } // NOTE: very light test diff --git a/barretenberg/cpp/src/barretenberg/api/prove_tube.cpp b/barretenberg/cpp/src/barretenberg/api/prove_tube.cpp index a25d07caf2eb..22065a1228f0 100644 --- a/barretenberg/cpp/src/barretenberg/api/prove_tube.cpp +++ b/barretenberg/cpp/src/barretenberg/api/prove_tube.cpp @@ -1,9 +1,11 @@ #include "prove_tube.hpp" #include "barretenberg/api/file_io.hpp" #include "barretenberg/common/map.hpp" +#include "barretenberg/common/serialize.hpp" #include "barretenberg/honk/proof_system/types/proof.hpp" #include "barretenberg/stdlib/client_ivc_verifier/client_ivc_recursive_verifier.hpp" #include "barretenberg/stdlib/special_public_inputs/special_public_inputs.hpp" +#include namespace bb { /** @@ -70,28 +72,9 @@ void prove_tube(const std::string& output_path, const std::string& vk_path) write_file(tubePublicInputsPath, to_buffer(public_inputs_and_proof.public_inputs)); write_file(tubeProofPath, to_buffer(public_inputs_and_proof.proof)); - std::string tubePublicInputsAsFieldsPath = output_path + "/public_inputs_fields.json"; - std::string tubeProofAsFieldsPath = output_path + "/proof_fields.json"; - const auto to_json = [](const std::vector& data) { - if (data.empty()) { - return std::string("[]"); - } - return format("[", join(transform::map(data, [](auto fr) { return format("\"", fr, "\""); })), "]"); - }; - auto public_inputs_data = to_json(public_inputs_and_proof.public_inputs); - auto proof_data = to_json(public_inputs_and_proof.proof); - write_file(tubePublicInputsAsFieldsPath, { public_inputs_data.begin(), public_inputs_data.end() }); - write_file(tubeProofAsFieldsPath, { proof_data.begin(), proof_data.end() }); - std::string tubeVkPath = output_path + "/vk"; write_file(tubeVkPath, to_buffer(tube_verification_key)); - std::string tubeAsFieldsVkPath = output_path + "/vk_fields.json"; - auto field_els = tube_verification_key->to_field_elements(); - info("verificaton key length in fields:", field_els.size()); - auto data = to_json(field_els); - write_file(tubeAsFieldsVkPath, { data.begin(), data.end() }); - info("Native verification of the tube_proof"); VerifierCommitmentKey ipa_verification_key(1 << CONST_ECCVM_LOG_N); Verifier tube_verifier(tube_verification_key, ipa_verification_key); diff --git a/barretenberg/cpp/src/barretenberg/bb/cli.cpp b/barretenberg/cpp/src/barretenberg/bb/cli.cpp index 76d7510bc19f..4ebbaab9075e 100644 --- a/barretenberg/cpp/src/barretenberg/bb/cli.cpp +++ b/barretenberg/cpp/src/barretenberg/bb/cli.cpp @@ -142,7 +142,6 @@ int parse_and_run_cli_command(int argc, char* argv[]) std::filesystem::path vk_path{ "./target/vk" }; flags.scheme = ""; flags.oracle_hash_type = "poseidon2"; - flags.output_format = "bytes"; flags.crs_path = srs::bb_crs_path(); flags.include_gates_per_opcode = false; const auto add_output_path_option = [&](CLI::App* subcommand, auto& _output_path) { @@ -193,18 +192,6 @@ int parse_and_run_cli_command(int argc, char* argv[]) ->check(CLI::IsMember({ "poseidon2", "keccak", "starknet" }).name("is_member")); }; - const auto add_output_format_option = [&](CLI::App* subcommand) { - return subcommand - ->add_option( - "--output_format", - flags.output_format, - "The type of the data to be written by the command. If bytes, output the raw bytes prefixed with " - "header information for deserialization. If fields, output a string representation of an array of " - "field elements. If bytes_and_fields do both. If fields_msgpack, outputs a msgpack buffer of Fr " - "elements.") - ->check(CLI::IsMember({ "bytes", "fields", "bytes_and_fields", "fields_msgpack" }).name("is_member")); - }; - const auto add_write_vk_flag = [&](CLI::App* subcommand) { return subcommand->add_flag("--write_vk", flags.write_vk, "Write the provided circuit's verification key"); }; @@ -352,7 +339,6 @@ int parse_and_run_cli_command(int argc, char* argv[]) add_debug_flag(prove); add_crs_path_option(prove); add_oracle_hash_option(prove); - add_output_format_option(prove); add_write_vk_flag(prove); add_ipa_accumulation_flag(prove); remove_zk_option(prove); @@ -378,7 +364,6 @@ int parse_and_run_cli_command(int argc, char* argv[]) add_verbose_flag(write_vk); add_debug_flag(write_vk); - add_output_format_option(write_vk); add_crs_path_option(write_vk); add_oracle_hash_option(write_vk); add_ipa_accumulation_flag(write_vk); diff --git a/barretenberg/cpp/src/barretenberg/bbapi/bbapi_client_ivc.cpp b/barretenberg/cpp/src/barretenberg/bbapi/bbapi_client_ivc.cpp index 596f180d8595..a511fc2c4b71 100644 --- a/barretenberg/cpp/src/barretenberg/bbapi/bbapi_client_ivc.cpp +++ b/barretenberg/cpp/src/barretenberg/bbapi/bbapi_client_ivc.cpp @@ -56,6 +56,7 @@ ClientIvcAccumulate::Response ClientIvcAccumulate::execute(BBApiRequest& request std::shared_ptr precomputed_vk; if (!request.loaded_circuit_vk.empty()) { + // Deserialize directly from buffer precomputed_vk = from_buffer>(request.loaded_circuit_vk); } @@ -100,8 +101,8 @@ ClientIvcProve::Response ClientIvcProve::execute(BBApiRequest& request) && ClientIvcVerify::Response ClientIvcVerify::execute(const BBApiRequest& /*request*/) && { BB_BENCH_NAME(MSGPACK_SCHEMA_NAME); - // Deserialize the verification key from the byte buffer - const auto verification_key = from_buffer(vk); + // Deserialize the verification key directly from buffer + ClientIVC::VerificationKey verification_key = from_buffer(vk); // Verify the proof using ClientIVC's static verify method const bool verified = ClientIVC::verify(proof, verification_key); @@ -169,12 +170,13 @@ ClientIvcCheckPrecomputedVk::Response ClientIvcCheckPrecomputedVk::execute(const throw_or_abort("Missing precomputed VK"); } + // Deserialize directly from buffer auto precomputed_vk = from_buffer>(circuit.verification_key); Response response; response.valid = true; std::string error_message = "Precomputed vk does not match computed vk for function " + circuit.name; - if (!msgpack::msgpack_check_eq(*computed_vk, *precomputed_vk, error_message)) { + if (*computed_vk != *precomputed_vk) { response.valid = false; response.actual_vk = to_buffer(computed_vk); } @@ -204,7 +206,7 @@ ClientIvcStats::Response ClientIvcStats::execute(BBApiRequest& request) && builder.finalize_circuit(/*ensure_nonzero=*/true); // Set response values - response.acir_opcodes = static_cast(program.constraints.num_acir_opcodes); + response.acir_opcodes = program.constraints.num_acir_opcodes; response.circuit_size = static_cast(builder.num_gates); // Optionally include gates per opcode diff --git a/barretenberg/cpp/src/barretenberg/bbapi/bbapi_client_ivc.test.cpp b/barretenberg/cpp/src/barretenberg/bbapi/bbapi_client_ivc.test.cpp new file mode 100644 index 000000000000..b144e7adf1f4 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/bbapi/bbapi_client_ivc.test.cpp @@ -0,0 +1,47 @@ +#include "barretenberg/bbapi/bbapi_client_ivc.hpp" +#include "barretenberg/client_ivc/acir_bincode_mocks.hpp" +#include "barretenberg/client_ivc/client_ivc.hpp" +#include "barretenberg/common/serialize.hpp" +#include "barretenberg/dsl/acir_format/acir_format.hpp" +#include "barretenberg/dsl/acir_format/acir_to_constraint_buf.hpp" +#include "barretenberg/flavor/mega_flavor.hpp" +#include + +namespace bb::bbapi { + +class BBApiClientIvcTest : public ::testing::Test { + protected: + static void SetUpTestSuite() { bb::srs::init_file_crs_factory(bb::srs::bb_crs_path()); } +}; + +TEST_F(BBApiClientIvcTest, StandaloneVerificationKeySerialization) +{ + auto [bytecode, witness] = acir_bincode_mocks::create_simple_circuit_bytecode(); + + bbapi::ProofSystemSettings settings{ .ipa_accumulation = false, + .oracle_hash_type = "poseidon2", + .disable_zk = true }; + + // Compute standalone VK using ClientIvcComputeStandaloneVk + auto vk_response = + ClientIvcComputeStandaloneVk{ .circuit = { .name = "test_circuit", .bytecode = bytecode } }.execute(); + + // Create a VK from the field elements + auto vk = + std::make_shared(from_buffer(vk_response.bytes)); + EXPECT_EQ(vk->to_field_elements(), vk_response.fields) + << "Serialized field elements should match original field elements"; +} + +TEST_F(BBApiClientIvcTest, ClientIvcVkSerialization) +{ + auto [bytecode, _witness] = acir_bincode_mocks::create_simple_circuit_bytecode(); + auto vk_response = ClientIvcComputeIvcVk{ .circuit = { .name = "test_circuit", .bytecode = bytecode } }.execute(); + + // Create a VK from the field elements + ClientIVC::VerificationKey vk = from_buffer(vk_response.bytes); + EXPECT_EQ(to_buffer(vk.to_field_elements()), vk_response.bytes) + << "Serialized field elements should match original field elements"; +} + +} // namespace bb::bbapi diff --git a/barretenberg/cpp/src/barretenberg/bbapi/bbapi_ultra_honk.cpp b/barretenberg/cpp/src/barretenberg/bbapi/bbapi_ultra_honk.cpp index 600b8bf10f9d..bc10ec6e0504 100644 --- a/barretenberg/cpp/src/barretenberg/bbapi/bbapi_ultra_honk.cpp +++ b/barretenberg/cpp/src/barretenberg/bbapi/bbapi_ultra_honk.cpp @@ -13,7 +13,10 @@ #include "barretenberg/dsl/acir_proofs/honk_zk_contract.hpp" #include "barretenberg/flavor/mega_flavor.hpp" #include "barretenberg/flavor/ultra_flavor.hpp" +#include "barretenberg/flavor/ultra_keccak_flavor.hpp" +#include "barretenberg/flavor/ultra_keccak_zk_flavor.hpp" #include "barretenberg/flavor/ultra_rollup_flavor.hpp" +#include "barretenberg/flavor/ultra_zk_flavor.hpp" #include "barretenberg/numeric/uint256/uint256.hpp" #include "barretenberg/special_public_inputs/special_public_inputs.hpp" #include "barretenberg/ultra_honk/decider_proving_key.hpp" @@ -72,7 +75,6 @@ std::shared_ptr> _compute_proving_key(std::vector CircuitProve::Response _prove(std::vector&& bytecode, std::vector&& witness, diff --git a/barretenberg/cpp/src/barretenberg/bbapi/c_bind.cpp b/barretenberg/cpp/src/barretenberg/bbapi/c_bind.cpp index e6cfe20f3fce..7e4c02338d8a 100644 --- a/barretenberg/cpp/src/barretenberg/bbapi/c_bind.cpp +++ b/barretenberg/cpp/src/barretenberg/bbapi/c_bind.cpp @@ -13,10 +13,6 @@ namespace bb::bbapi { namespace { // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) BBApiRequest global_request; -#ifndef NO_MULTITHREADING -// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) -std::mutex request_mutex; -#endif } // namespace /** @@ -27,14 +23,6 @@ std::mutex request_mutex; */ CommandResponse bbapi(Command&& command) { -#ifndef NO_MULTITHREADING - // Try to lock, but error if it would block (indicating concurrent access) - std::unique_lock lock(request_mutex, std::try_to_lock); - if (!lock.owns_lock()) { - throw_or_abort("BB API is meant for single-threaded (queued) use only"); - } -#endif - // Execute the command using the global request and return the response return execute(global_request, std::move(command)); } diff --git a/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.hpp b/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.hpp index 25eabd05b07c..bb6ed95e00c2 100644 --- a/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.hpp +++ b/barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.hpp @@ -169,7 +169,59 @@ class ClientIVC { std::shared_ptr eccvm; std::shared_ptr translator; - MSGPACK_FIELDS(mega, eccvm, translator); + /** + * @brief Calculate the number of field elements needed for serialization + * @return size_t Number of field elements + */ + static size_t calc_num_data_types() + { + return MegaVerificationKey::calc_num_data_types() + ECCVMVerificationKey::calc_num_data_types() + + TranslatorVerificationKey::calc_num_data_types(); + } + + /** + * @brief Serialize verification key to field elements + * @return std::vector The serialized field elements + */ + std::vector to_field_elements() const + { + std::vector elements; + + auto mega_elements = mega->to_field_elements(); + elements.insert(elements.end(), mega_elements.begin(), mega_elements.end()); + + auto eccvm_elements = eccvm->to_field_elements(); + elements.insert(elements.end(), eccvm_elements.begin(), eccvm_elements.end()); + + auto translator_elements = translator->to_field_elements(); + elements.insert(elements.end(), translator_elements.begin(), translator_elements.end()); + + return elements; + } + + /** + * @brief Deserialize verification key from field elements + * @param elements The field elements to deserialize from + * @return size_t Number of field elements read + */ + size_t from_field_elements(std::span elements) + { + size_t read_idx = 0; + + mega = std::make_shared(); + size_t mega_read = mega->from_field_elements(elements.subspan(read_idx)); + read_idx += mega_read; + + eccvm = std::make_shared(); + size_t eccvm_read = eccvm->from_field_elements(elements.subspan(read_idx)); + read_idx += eccvm_read; + + translator = std::make_shared(); + size_t translator_read = translator->from_field_elements(elements.subspan(read_idx)); + read_idx += translator_read; + + return read_idx; + } }; // Specifies proof type or equivalently the type of recursive verification to be performed on a given proof @@ -320,4 +372,32 @@ class ClientIVC { bool is_kernel); }; +// Serialization methods for ClientIVC::VerificationKey +inline void read(uint8_t const*& it, ClientIVC::VerificationKey& vk) +{ + using serialize::read; + + size_t num_frs = ClientIVC::VerificationKey::calc_num_data_types(); + + // Read exactly num_frs field elements from the buffer + std::vector field_elements(num_frs); + for (auto& element : field_elements) { + read(it, element); + } + + // Then use from_field_elements to populate the verification key + vk.from_field_elements(field_elements); +} + +inline void write(std::vector& buf, ClientIVC::VerificationKey const& vk) +{ + using serialize::write; + + // Convert to field elements and write them directly without length prefix + auto field_elements = vk.to_field_elements(); + for (const auto& element : field_elements) { + write(buf, element); + } +} + } // namespace bb diff --git a/barretenberg/cpp/src/barretenberg/client_ivc/private_execution_steps.cpp b/barretenberg/cpp/src/barretenberg/client_ivc/private_execution_steps.cpp index a87f5f642a04..3e090973569e 100644 --- a/barretenberg/cpp/src/barretenberg/client_ivc/private_execution_steps.cpp +++ b/barretenberg/cpp/src/barretenberg/client_ivc/private_execution_steps.cpp @@ -1,4 +1,5 @@ #include "private_execution_steps.hpp" +#include "barretenberg/client_ivc/client_ivc.hpp" #include "barretenberg/common/serialize.hpp" #include "barretenberg/dsl/acir_format/acir_to_constraint_buf.hpp" #include @@ -132,8 +133,7 @@ void PrivateExecutionSteps::parse(std::vector&& steps) // For backwards compatibility, but it affects performance and correctness. precomputed_vks[i] = nullptr; } else { - auto vk = from_buffer>(step.vk); - precomputed_vks[i] = vk; + precomputed_vks[i] = from_buffer>(step.vk); } function_names[i] = step.function_name; } @@ -173,8 +173,6 @@ void PrivateExecutionStepRaw::compress_and_save(std::vector inline void read(B& it, std::optional& opt_ opt_value = T(value); } -template -concept HasGetAll = requires(T t) { t.get_all(); } && !msgpack_concepts::HasMsgPack; - -// Write out a struct that defines get_all() -template inline void write(B& buf, T const& value) -{ - using serialize::write; - for (auto& reference : value.get_all()) { - write(buf, reference); - } -} - // Write std::optional. // Note: It takes up a different amount of space, depending on whether it's std::nullopt or populated with an actual // value. diff --git a/barretenberg/cpp/src/barretenberg/dsl/acir_format/mock_verifier_inputs.test.cpp b/barretenberg/cpp/src/barretenberg/dsl/acir_format/mock_verifier_inputs.test.cpp index 50a721643c45..a9706cd0171b 100644 --- a/barretenberg/cpp/src/barretenberg/dsl/acir_format/mock_verifier_inputs.test.cpp +++ b/barretenberg/cpp/src/barretenberg/dsl/acir_format/mock_verifier_inputs.test.cpp @@ -165,7 +165,7 @@ TEST(MockVerifierInputsTest, MockMegaHonkProofSize) * @brief Check that the size of a mock Honk proof matches expectation for Ultra flavors * */ -TYPED_TEST(MockVerifierInputsTest, MockHonkProofSize) +TYPED_TEST(MockVerifierInputsTest, MockUltraHonkProofSize) { using Flavor = TypeParam; using Builder = Flavor::CircuitBuilder; diff --git a/barretenberg/cpp/src/barretenberg/ecc/curves/bn254/bn254.hpp b/barretenberg/cpp/src/barretenberg/ecc/curves/bn254/bn254.hpp index a3831f6a63ec..748f6968f793 100644 --- a/barretenberg/cpp/src/barretenberg/ecc/curves/bn254/bn254.hpp +++ b/barretenberg/cpp/src/barretenberg/ecc/curves/bn254/bn254.hpp @@ -45,4 +45,4 @@ class BN254 { // max(BATCHED_PARTIAL_RELATION_LENGTH) for BN254 Flavors with ZK static constexpr uint32_t LIBRA_UNIVARIATES_LENGTH = 9; }; -} // namespace bb::curve \ No newline at end of file +} // namespace bb::curve diff --git a/barretenberg/cpp/src/barretenberg/eccvm/eccvm_flavor.hpp b/barretenberg/cpp/src/barretenberg/eccvm/eccvm_flavor.hpp index 479721549e5e..55401e268054 100644 --- a/barretenberg/cpp/src/barretenberg/eccvm/eccvm_flavor.hpp +++ b/barretenberg/cpp/src/barretenberg/eccvm/eccvm_flavor.hpp @@ -789,10 +789,6 @@ class ECCVMFlavor { */ class VerificationKey : public NativeVerificationKey_, Transcript> { public: - // Serialized Verification Key length in fields - static constexpr size_t VERIFICATION_KEY_LENGTH = - /* 1. NUM_PRECOMPUTED_ENTITIES commitments */ (NUM_PRECOMPUTED_ENTITIES * num_frs_comm); - bool operator==(const VerificationKey&) const = default; // IPA verification key requires one more point. @@ -827,26 +823,6 @@ class ECCVMFlavor { } } - /** - * @brief Serialize verification key to field elements - * - * @return std::vector - */ - std::vector to_field_elements() const override - { - using namespace bb::field_conversion; - - auto serialize_to_field_buffer = [](const T& input, std::vector& buffer) { - std::vector input_fields = convert_to_bn254_frs(input); - buffer.insert(buffer.end(), input_fields.begin(), input_fields.end()); - }; - - std::vector elements; - for (const Commitment& commitment : this->get_all()) { - serialize_to_field_buffer(commitment, elements); - } - return elements; - } /** * @brief Unused function because vk is hardcoded in recursive verifier, so no transcript hashing is needed. * @@ -861,11 +837,7 @@ class ECCVMFlavor { } // TODO(https://github.com/AztecProtocol/barretenberg/issues/1324): Remove `circuit_size` and `log_circuit_size` - // from MSGPACK and the verification key. - // Don't statically check for object completeness. - using MSGPACK_NO_STATIC_CHECK = std::true_type; - MSGPACK_FIELDS( - log_circuit_size, num_public_inputs, pub_inputs_offset, lagrange_first, lagrange_second, lagrange_last); + // from the verification key. }; /** @@ -1077,7 +1049,4 @@ class ECCVMFlavor { (polynomials.transcript_op[edge_idx] == 0 && polynomials.transcript_op[edge_idx + 1] == 0); } }; - -// NOLINTEND(cppcoreguidelines-avoid-const-or-ref-data-members) - } // namespace bb diff --git a/barretenberg/cpp/src/barretenberg/flavor/flavor.hpp b/barretenberg/cpp/src/barretenberg/flavor/flavor.hpp index 9e9c66ada348..bbdcfad0d958 100644 --- a/barretenberg/cpp/src/barretenberg/flavor/flavor.hpp +++ b/barretenberg/cpp/src/barretenberg/flavor/flavor.hpp @@ -99,6 +99,15 @@ namespace bb { +/** + * @brief Enum to control verification key metadata serialization + */ +enum class VKSerializationMode { + FULL, // Serialize all metadata (log_circuit_size, num_public_inputs, pub_inputs_offset) + NO_METADATA, // Serialize only commitments, no metadata + NO_PUB_OFFSET // Serialize log_circuit_size and num_public_inputs, but not pub_inputs_offset +}; + // Specifies the regions of the execution trace containing non-trivial wire values struct ActiveRegionData { void add_range(const size_t start, const size_t end) @@ -148,7 +157,9 @@ template struct Precomput * * @tparam PrecomputedEntities An instance of PrecomputedEntities_ with affine_element data type and handle type. */ -template +template class NativeVerificationKey_ : public PrecomputedCommitments { public: using Commitment = typename PrecomputedCommitments::DataType; @@ -165,6 +176,28 @@ class NativeVerificationKey_ : public PrecomputedCommitments { this->num_public_inputs = num_public_inputs; }; + /** + * @brief Calculate the number of field elements needed for serialization + * @return size_t Number of field elements + */ + static size_t calc_num_data_types() + { + using namespace bb::field_conversion; + // Create a temporary instance to get the number of precomputed entities + size_t commitments_size = + PrecomputedCommitments::size() * Transcript::template calc_num_data_types(); + size_t metadata_size = 0; + if constexpr (SerializeMetadata == VKSerializationMode::FULL) { + // 3 metadata fields + commitments + metadata_size = 3 * Transcript::template calc_num_data_types(); + } else if constexpr (SerializeMetadata == VKSerializationMode::NO_PUB_OFFSET) { + // 2 metadata fields + commitments + metadata_size = 2 * Transcript::template calc_num_data_types(); + } + // else NO_METADATA: metadata_size remains 0 + return metadata_size + commitments_size; + } + /** * @brief Serialize verification key to field elements * @@ -181,17 +214,56 @@ class NativeVerificationKey_ : public PrecomputedCommitments { std::vector elements; - serialize(this->log_circuit_size, elements); - serialize(this->num_public_inputs, elements); - serialize(this->pub_inputs_offset, elements); + if constexpr (SerializeMetadata == VKSerializationMode::FULL) { + serialize(this->log_circuit_size, elements); + serialize(this->num_public_inputs, elements); + serialize(this->pub_inputs_offset, elements); + } else if constexpr (SerializeMetadata == VKSerializationMode::NO_PUB_OFFSET) { + serialize(this->log_circuit_size, elements); + serialize(this->num_public_inputs, elements); + } + // else NO_METADATA: skip metadata serialization for (const Commitment& commitment : this->get_all()) { serialize(commitment, elements); } + NativeVerificationKey_ key; + key.from_field_elements(elements); return elements; }; + /** + * @brief Populate verification key from field elements + * @param elements Field elements to deserialize from + */ + size_t from_field_elements(const std::span& elements) + { + using namespace bb::field_conversion; + + size_t idx = 0; + auto deserialize = [&idx, &elements](T& target) { + size_t size = Transcript::template calc_num_data_types(); + target = Transcript::template deserialize(elements.subspan(idx, size)); + idx += size; + }; + + if constexpr (SerializeMetadata == VKSerializationMode::FULL) { + deserialize(this->log_circuit_size); + deserialize(this->num_public_inputs); + deserialize(this->pub_inputs_offset); + } else if constexpr (SerializeMetadata == VKSerializationMode::NO_PUB_OFFSET) { + deserialize(this->log_circuit_size); + deserialize(this->num_public_inputs); + } + // else NO_METADATA: skip metadata deserialization + + for (Commitment& commitment : this->get_all()) { + deserialize(commitment); + } + return idx; + } + /** * @brief A model function to show how to compute the VK hash(without the Transcript abstracting things away) * @details Currently only used in testing. @@ -237,8 +309,11 @@ class NativeVerificationKey_ : public PrecomputedCommitments { * @tparam Builder * @tparam FF * @tparam PrecomputedCommitments + * @tparam SerializeMetadata Controls how metadata is serialized (FULL, NO_METADATA, NO_PUB_OFFSET) */ -template +template class StdlibVerificationKey_ : public PrecomputedCommitments { public: using Builder = Builder_; @@ -469,6 +544,44 @@ template class UltraRollupRecursiveFlavor_; template class MegaRecursiveFlavor_; template class MegaZKRecursiveFlavor_; +// Serialization methods for NativeVerificationKey_. +// These should cover all base classes that do not need additional members, as long as the appropriate SerializeMetadata +// is set in the template parameters. +template +inline void read(uint8_t const*& it, NativeVerificationKey_& vk) +{ + using serialize::read; + + // Get the size directly from the static method + size_t num_frs = + NativeVerificationKey_::calc_num_data_types(); + + // Read exactly num_frs field elements from the buffer + std::vector field_elements(num_frs); + for (auto& element : field_elements) { + read(it, element); + } + // Then use from_field_elements to populate the verification key + vk.from_field_elements(field_elements); +} + +template +inline void write(std::vector& buf, + NativeVerificationKey_ const& vk) +{ + using serialize::write; + size_t before = buf.size(); + // Convert to field elements and write them directly without length prefix + auto field_elements = vk.to_field_elements(); + for (const auto& element : field_elements) { + write(buf, element); + } + size_t after = buf.size(); + size_t num_frs = + NativeVerificationKey_::calc_num_data_types(); + BB_ASSERT_EQ(after - before, num_frs * sizeof(bb::fr), "VK serialization mismatch"); +} + namespace avm2 { class AvmRecursiveFlavor; } diff --git a/barretenberg/cpp/src/barretenberg/flavor/mega_flavor.hpp b/barretenberg/cpp/src/barretenberg/flavor/mega_flavor.hpp index 086b9e3b4bdd..8dce7ecfff43 100644 --- a/barretenberg/cpp/src/barretenberg/flavor/mega_flavor.hpp +++ b/barretenberg/cpp/src/barretenberg/flavor/mega_flavor.hpp @@ -439,11 +439,6 @@ class MegaFlavor { */ class VerificationKey : public NativeVerificationKey_, Transcript> { public: - // Serialized Verification Key length in fields - static constexpr size_t VERIFICATION_KEY_LENGTH = - /* 1. Metadata (log_circuit_size, num_public_inputs, pub_inputs_offset) */ (3 * num_frs_fr) + - /* 2. NUM_PRECOMPUTED_ENTITIES commitments */ (NUM_PRECOMPUTED_ENTITIES * num_frs_comm); - VerificationKey() = default; VerificationKey(const size_t circuit_size, const size_t num_public_inputs) : NativeVerificationKey_(circuit_size, num_public_inputs) @@ -467,44 +462,8 @@ class MegaFlavor { commitment = commitment_key.commit(polynomial); } } - - // Don't statically check for object completeness. - using MSGPACK_NO_STATIC_CHECK = std::true_type; - MSGPACK_FIELDS(log_circuit_size, - num_public_inputs, - pub_inputs_offset, - q_m, - q_c, - q_l, - q_r, - q_o, - q_4, - q_busread, - q_lookup, - q_arith, - q_delta_range, - q_elliptic, - q_memory, - q_nnf, - q_poseidon2_external, - q_poseidon2_internal, - sigma_1, - sigma_2, - sigma_3, - sigma_4, - id_1, - id_2, - id_3, - id_4, - table_1, - table_2, - table_3, - table_4, - lagrange_first, - lagrange_last, - lagrange_ecc_op, - databus_id); }; + /** * @brief A container for storing the partially evaluated multivariates produced by sumcheck. */ diff --git a/barretenberg/cpp/src/barretenberg/flavor/native_verification_key.test.cpp b/barretenberg/cpp/src/barretenberg/flavor/native_verification_key.test.cpp index 0fa635745adf..7650934839e4 100644 --- a/barretenberg/cpp/src/barretenberg/flavor/native_verification_key.test.cpp +++ b/barretenberg/cpp/src/barretenberg/flavor/native_verification_key.test.cpp @@ -105,5 +105,5 @@ TYPED_TEST(NativeVerificationKeyTests, VKSizeCheck) using VerificationKey = typename Flavor::VerificationKey; VerificationKey vk(TestFixture::create_vk()); - EXPECT_EQ(vk.to_field_elements().size(), VerificationKey::VERIFICATION_KEY_LENGTH); + EXPECT_EQ(vk.to_field_elements().size(), VerificationKey::calc_num_data_types()); } diff --git a/barretenberg/cpp/src/barretenberg/flavor/ultra_flavor.hpp b/barretenberg/cpp/src/barretenberg/flavor/ultra_flavor.hpp index 69acf5caa0a6..f7d4f1cc213e 100644 --- a/barretenberg/cpp/src/barretenberg/flavor/ultra_flavor.hpp +++ b/barretenberg/cpp/src/barretenberg/flavor/ultra_flavor.hpp @@ -473,11 +473,6 @@ class UltraFlavor { */ class VerificationKey : public NativeVerificationKey_, Transcript> { public: - // Serialized Verification Key length in fields - static constexpr size_t VERIFICATION_KEY_LENGTH = - /* 1. Metadata (log_circuit_size, num_public_inputs, pub_inputs_offset) */ (3 * num_frs_fr) + - /* 2. NUM_PRECOMPUTED_ENTITIES commitments */ (NUM_PRECOMPUTED_ENTITIES * num_frs_comm); - bool operator==(const VerificationKey&) const = default; VerificationKey() = default; VerificationKey(const size_t circuit_size, const size_t num_public_inputs) @@ -495,42 +490,6 @@ class UltraFlavor { commitment = commitment_key.commit(polynomial); } } - - // Don't statically check for object completeness. - using MSGPACK_NO_STATIC_CHECK = std::true_type; - - // For serialising and deserializing data - MSGPACK_FIELDS(log_circuit_size, - num_public_inputs, - pub_inputs_offset, - q_m, - q_c, - q_l, - q_r, - q_o, - q_4, - q_lookup, - q_arith, - q_delta_range, - q_elliptic, - q_memory, - q_nnf, - q_poseidon2_external, - q_poseidon2_internal, - sigma_1, - sigma_2, - sigma_3, - sigma_4, - id_1, - id_2, - id_3, - id_4, - table_1, - table_2, - table_3, - table_4, - lagrange_first, - lagrange_last); }; /** diff --git a/barretenberg/cpp/src/barretenberg/flavor/ultra_keccak_flavor.hpp b/barretenberg/cpp/src/barretenberg/flavor/ultra_keccak_flavor.hpp index 4bb704aa0cfd..38ccefbeaf5a 100644 --- a/barretenberg/cpp/src/barretenberg/flavor/ultra_keccak_flavor.hpp +++ b/barretenberg/cpp/src/barretenberg/flavor/ultra_keccak_flavor.hpp @@ -75,7 +75,6 @@ class UltraKeccakFlavor : public bb::UltraFlavor { static constexpr size_t VERIFICATION_KEY_LENGTH = /* 1. Metadata (log_circuit_size, num_public_inputs, pub_inputs_offset) */ (3 * num_elements_fr) + /* 2. NUM_PRECOMPUTED_ENTITIES commitments */ (NUM_PRECOMPUTED_ENTITIES * num_elements_comm); - VerificationKey() = default; VerificationKey(const size_t circuit_size, const size_t num_public_inputs) : NativeVerificationKey_(circuit_size, num_public_inputs) @@ -92,42 +91,6 @@ class UltraKeccakFlavor : public bb::UltraFlavor { commitment = commitment_key.commit(polynomial); } } - - // Don't statically check for object completeness. - using MSGPACK_NO_STATIC_CHECK = std::true_type; - - // For serialising and deserializing data - MSGPACK_FIELDS(log_circuit_size, - num_public_inputs, - pub_inputs_offset, - q_m, - q_c, - q_l, - q_r, - q_o, - q_4, - q_lookup, - q_arith, - q_delta_range, - q_elliptic, - q_memory, - q_nnf, - q_poseidon2_external, - q_poseidon2_internal, - sigma_1, - sigma_2, - sigma_3, - sigma_4, - id_1, - id_2, - id_3, - id_4, - table_1, - table_2, - table_3, - table_4, - lagrange_first, - lagrange_last); }; // Specialize for Ultra (general case used in UltraRecursive). diff --git a/barretenberg/cpp/src/barretenberg/flavor/ultra_rollup_flavor.hpp b/barretenberg/cpp/src/barretenberg/flavor/ultra_rollup_flavor.hpp index d2f05939a1cf..f28bf66e37f3 100644 --- a/barretenberg/cpp/src/barretenberg/flavor/ultra_rollup_flavor.hpp +++ b/barretenberg/cpp/src/barretenberg/flavor/ultra_rollup_flavor.hpp @@ -33,8 +33,6 @@ class UltraRollupFlavor : public bb::UltraFlavor { */ class VerificationKey : public NativeVerificationKey_, Transcript> { public: - static constexpr size_t VERIFICATION_KEY_LENGTH = UltraFlavor::VerificationKey::VERIFICATION_KEY_LENGTH; - virtual ~VerificationKey() = default; bool operator==(const VerificationKey&) const = default; @@ -54,42 +52,6 @@ class UltraRollupFlavor : public bb::UltraFlavor { commitment = commitment_key.commit(polynomial); } } - - // Don't statically check for object completeness. - using MSGPACK_NO_STATIC_CHECK = std::true_type; - - // For serialising and deserializing data - MSGPACK_FIELDS(log_circuit_size, - num_public_inputs, - pub_inputs_offset, - q_m, - q_c, - q_l, - q_r, - q_o, - q_4, - q_lookup, - q_arith, - q_delta_range, - q_elliptic, - q_memory, - q_nnf, - q_poseidon2_external, - q_poseidon2_internal, - sigma_1, - sigma_2, - sigma_3, - sigma_4, - id_1, - id_2, - id_3, - id_4, - table_1, - table_2, - table_3, - table_4, - lagrange_first, - lagrange_last); }; using VerifierCommitments = VerifierCommitments_; diff --git a/barretenberg/cpp/src/barretenberg/serialize/msgpack.hpp b/barretenberg/cpp/src/barretenberg/serialize/msgpack.hpp index b1b427ac7062..cf09bc92c195 100644 --- a/barretenberg/cpp/src/barretenberg/serialize/msgpack.hpp +++ b/barretenberg/cpp/src/barretenberg/serialize/msgpack.hpp @@ -110,6 +110,7 @@ to the object itself, do break up the above to keep a reference to the handle, f #include "msgpack_impl/concepts.hpp" #include "msgpack_impl/name_value_pair_macro.hpp" #include + #include // Helper for above documented syntax diff --git a/barretenberg/cpp/src/barretenberg/serialize/msgpack_apply.hpp b/barretenberg/cpp/src/barretenberg/serialize/msgpack_apply.hpp index 7816f8a203d7..95fe8ad76693 100644 --- a/barretenberg/cpp/src/barretenberg/serialize/msgpack_apply.hpp +++ b/barretenberg/cpp/src/barretenberg/serialize/msgpack_apply.hpp @@ -1,5 +1,6 @@ #pragma once +#include "barretenberg/common/try_catch_shim.hpp" #include "msgpack.hpp" #include "msgpack_impl/drop_keys.hpp" diff --git a/barretenberg/cpp/src/barretenberg/stdlib/eccvm_verifier/eccvm_recursive_flavor.hpp b/barretenberg/cpp/src/barretenberg/stdlib/eccvm_verifier/eccvm_recursive_flavor.hpp index 5d2193de2565..dadc0d53617b 100644 --- a/barretenberg/cpp/src/barretenberg/stdlib/eccvm_verifier/eccvm_recursive_flavor.hpp +++ b/barretenberg/cpp/src/barretenberg/stdlib/eccvm_verifier/eccvm_recursive_flavor.hpp @@ -91,8 +91,9 @@ class ECCVMRecursiveFlavor { * resolve that, and split out separate PrecomputedPolynomials/Commitments data for clarity but also for * portability of our circuits. */ - class VerificationKey - : public StdlibVerificationKey_> { + class VerificationKey : public StdlibVerificationKey_, + VKSerializationMode::NO_METADATA> { public: VerifierCommitmentKey pcs_verification_key; @@ -119,27 +120,6 @@ class ECCVMRecursiveFlavor { } } - /** - * @brief Serialize verification key to field elements. - * - * @return std::vector - */ - std::vector to_field_elements() const override - { - using namespace bb::stdlib::field_conversion; - auto serialize_to_field_buffer = [](const T& input, std::vector& buffer) { - std::vector input_fields = convert_to_bn254_frs(input); - buffer.insert(buffer.end(), input_fields.begin(), input_fields.end()); - }; - - std::vector elements; - for (const Commitment& commitment : this->get_all()) { - serialize_to_field_buffer(commitment, elements); - } - - return elements; - } - /** * @brief Unused function because vk is hardcoded in recursive verifier, so no transcript hashing is needed. * diff --git a/barretenberg/cpp/src/barretenberg/stdlib/translator_vm_verifier/translator_recursive_flavor.hpp b/barretenberg/cpp/src/barretenberg/stdlib/translator_vm_verifier/translator_recursive_flavor.hpp index 34a080a7eb11..f75ffe84d37f 100644 --- a/barretenberg/cpp/src/barretenberg/stdlib/translator_vm_verifier/translator_recursive_flavor.hpp +++ b/barretenberg/cpp/src/barretenberg/stdlib/translator_vm_verifier/translator_recursive_flavor.hpp @@ -102,8 +102,9 @@ class TranslatorRecursiveFlavor { * resolve that, and split out separate PrecomputedPolynomials/Commitments data for clarity but also for * portability of our circuits. */ - class VerificationKey - : public StdlibVerificationKey_> { + class VerificationKey : public StdlibVerificationKey_, + VKSerializationMode::NO_METADATA> { public: VerificationKey(CircuitBuilder* builder, const std::shared_ptr& native_key) { @@ -119,27 +120,6 @@ class TranslatorRecursiveFlavor { } } - /** - * @brief Serialize verification key to field elements. - * - * @return std::vector - */ - std::vector to_field_elements() const override - { - using namespace bb::stdlib::field_conversion; - auto serialize_to_field_buffer = [](const T& input, std::vector& buffer) { - std::vector input_fields = convert_to_bn254_frs(input); - buffer.insert(buffer.end(), input_fields.begin(), input_fields.end()); - }; - - std::vector elements; - for (const Commitment& commitment : this->get_all()) { - serialize_to_field_buffer(commitment, elements); - } - - return elements; - } - /** * @brief Unused function because vk is hardcoded in recursive verifier, so no transcript hashing is needed. * diff --git a/barretenberg/cpp/src/barretenberg/transcript/transcript.hpp b/barretenberg/cpp/src/barretenberg/transcript/transcript.hpp index c46e5fcea583..e44d652f13a6 100644 --- a/barretenberg/cpp/src/barretenberg/transcript/transcript.hpp +++ b/barretenberg/cpp/src/barretenberg/transcript/transcript.hpp @@ -350,11 +350,21 @@ template class BaseTranscript { * @param element * @return std::vector */ - template static inline std::vector serialize(const T& element) + template static std::vector serialize(const T& element) { return TranscriptParams::serialize(element); } + template static T deserialize(std::span frs) + { + return TranscriptParams::template deserialize(frs); + } + + template static size_t calc_num_data_types() + { + return TranscriptParams::template calc_num_data_types(); + } + /** * @brief After all the prover messages have been sent, finalize the round by hashing all the data and then * create the number of requested challenges. diff --git a/barretenberg/cpp/src/barretenberg/translator_vm/translator_flavor.hpp b/barretenberg/cpp/src/barretenberg/translator_vm/translator_flavor.hpp index ffdd110b4f2b..4ea2ed1245d9 100644 --- a/barretenberg/cpp/src/barretenberg/translator_vm/translator_flavor.hpp +++ b/barretenberg/cpp/src/barretenberg/translator_vm/translator_flavor.hpp @@ -753,10 +753,6 @@ class TranslatorFlavor { */ class VerificationKey : public NativeVerificationKey_, Transcript> { public: - // Serialized Verification Key length in fields - static constexpr size_t VERIFICATION_KEY_LENGTH = - /* 1. NUM_PRECOMPUTED_ENTITIES commitments */ (NUM_PRECOMPUTED_ENTITIES * num_frs_comm); - // Default constuct the fixed VK based on circuit size 1 << CONST_TRANSLATOR_LOG_N VerificationKey() : NativeVerificationKey_(1UL << CONST_TRANSLATOR_LOG_N, /*num_public_inputs=*/0) @@ -782,27 +778,6 @@ class TranslatorFlavor { } } - /** - * @brief Serialize verification key to field elements - * - * @return std::vector - */ - std::vector to_field_elements() const override - { - using namespace bb::field_conversion; - - auto serialize_to_field_buffer = [](const T& input, std::vector& buffer) { - std::vector input_fields = convert_to_bn254_frs(input); - buffer.insert(buffer.end(), input_fields.begin(), input_fields.end()); - }; - - std::vector elements; - for (const Commitment& commitment : this->get_all()) { - serialize_to_field_buffer(commitment, elements); - } - return elements; - } - /** * @brief Unused function because vk is hardcoded in recursive verifier, so no transcript hashing is needed. * @@ -814,22 +789,6 @@ class TranslatorFlavor { { throw_or_abort("Not intended to be used because vk is hardcoded in circuit."); } - - // Don't statically check for object completeness. - using MSGPACK_NO_STATIC_CHECK = std::true_type; - - MSGPACK_FIELDS(num_public_inputs, - pub_inputs_offset, - ordered_extra_range_constraints_numerator, - lagrange_first, - lagrange_last, - lagrange_odd_in_minicircuit, - lagrange_even_in_minicircuit, - lagrange_result_row, - lagrange_last_in_minicircuit, - lagrange_masking, - lagrange_mini_masking, - lagrange_real_last); }; /** @@ -1029,4 +988,5 @@ class TranslatorFlavor { } using VerifierCommitments = VerifierCommitments_; }; + } // namespace bb diff --git a/barretenberg/cpp/src/barretenberg/ultra_honk/decider_verification_key.hpp b/barretenberg/cpp/src/barretenberg/ultra_honk/decider_verification_key.hpp index c94d58dbdcaf..99e4a17cea16 100644 --- a/barretenberg/cpp/src/barretenberg/ultra_honk/decider_verification_key.hpp +++ b/barretenberg/cpp/src/barretenberg/ultra_honk/decider_verification_key.hpp @@ -82,4 +82,5 @@ template class DeciderVerificationKey_ { MSGPACK_FIELDS(vk, relation_parameters, alphas, is_complete, gate_challenges, target_sum, witness_commitments); }; + } // namespace bb diff --git a/barretenberg/cpp/src/barretenberg/vm2/constraining/flavor.cpp b/barretenberg/cpp/src/barretenberg/vm2/constraining/flavor.cpp index 272c2072e119..56e20f74137a 100644 --- a/barretenberg/cpp/src/barretenberg/vm2/constraining/flavor.cpp +++ b/barretenberg/cpp/src/barretenberg/vm2/constraining/flavor.cpp @@ -97,21 +97,4 @@ AvmFlavor::ProvingKey::ProvingKey() // The proving key's polynomials are not allocated here because they are later overwritten // AvmComposer::compute_witness(). We should probably refactor this flow. }; - -/** - * @brief Serialize verification key to field elements - * - * @return std::vector - */ -std::vector AvmFlavor::VerificationKey::to_field_elements() const -{ - std::vector elements; - - for (auto const& comm : get_all()) { - std::vector comm_as_fields = field_conversion::convert_to_bn254_frs(comm); - elements.insert(elements.end(), comm_as_fields.begin(), comm_as_fields.end()); - } - return elements; -} - } // namespace bb::avm2 diff --git a/barretenberg/cpp/src/barretenberg/vm2/constraining/flavor.hpp b/barretenberg/cpp/src/barretenberg/vm2/constraining/flavor.hpp index 8bb90d560e88..0fc06af31c9d 100644 --- a/barretenberg/cpp/src/barretenberg/vm2/constraining/flavor.hpp +++ b/barretenberg/cpp/src/barretenberg/vm2/constraining/flavor.hpp @@ -237,7 +237,9 @@ class AvmFlavor { auto get_to_be_shifted() { return AvmFlavor::get_to_be_shifted(*this); } }; - class VerificationKey : public NativeVerificationKey_, Transcript> { + class VerificationKey : public NativeVerificationKey_, + Transcript, + VKSerializationMode::NO_PUB_OFFSET> { public: static constexpr size_t NUM_PRECOMPUTED_COMMITMENTS = NUM_PRECOMPUTED_ENTITIES; @@ -260,7 +262,6 @@ class AvmFlavor { } } - std::vector to_field_elements() const override; /** * @brief Unimplemented because AVM VK is hardcoded so hash does not need to be computed. Rather, we just add * the provided VK hash directly to the transcript. diff --git a/barretenberg/docs/docs/how_to_guides/how-to-solidity-verifier.md b/barretenberg/docs/docs/how_to_guides/how-to-solidity-verifier.md index 1a686961a985..c43e18bffea0 100644 --- a/barretenberg/docs/docs/how_to_guides/how-to-solidity-verifier.md +++ b/barretenberg/docs/docs/how_to_guides/how-to-solidity-verifier.md @@ -101,14 +101,12 @@ This will generate a `Prover.toml` you can fill with the values you want to prov ```bash nargo execute -bb prove -b ./target/.json -w ./target/ -o ./target --oracle_hash keccak --output_format bytes_and_fields +bb prove -b ./target/.json -w ./target/ -o ./target --oracle_hash keccak ``` -Why `bytes_and_fields? +Binary Output Format -By default, barretenberg appends the public inputs to the proof, which may or may not be useful. You'll find that it is definitely not useful on a Solidity verifier, as the public inputs will often be provided by the contract. - -The flag `--output_format bytes_and_fields` makes `bb` output a `proof` file with _just_ the proof, and the `public_inputs_fields.json` file with _just_ the public inputs. +Barretenberg outputs `proof` and `public_inputs` files in binary format. The binary format is fields-compatible, meaning it can be split into 32-byte chunks where each chunk represents a field element. A programmatic example of how the `verify` function is called can be seen in the example zk voting application [here](https://github.com/noir-lang/noir-examples/blob/33e598c257e2402ea3a6b68dd4c5ad492bce1b0a/foundry-voting/src/zkVote.sol#L35): diff --git a/barretenberg/docs/scripts/build_docs.sh b/barretenberg/docs/scripts/build_docs.sh index bdbee1be9f9c..5b90d0657f5e 100755 --- a/barretenberg/docs/scripts/build_docs.sh +++ b/barretenberg/docs/scripts/build_docs.sh @@ -1,34 +1,26 @@ #!/bin/bash -# Script to build Doxygen documentation and copy to Docusaurus static directory - set -e echo "Building Doxygen documentation..." - -# Navigate to the C++ root directory so paths in Doxyfile are correct cd "$(dirname "$0")/../../cpp" - -# Build the documentation using Doxygen from the docs subdirectory -doxygen docs/Doxyfile +doxygen -q docs/Doxyfile echo "Copying Doxygen HTML to Docusaurus static directory..." -# Create the static/api directory if it doesn't exist -mkdir -p "../docs/static/api" - -# Copy the built documentation from the correct path -if [ -d "docs/build" ]; then - # First, clean the destination to avoid any leftover files - rm -rf ../docs/static/api/* - - # Copy the built documentation - cp -R docs/build/* ../docs/static/api/ - -else +if ! [ -d "docs/build" ]; then echo "Error: docs/build directory not found!" exit 1 fi +# First, clean the destination to avoid any leftover files +rm -rf ../docs/static/api/* + +# Copy the built documentation +mkdir -p ../docs/static/api/ +cp -R docs/build/* ../docs/static/api/ + +# NOTE(AD): hack - but was blocked and couldn't figure out why we had two examples for something called 'if' with different casing. +rm ../docs/static/api/if-example.html echo "Doxygen documentation successfully built and copied to Docusaurus!" echo "You can now build and serve the Docusaurus site to view the integrated documentation." diff --git a/barretenberg/docs/versioned_docs/version-v0.87.0/how_to_guides/how-to-solidity-verifier.md b/barretenberg/docs/versioned_docs/version-v0.87.0/how_to_guides/how-to-solidity-verifier.md index 1a686961a985..b0c77df00c23 100644 --- a/barretenberg/docs/versioned_docs/version-v0.87.0/how_to_guides/how-to-solidity-verifier.md +++ b/barretenberg/docs/versioned_docs/version-v0.87.0/how_to_guides/how-to-solidity-verifier.md @@ -101,14 +101,14 @@ This will generate a `Prover.toml` you can fill with the values you want to prov ```bash nargo execute -bb prove -b ./target/.json -w ./target/ -o ./target --oracle_hash keccak --output_format bytes_and_fields +bb prove -b ./target/.json -w ./target/ -o ./target --oracle_hash keccak ``` -Why `bytes_and_fields? +Binary Output Format -By default, barretenberg appends the public inputs to the proof, which may or may not be useful. You'll find that it is definitely not useful on a Solidity verifier, as the public inputs will often be provided by the contract. +Barretenberg outputs `proof` and `public_inputs` files in binary format. The binary format is fields-compatible, meaning it can be split into 32-byte chunks where each chunk represents a field element. -The flag `--output_format bytes_and_fields` makes `bb` output a `proof` file with _just_ the proof, and the `public_inputs_fields.json` file with _just_ the public inputs. +This produces a `proof` file with the proof data and a `public_inputs` file with the public input values. A programmatic example of how the `verify` function is called can be seen in the example zk voting application [here](https://github.com/noir-lang/noir-examples/blob/33e598c257e2402ea3a6b68dd4c5ad492bce1b0a/foundry-voting/src/zkVote.sol#L35): diff --git a/barretenberg/ts/src/barretenberg/backend.ts b/barretenberg/ts/src/barretenberg/backend.ts index 4bc08be7086a..5012031c5c0a 100644 --- a/barretenberg/ts/src/barretenberg/backend.ts +++ b/barretenberg/ts/src/barretenberg/backend.ts @@ -1,8 +1,6 @@ import { BackendOptions, Barretenberg, CircuitOptions } from './index.js'; import { RawBuffer } from '../types/raw_buffer.js'; import { - deflattenFields, - flattenFieldsAsArray, ProofData, reconstructHonkProof, splitHonkProof, @@ -47,6 +45,59 @@ export type UltraHonkBackendOptions = { starknetZK?: boolean; }; + +function getProofSettingsFromOptions( + options?: UltraHonkBackendOptions, +): { ipaAccumulation: boolean; oracleHashType: string; disableZk: boolean } { + return { + ipaAccumulation: false, + oracleHashType: options?.keccak || options?.keccakZK ? 'keccak' : (options?.starknet || options?.starknetZK ? 'starknet' : 'poseidon2'), + // TODO no current way to target non-zk poseidon2 hash + disableZk: options?.keccak || options?.starknet ? true : false, + }; +} + +export class UltraHonkVerifierBackend { + protected api!: Barretenberg; + + constructor( + protected backendOptions: BackendOptions = { threads: 1 }, + protected circuitOptions: CircuitOptions = { recursive: false }, + ) {} + /** @ignore */ + private async instantiate(): Promise { + if (!this.api) { + const api = await Barretenberg.new(this.backendOptions); + const honkRecursion = true; + await api.initSRSForCircuitSize(0); + + this.api = api; + } + } + + async verifyProof(proofData: ProofData & { verificationKey: Uint8Array }, options?: UltraHonkBackendOptions): Promise { + await this.instantiate(); + + const proofFrs: Uint8Array[] = []; + for (let i = 0; i < proofData.proof.length; i += 32) { + proofFrs.push(proofData.proof.slice(i, i + 32)); + } + const { verified } = await this.api.circuitVerify({ + verificationKey: proofData.verificationKey, + publicInputs: proofData.publicInputs.map(hexToUint8Array), + proof: proofFrs, + settings: getProofSettingsFromOptions(options), + }); + return verified; + } + destroy(): Promise { + if (!this.api) { + return Promise.resolve(); + } + return this.api.destroy(); + } +} + export class UltraHonkBackend { // These type assertions are used so that we don't // have to initialize `api` in the constructor. @@ -97,7 +148,7 @@ export class UltraHonkBackend { bytecode: Buffer.from(this.acirUncompressedBytecode), verificationKey: Buffer.from([]), // Empty VK - lower performance. }, - settings: this.getProofSettingsFromOptions(options) + settings: getProofSettingsFromOptions(options) }); console.log(`Generated proof for circuit with ${publicInputs.length} public inputs and ${proof.length} fields.`); @@ -123,13 +174,13 @@ export class UltraHonkBackend { name: 'circuit', bytecode: this.acirUncompressedBytecode, }, - settings: this.getProofSettingsFromOptions(options), + settings: getProofSettingsFromOptions(options), }); const {verified} = await this.api.circuitVerify({ verificationKey: vkResult.bytes, publicInputs: proofData.publicInputs.map(hexToUint8Array), proof: proofFrs, - settings: this.getProofSettingsFromOptions(options), + settings: getProofSettingsFromOptions(options), }); return verified; } @@ -142,7 +193,7 @@ export class UltraHonkBackend { name: 'circuit', bytecode: Buffer.from(this.acirUncompressedBytecode), }, - settings: this.getProofSettingsFromOptions(options), + settings: getProofSettingsFromOptions(options), }); return vkResult.bytes; } @@ -177,13 +228,20 @@ export class UltraHonkBackend { name: 'circuit', bytecode: Buffer.from(this.acirUncompressedBytecode), }, - settings: this.getProofSettingsFromOptions({}), + settings: getProofSettingsFromOptions({}), }); + // Convert VK bytes to field elements (32-byte chunks) + const vkAsFields: string[] = []; + for (let i = 0; i < vkResult.bytes.length; i += 32) { + const chunk = vkResult.bytes.slice(i, i + 32); + vkAsFields.push(uint8ArrayToHex(chunk)); + } + return { // TODO(https://github.com/noir-lang/noir/issues/5661) proofAsFields: [], - vkAsFields: vkResult.fields.map(field => field.toString()), + vkAsFields, // We use an empty string for the vk hash here as it is unneeded as part of the recursive artifacts // The user can be expected to hash the vk inside their circuit to check whether the vk is the circuit // they expect diff --git a/barretenberg/ts/src/barretenberg/index.ts b/barretenberg/ts/src/barretenberg/index.ts index 289beb381a93..6454512559c2 100644 --- a/barretenberg/ts/src/barretenberg/index.ts +++ b/barretenberg/ts/src/barretenberg/index.ts @@ -10,8 +10,7 @@ import { createDebugLogger } from '../log/index.js'; import { AsyncApi } from '../cbind/generated/async.js'; import { BbApiBase, CircuitComputeVk, CircuitProve, CircuitVerify, ClientIvcAccumulate, ClientIvcComputeIvcVk, ClientIvcStats, ClientIvcLoad, ClientIvcProve, ClientIvcStart, ClientIvcVerify, VkAsFields } from '../cbind/generated/api_types.js'; -export { BarretenbergVerifier } from './verifier.js'; -export { UltraHonkBackend, AztecClientBackend } from './backend.js'; +export { UltraHonkBackend, UltraHonkVerifierBackend, AztecClientBackend } from './backend.js'; export type BackendOptions = { /** @description Number of threads to run the backend worker on */ @@ -133,9 +132,6 @@ export class Barretenberg extends BarretenbergApi { return this.bbApi.clientIvcAccumulate(command); } - - - async clientIvcProve(command: ClientIvcProve) { return this.bbApi.clientIvcProve(command); } @@ -168,6 +164,7 @@ export class Barretenberg extends BarretenbergApi { async vkAsFields(command: VkAsFields) { return this.bbApi.vkAsFields(command); } + } let barretenbergSyncSingletonPromise: Promise; diff --git a/barretenberg/ts/src/barretenberg/verifier.ts b/barretenberg/ts/src/barretenberg/verifier.ts deleted file mode 100644 index 76372ec84a89..000000000000 --- a/barretenberg/ts/src/barretenberg/verifier.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { BackendOptions, Barretenberg } from './index.js'; -import { RawBuffer } from '../types/raw_buffer.js'; -import { flattenFieldsAsArray, ProofData, reconstructHonkProof } from '../proof/index.js'; - -// TODO: once UP is removed we can just roll this into the bas `Barretenberg` class. - -export class BarretenbergVerifier { - // These type assertions are used so that we don't - // have to initialize `api` and `acirComposer` in the constructor. - // These are initialized asynchronously in the `init` function, - // constructors cannot be asynchronous which is why we do this. - - private api!: Barretenberg; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - private acirComposer: any; - - constructor(private options: BackendOptions = { threads: 1 }) {} - - /** @ignore */ - async instantiate(): Promise { - if (!this.api) { - const api = await Barretenberg.new(this.options); - await api.initSRSForCircuitSize(0); - - this.api = api; - } - } - - /** @description Verifies a proof */ - async verifyUltraHonkProof(proofData: ProofData, verificationKey: Uint8Array): Promise { - await this.instantiate(); - - const proof = reconstructHonkProof(flattenFieldsAsArray(proofData.publicInputs), proofData.proof); - return await this.api.acirVerifyUltraZKHonk(proof, new RawBuffer(verificationKey)); - } - - async destroy(): Promise { - if (!this.api) { - return; - } - await this.api.destroy(); - } -} diff --git a/barretenberg/ts/src/index.ts b/barretenberg/ts/src/index.ts index d13cc2374f46..c8330ef3f640 100644 --- a/barretenberg/ts/src/index.ts +++ b/barretenberg/ts/src/index.ts @@ -3,7 +3,7 @@ export { type BackendOptions, Barretenberg, BarretenbergSync, - BarretenbergVerifier, + UltraHonkVerifierBackend, UltraHonkBackend, AztecClientBackend, } from './barretenberg/index.js'; diff --git a/noir-projects/noir-protocol-circuits/bootstrap.sh b/noir-projects/noir-protocol-circuits/bootstrap.sh index 6ee4c0968bd6..07b7ce5ce0ad 100755 --- a/noir-projects/noir-protocol-circuits/bootstrap.sh +++ b/noir-projects/noir-protocol-circuits/bootstrap.sh @@ -41,6 +41,11 @@ function on_exit { } trap on_exit EXIT +function hex_to_fields_json { + # 1. split encoded hex into 64-character lines 3. encode as JSON array of hex strings + fold -w64 | jq -R -s -c 'split("\n") | map(select(length > 0)) | map("0x" + .)' +} + function compile { set -euo pipefail local dir=$1 @@ -106,11 +111,13 @@ function compile { SECONDS=0 outdir=$(mktemp -d) trap "rm -rf $outdir" EXIT - local vk_cmd="jq -r '.bytecode' $json_path | base64 -d | gunzip | $BB $write_vk_cmd -b - -o $outdir --output_format bytes_and_fields" + local vk_cmd="jq -r '.bytecode' $json_path | base64 -d | gunzip | $BB $write_vk_cmd -b - -o $outdir" echo_stderr $vk_cmd dump_fail "$vk_cmd" vk_bytes=$(cat $outdir/vk | xxd -p -c 0) - vk_fields=$(cat $outdir/vk_fields.json) + # Split the hex-encoded vk bytes into fields boundaries (but still hex-encoded), first making 64-character lines and then encoding as JSON. + # This used to be done by barretenberg itself, but with serialization now always being in field elements we can do it outside of bb. + vk_fields=$(echo "$vk_bytes" | hex_to_fields_json) # echo_stderr $vkf_cmd jq -n --arg vk "$vk_bytes" --argjson vkf "$vk_fields" '{keyAsBytes: $vk, keyAsFields: $vkf}' > $key_path echo_stderr "Key output at: $key_path (${SECONDS}s)" @@ -137,7 +144,7 @@ function compile { fi fi } -export -f compile +export -f hex_to_fields_json compile function build { set -eu diff --git a/yarn-project/bb-prover/src/bb/execute.ts b/yarn-project/bb-prover/src/bb/execute.ts index 2b3554c77238..6e002785254a 100644 --- a/yarn-project/bb-prover/src/bb/execute.ts +++ b/yarn-project/bb-prover/src/bb/execute.ts @@ -10,11 +10,8 @@ import { basename, dirname, join } from 'path'; import type { UltraHonkFlavor } from '../honk.js'; export const VK_FILENAME = 'vk'; -export const VK_FIELDS_FILENAME = 'vk_fields.json'; export const PUBLIC_INPUTS_FILENAME = 'public_inputs'; -export const PUBLIC_INPUTS_FIELDS_FILENAME = 'public_inputs_fields.json'; export const PROOF_FILENAME = 'proof'; -export const PROOF_FIELDS_FILENAME = 'proof_fields.json'; export const AVM_INPUTS_FILENAME = 'avm_inputs.bin'; export const AVM_BYTECODE_FILENAME = 'avm_bytecode.bin'; export const AVM_PUBLIC_INPUTS_FILENAME = 'avm_public_inputs.bin'; @@ -248,8 +245,6 @@ export async function generateProof( // TODO(#15043): Avoid write_vk flag here. const args = getArgs(flavor).concat([ '--disable_zk', - '--output_format', - 'bytes_and_fields', '--write_vk', '-o', outputPath, diff --git a/yarn-project/bb-prover/src/prover/proof_utils.ts b/yarn-project/bb-prover/src/prover/proof_utils.ts index 8003d822cab2..28dd1f2f9567 100644 --- a/yarn-project/bb-prover/src/prover/proof_utils.ts +++ b/yarn-project/bb-prover/src/prover/proof_utils.ts @@ -14,12 +14,7 @@ import assert from 'assert'; import { promises as fs } from 'fs'; import * as path from 'path'; -import { - CLIENT_IVC_PROOF_FILE_NAME, - PROOF_FIELDS_FILENAME, - PROOF_FILENAME, - PUBLIC_INPUTS_FILENAME, -} from '../bb/execute.js'; +import { CLIENT_IVC_PROOF_FILE_NAME, PROOF_FILENAME, PUBLIC_INPUTS_FILENAME } from '../bb/execute.js'; /** * Create a ClientIvcProof proof file. @@ -43,6 +38,14 @@ export async function writeClientIVCProofToOutputDirectory(clientIvcProof: Clien await fs.writeFile(path.join(directory, CLIENT_IVC_PROOF_FILE_NAME), clientIvcProofBuffer); } +function splitBufferIntoFields(buffer: Buffer): Fr[] { + const fields: Fr[] = []; + for (let i = 0; i < buffer.length / Fr.SIZE_IN_BYTES; i++) { + fields.push(Fr.fromBuffer(buffer.subarray(i * Fr.SIZE_IN_BYTES, (i + 1) * Fr.SIZE_IN_BYTES))); + } + return fields; +} + export async function readProofAsFields( filePath: string, vkData: VerificationKeyData, @@ -51,15 +54,13 @@ export async function readProofAsFields( ): Promise> { const publicInputsFilename = path.join(filePath, PUBLIC_INPUTS_FILENAME); const proofFilename = path.join(filePath, PROOF_FILENAME); - const proofFieldsFilename = path.join(filePath, PROOF_FIELDS_FILENAME); - const [binaryPublicInputs, binaryProof, proofString] = await Promise.all([ + const [binaryPublicInputs, binaryProof] = await Promise.all([ fs.readFile(publicInputsFilename), fs.readFile(proofFilename), - fs.readFile(proofFieldsFilename, { encoding: 'utf-8' }), ]); - const json = JSON.parse(proofString); + const fieldsWithoutPublicInputs = splitBufferIntoFields(binaryProof); let numPublicInputs = vkData.numPublicInputs - PAIRING_POINTS_SIZE; assert( @@ -72,19 +73,20 @@ export async function readProofAsFields( numPublicInputs -= IPA_CLAIM_SIZE; } - assert(json.length == proofLength, `Proof length mismatch: ${json.length} != ${proofLength}`); - - const fieldsWithoutPublicInputs = json.map(Fr.fromHexString); + assert( + fieldsWithoutPublicInputs.length == proofLength, + `Proof length mismatch: ${fieldsWithoutPublicInputs.length} != ${proofLength}`, + ); // Concat binary public inputs and binary proof // This buffer will have the form: [binary public inputs, binary proof] const binaryProofWithPublicInputs = Buffer.concat([binaryPublicInputs, binaryProof]); logger.debug( - `Circuit path: ${filePath}, complete proof length: ${json.length}, num public inputs: ${numPublicInputs}, circuit size: ${vkData.circuitSize}, raw length: ${binaryProofWithPublicInputs.length}`, + `Circuit path: ${filePath}, complete proof length: ${fieldsWithoutPublicInputs.length}, num public inputs: ${numPublicInputs}, circuit size: ${vkData.circuitSize}`, ); assert( - binaryProofWithPublicInputs.length == numPublicInputs * 32 + proofLength * 32, - `Proof length mismatch: ${binaryProofWithPublicInputs.length} != ${numPublicInputs * 32 + proofLength * 32}`, + fieldsWithoutPublicInputs.length == numPublicInputs + proofLength, + `Proof length mismatch: ${fieldsWithoutPublicInputs.length} != ${numPublicInputs + proofLength}`, ); return new RecursiveProof( fieldsWithoutPublicInputs, diff --git a/yarn-project/bb-prover/src/verification_key/verification_key_data.ts b/yarn-project/bb-prover/src/verification_key/verification_key_data.ts index 8477922c6b91..2dd4b233ef6d 100644 --- a/yarn-project/bb-prover/src/verification_key/verification_key_data.ts +++ b/yarn-project/bb-prover/src/verification_key/verification_key_data.ts @@ -7,7 +7,7 @@ import { strict as assert } from 'assert'; import { promises as fs } from 'fs'; import * as path from 'path'; -import { VK_FIELDS_FILENAME, VK_FILENAME } from '../bb/execute.js'; +import { VK_FILENAME } from '../bb/execute.js'; /** * Reads the verification key data stored at the specified location and parses into a VerificationKeyData @@ -15,12 +15,13 @@ import { VK_FIELDS_FILENAME, VK_FILENAME } from '../bb/execute.js'; * @returns The verification key data */ export async function extractVkData(vkDirectoryPath: string): Promise { - const [rawFields, rawBinary] = await Promise.all([ - fs.readFile(path.join(vkDirectoryPath, VK_FIELDS_FILENAME), { encoding: 'utf-8' }), - fs.readFile(path.join(vkDirectoryPath, VK_FILENAME)), - ]); - const fieldsJson = JSON.parse(rawFields); - const fields = fieldsJson.map(Fr.fromHexString); + const rawBinary = await fs.readFile(path.join(vkDirectoryPath, VK_FILENAME)); + + // Convert binary to field elements (32 bytes per field) + const numFields = rawBinary.length / Fr.SIZE_IN_BYTES; + const reader = BufferReader.asReader(rawBinary); + const fields = reader.readArray(numFields, Fr); + const vkAsFields = await VerificationKeyAsFields.fromKey(fields); return new VerificationKeyData(vkAsFields, rawBinary); }