From 1a2c7edf4055ac2dbced8acad2a48b7ae3114019 Mon Sep 17 00:00:00 2001 From: mrmr1993 Date: Sun, 11 Feb 2024 07:48:55 +0000 Subject: [PATCH 01/45] Update snarky to expose callback-oriented helpers, use in pickles step --- src/lib/pickles/step.ml | 100 +++++++++++++++++----------------------- src/lib/snarky | 2 +- 2 files changed, 44 insertions(+), 58 deletions(-) diff --git a/src/lib/pickles/step.ml b/src/lib/pickles/step.ml index 114431cbaaf..f1f6590524c 100644 --- a/src/lib/pickles/step.ml +++ b/src/lib/pickles/step.ml @@ -791,64 +791,50 @@ struct in let { Domains.h } = Vector.nth_exn step_domains branch_data.index in ksprintf Common.time "step-prover %d (%d)" branch_data.index - (Domain.size h) - (fun () -> - let promise_or_error = - (* Use a try_with to give an informative backtrace. - If we don't do this, the backtrace will be obfuscated by the - Promise, and it's significantly harder to track down errors. - This only applies to errors in the 'witness generation' stage; - proving errors are emitted inside the promise, and are therefore - unaffected. - *) - Or_error.try_with ~backtrace:true (fun () -> - [%log internal] "Step_generate_witness_conv" ; - Impls.Step.generate_witness_conv - ~f:(fun { Impls.Step.Proof_inputs.auxiliary_inputs - ; public_inputs - } next_statement_hashed -> - [%log internal] "Backend_tick_proof_create_async" ; - let create_proof () = - Backend.Tick.Proof.create_async ~primary:public_inputs - ~auxiliary:auxiliary_inputs - ~message: - (Lazy.force prev_challenge_polynomial_commitments) - pk - in - let%map.Promise proof = - match proof_cache with - | None -> - create_proof () - | Some proof_cache -> ( - match - Proof_cache.get_step_proof proof_cache ~keypair:pk - ~public_input:public_inputs - with - | None -> - if - Proof_cache - .is_env_var_set_requesting_error_for_proofs () - then failwith "Regenerated proof" ; - let%map.Promise proof = create_proof () in - Proof_cache.set_step_proof proof_cache ~keypair:pk - ~public_input:public_inputs proof.proof ; - proof - | Some proof -> - Promise.return - ( { proof; public_evals = None } - : Tick.Proof.with_public_evals ) ) - in - [%log internal] "Backend_tick_proof_create_async_done" ; - (proof, next_statement_hashed) ) - ~input_typ:Impls.Step.Typ.unit ~return_typ:input - (fun () () -> - Impls.Step.handle - (fun () -> conv_inv (branch_data.main ~step_domains ())) - handler ) ) + (Domain.size h) (fun () -> + [%log internal] "Step_generate_witness_conv" ; + let builder = + Impls.Step.generate_witness_manual ~handlers:[ handler ] + ~input_typ:Impls.Step.Typ.unit ~return_typ:input () + in + let res = + builder.run_circuit (fun () () -> + conv_inv (branch_data.main ~step_domains ()) ) + in + let ( { Impls.Step.Proof_inputs.auxiliary_inputs; public_inputs } + , next_statement_hashed ) = + builder.finish_computation res + in + [%log internal] "Backend_tick_proof_create_async" ; + let create_proof () = + Backend.Tick.Proof.create_async ~primary:public_inputs + ~auxiliary:auxiliary_inputs + ~message:(Lazy.force prev_challenge_polynomial_commitments) + pk + in + let%map.Promise proof = + match proof_cache with + | None -> + create_proof () + | Some proof_cache -> ( + match + Proof_cache.get_step_proof proof_cache ~keypair:pk + ~public_input:public_inputs + with + | None -> + if Proof_cache.is_env_var_set_requesting_error_for_proofs () + then failwith "Regenerated proof" ; + let%map.Promise proof = create_proof () in + Proof_cache.set_step_proof proof_cache ~keypair:pk + ~public_input:public_inputs proof.proof ; + proof + | Some proof -> + Promise.return + ( { proof; public_evals = None } + : Tick.Proof.with_public_evals ) ) in - (* Re-raise any captured errors, complete with their backtrace. *) - Or_error.ok_exn promise_or_error ) - () + [%log internal] "Backend_tick_proof_create_async_done" ; + (proof, next_statement_hashed) ) in let prev_evals = extract_from_proofs diff --git a/src/lib/snarky b/src/lib/snarky index 94b2df82129..8cfd780d211 160000 --- a/src/lib/snarky +++ b/src/lib/snarky @@ -1 +1 @@ -Subproject commit 94b2df82129658d505b612806a5804bc192f13f0 +Subproject commit 8cfd780d2119a77e5c99955faf8c6a2445e9397b From 8918371c1070fc9ea6651ffa51cf137ccc1cb74e Mon Sep 17 00:00:00 2001 From: mrmr1993 Date: Sun, 11 Feb 2024 11:26:05 +0000 Subject: [PATCH 02/45] WIP Pickles nearly compiling --- src/lib/pickles/cache.ml | 61 +-- src/lib/pickles/cache.mli | 29 +- src/lib/pickles/cache_handle.ml | 7 +- src/lib/pickles/cache_handle.mli | 4 +- src/lib/pickles/compile.ml | 149 ++++--- src/lib/pickles/fix_domains.ml | 9 +- src/lib/pickles/fix_domains.mli | 4 +- src/lib/pickles/inductive_rule.ml | 2 +- src/lib/pickles/inductive_rule.mli | 2 +- src/lib/pickles/pickles.ml | 238 ++++++----- src/lib/pickles/pickles_intf.mli | 4 +- src/lib/pickles/step.ml | 47 ++- src/lib/pickles/step_branch_data.ml | 3 +- src/lib/pickles/step_branch_data.mli | 2 + src/lib/pickles/step_main.ml | 105 +++-- src/lib/pickles/step_main.mli | 3 +- src/lib/pickles/types_map.ml | 26 +- src/lib/pickles/types_map.mli | 14 +- src/lib/pickles/wrap_domains.ml | 34 +- src/lib/pickles/wrap_domains.mli | 4 +- src/lib/pickles/wrap_main.ml | 594 ++++++++++++++------------- src/lib/pickles/wrap_main.mli | 5 +- 22 files changed, 764 insertions(+), 582 deletions(-) diff --git a/src/lib/pickles/cache.ml b/src/lib/pickles/cache.ml index 987513e73a9..33d2c8c061b 100644 --- a/src/lib/pickles/cache.ml +++ b/src/lib/pickles/cache.ml @@ -96,35 +96,45 @@ module Step = struct let open Impls.Step in let pk = lazy - ( match - Common.time "step keypair read" (fun () -> - Key_cache.Sync.read cache s_p (Lazy.force k_p) ) - with - | Ok (pk, dirty) -> - Common.time "step keypair create" (fun () -> (pk, dirty)) - | Error _e -> - let r = - Common.time "stepkeygen" (fun () -> - constraint_system ~input_typ:typ ~return_typ main - |> Keypair.generate ~prev_challenges ) - in - Timer.clock __LOC__ ; - ignore - ( Key_cache.Sync.write cache s_p (Lazy.force k_p) (Keypair.pk r) - : unit Or_error.t ) ; - (Keypair.pk r, `Generated_something) ) + (let%bind.Promise k_p = Lazy.force k_p in + match + Common.time "step keypair read" (fun () -> + Key_cache.Sync.read cache s_p k_p ) + with + | Ok (pk, dirty) -> + Common.time "step keypair create" (fun () -> + Promise.return (pk, dirty) ) + | Error _e -> + let%map.Promise r = + Common.time "stepkeygen" (fun () -> + let%map.Promise constraint_system = + let constraint_builder = + constraint_system_manual ~input_typ:typ ~return_typ + in + let%map.Promise res = + constraint_builder.run_circuit main + in + constraint_builder.finish_computation res + in + constraint_system |> Keypair.generate ~prev_challenges ) + in + Timer.clock __LOC__ ; + ignore + ( Key_cache.Sync.write cache s_p k_p (Keypair.pk r) + : unit Or_error.t ) ; + (Keypair.pk r, `Generated_something) ) in let vk = lazy - (let k_v = Lazy.force k_v in + (let%bind.Promise k_v = Lazy.force k_v in match Common.time "step vk read" (fun () -> Key_cache.Sync.read cache s_v k_v ) with | Ok (vk, _) -> - (vk, `Cache_hit) + Promise.return (vk, `Cache_hit) | Error _e -> - let pk, c = Lazy.force pk in + let%map.Promise pk, c = Lazy.force pk in let vk = Backend.Tick.Keypair.vk pk in ignore (Key_cache.Sync.write cache s_v k_v vk : unit Or_error.t) ; (vk, c) ) @@ -232,14 +242,15 @@ module Wrap = struct let open Impls.Wrap in let pk = lazy - (let k = Lazy.force k_p in + (let%bind.Promise k = Lazy.force k_p in match Common.time "wrap key read" (fun () -> Key_cache.Sync.read cache s_p k ) with | Ok (pk, d) -> - (pk, d) + Promise.return (pk, d) | Error _e -> + let%map.Promise main = Lazy.force main in let r = Common.time "wrapkeygen" (fun () -> constraint_system ~input_typ:typ ~return_typ main @@ -252,12 +263,12 @@ module Wrap = struct in let vk = lazy - (let k_v = Lazy.force k_v in + (let%bind.Promise k_v = Lazy.force k_v in match Key_cache.Sync.read cache s_v k_v with | Ok (vk, d) -> - (vk, d) + Promise.return (vk, d) | Error _e -> - let pk, _dirty = Lazy.force pk in + let%map.Promise pk, _dirty = Lazy.force pk in let vk = Backend.Tock.Keypair.vk pk in let vk : Vk.t = { index = vk diff --git a/src/lib/pickles/cache.mli b/src/lib/pickles/cache.mli index 75d3893597b..e16d7dc0a74 100644 --- a/src/lib/pickles/cache.mli +++ b/src/lib/pickles/cache.mli @@ -40,18 +40,21 @@ module Step : sig prev_challenges:int -> Key_cache.Spec.t list -> ?s_p:storable - -> Key.Proving.t lazy_t + -> Key.Proving.t Promise.t Lazy.t -> ?s_v:vk_storable - -> Key.Verification.t lazy_t + -> Key.Verification.t Promise.t Lazy.t -> ('a, 'b) Impls.Step.Typ.t -> ('c, 'd) Impls.Step.Typ.t - -> ('a -> unit -> 'c) + -> ('a -> unit -> 'c Promise.t) -> ( Impls.Step.Proving_key.t - * [> `Cache_hit | `Generated_something | `Locally_generated ] ) - lazy_t + * ([> `Cache_hit | `Generated_something | `Locally_generated ] as 'e) ) + Promise.t + Lazy.t * ( Kimchi_bindings.Protocol.VerifierIndex.Fp.t - * [> `Cache_hit | `Generated_something | `Locally_generated ] ) - lazy_t + * ([> `Cache_hit | `Generated_something | `Locally_generated ] as 'e) + ) + Promise.t + Lazy.t end module Wrap : sig @@ -93,16 +96,18 @@ module Wrap : sig prev_challenges:Core_kernel.Int.t -> Key_cache.Spec.t list -> ?s_p:storable - -> Key.Proving.t lazy_t + -> Key.Proving.t Promise.t Lazy.t -> ?s_v:vk_storable - -> Key.Verification.t lazy_t + -> Key.Verification.t Promise.t Lazy.t -> ('a, 'b) Impls.Wrap.Typ.t -> ('c, 'd) Impls.Wrap.Typ.t - -> ('a -> unit -> 'c) + -> ('a -> unit -> 'c) Promise.t Lazy.t -> ( Impls.Wrap.Proving_key.t * [> `Cache_hit | `Generated_something | `Locally_generated ] ) - lazy_t + Promise.t + Lazy.t * ( Verification_key.Stable.V2.t * [> `Cache_hit | `Generated_something | `Locally_generated ] ) - lazy_t + Promise.t + Lazy.t end diff --git a/src/lib/pickles/cache_handle.ml b/src/lib/pickles/cache_handle.ml index 393c41dcfc5..62ea0818881 100644 --- a/src/lib/pickles/cache_handle.ml +++ b/src/lib/pickles/cache_handle.ml @@ -1,5 +1,8 @@ -type t = Dirty.t Lazy.t +type t = Dirty.t Promise.t Lazy.t let generate_or_load (t : t) = Lazy.force t -let ( + ) (t1 : t) (t2 : t) : t = lazy Dirty.(Lazy.force t1 + Lazy.force t2) +let ( + ) (t1 : t) (t2 : t) : t = + lazy + (let%map.Promise t1 = Lazy.force t1 and t2 = Lazy.force t2 in + Dirty.(t1 + t2) ) diff --git a/src/lib/pickles/cache_handle.mli b/src/lib/pickles/cache_handle.mli index a9bedc2f26c..0aa58477e7e 100644 --- a/src/lib/pickles/cache_handle.mli +++ b/src/lib/pickles/cache_handle.mli @@ -1,9 +1,9 @@ (** Cache handle. It is currently used to cache proving and verifying keys for pickles *) -type t = Dirty.t lazy_t +type t = Dirty.t Promise.t Lazy.t (** [generate_or_load hdl] is an alias for [Lazy.force]. *) -val generate_or_load : t -> Dirty.t +val generate_or_load : t -> Dirty.t Promise.t (** [(+)] is semantically equivalent to {!Dirty.(+)}. *) val ( + ) : t -> t -> t diff --git a/src/lib/pickles/compile.ml b/src/lib/pickles/compile.ml index fc07598eb6b..7c159695229 100644 --- a/src/lib/pickles/compile.ml +++ b/src/lib/pickles/compile.ml @@ -122,6 +122,7 @@ type ('max_proofs_verified, 'branches, 'prev_varss) wrap_main_generic = Wrap_verifier.index' , 'branches ) Vector.t + Promise.t Lazy.t -> (int, 'branches) Pickles_types.Vector.t -> (Import.Domains.t, 'branches) Pickles_types.Vector.t @@ -153,6 +154,8 @@ type ('max_proofs_verified, 'branches, 'prev_varss) wrap_main_generic = , Impls.Wrap.Field.t ) Composition_types.Wrap.Statement.t -> unit ) + Promise.t + Lazy.t (** An override for wrap_main, which allows for adversarial testing with an 'invalid' pickles statement by passing a dummy proof. *) @@ -293,10 +296,8 @@ struct module Lazy_keys = struct type t = - (Impls.Step.Proving_key.t * Dirty.t) Lazy.t - * (Kimchi_bindings.Protocol.VerifierIndex.Fp.t * Dirty.t) Lazy.t - - (* TODO Think this is right.. *) + (Impls.Step.Proving_key.t * Dirty.t) Promise.t Lazy.t + * (Kimchi_bindings.Protocol.VerifierIndex.Fp.t * Dirty.t) Promise.t Lazy.t end let log_step main _typ name index = @@ -320,7 +321,7 @@ struct if profile_constraints then Snarky_log.to_file (sprintf "step-snark-%s-%d.json" name index) log - let log_wrap main typ name id = + let _log_wrap main typ name id = let module Constraints = Snarky_log.Constraints (Impls.Wrap.Internal_Basic) in let log = let sys = Backend.Tock.R1CS_constraint_system.create () in @@ -511,18 +512,19 @@ struct let module V = H4.To_vector (Int) in V.f prev_varss_length (M.f choices) in - let step_data = + let%bind.Promise step_data = let i = ref 0 in Timer.clock __LOC__ ; let rec f : type a b c d. - (a, b, c, d) H4.T(IR).t -> (a, b, c, d) H4.T(Branch_data).t = function + (a, b, c, d) H4.T(IR).t -> (a, b, c, d) H4.T(Branch_data).t Promise.t + = function | [] -> - [] + Promise.return ([] : _ H4.T(Branch_data).t) | rule :: rules -> - let first = + let%bind.Promise first = Timer.clock __LOC__ ; - let res = + let%map.Promise res = Common.time "make step data" (fun () -> Step_branch_data.create ~index:!i ~feature_flags ~actual_feature_flags:rule.feature_flags @@ -533,7 +535,8 @@ struct in Timer.clock __LOC__ ; incr i ; res in - first :: f rules + let%map.Promise rest = f rules in + (first :: rest : _ H4.T(Branch_data).t) in f choices in @@ -548,7 +551,7 @@ struct let module V = H4.To_vector (Domains) in V.f prev_varss_length (M.f step_data) in - let cache_handle = ref (Lazy.return `Cache_hit) in + let cache_handle = ref (Lazy.return (Promise.return `Cache_hit)) in let accum_dirty t = cache_handle := Cache_handle.(!cache_handle + t) in Timer.clock __LOC__ ; let step_keypairs = @@ -565,22 +568,25 @@ struct let f (T b : _ Branch_data.t) = let (T (typ, _conv, conv_inv)) = etyp in let main () () = - let res = b.main ~step_domains () in + let%map.Promise res = b.main ~step_domains () in Impls.Step.with_label "conv_inv" (fun () -> conv_inv res) in let () = if true then log_step main typ name b.index in let open Impls.Step in (* HACK: TODO docs *) - if return_early_digest_exception then - raise - (Return_digest - ( constraint_system ~input_typ:Typ.unit ~return_typ:typ main - |> R1CS_constraint_system.digest ) ) ; + if return_early_digest_exception then failwith "TODO: Delete me" ; let k_p = lazy - (let cs = - constraint_system ~input_typ:Typ.unit ~return_typ:typ main + (let%map.Promise cs = + let constraint_builder = + Impl.constraint_system_manual ~input_typ:Typ.unit + ~return_typ:typ + in + let%map.Promise res = + constraint_builder.run_circuit main + in + constraint_builder.finish_computation res in let cs_hash = Md5.to_hex (R1CS_constraint_system.digest cs) @@ -597,10 +603,12 @@ struct let k_v = match disk_keys with | Some ks -> - Lazy.return ks.(b.index) + Lazy.return (Promise.return ks.(b.index)) | None -> lazy - (let id, _header, index, cs = Lazy.force k_p in + (let%map.Promise id, _header, index, cs = + Lazy.force k_p + in let digest = R1CS_constraint_system.digest cs in ( id , snark_keys_header @@ -619,8 +627,8 @@ struct (Snarky_backendless.Typ.unit ()) typ main ) in - accum_dirty (Lazy.map pk ~f:snd) ; - accum_dirty (Lazy.map vk ~f:snd) ; + accum_dirty (Lazy.map pk ~f:(Promise.map ~f:snd)) ; + accum_dirty (Lazy.map vk ~f:(Promise.map ~f:snd)) ; res end) in @@ -630,8 +638,17 @@ struct let step_vks = let module V = H4.To_vector (Lazy_keys) in lazy - (Vector.map (V.f prev_varss_length step_keypairs) ~f:(fun (_, vk) -> - Tick.Keypair.full_vk_commitments (fst (Lazy.force vk)) ) ) + (let step_keypairs = V.f prev_varss_length step_keypairs in + let%map.Promise () = + (* Wait for keypair promises to resolve. *) + Vector.fold ~init:(Promise.return ()) step_keypairs + ~f:(fun acc (_, vk) -> + let%bind.Promise _ = Lazy.force vk in + acc ) + in + Vector.map step_keypairs ~f:(fun (_, vk) -> + Tick.Keypair.full_vk_commitments + (fst (Option.value_exn @@ Promise.peek @@ Lazy.force vk)) ) ) in Timer.clock __LOC__ ; let wrap_requests, wrap_main = @@ -657,12 +674,17 @@ struct let (wrap_pk, wrap_vk), disk_key = let open Impls.Wrap in let (T (typ, conv, _conv_inv)) = input ~feature_flags () in - let main x () : unit = wrap_main (conv x) in - let () = if true then log_wrap main typ name self.id in + let main = + lazy + (let%map.Promise wrap_main = Lazy.force wrap_main in + fun x () -> wrap_main (conv x) ) + in + (*let () = if true then log_wrap main typ name self.id in*) let self_id = Type_equal.Id.uid self.id in let disk_key_prover = lazy - (let cs = + (let%map.Promise main = Lazy.force main in + let cs = constraint_system ~input_typ:typ ~return_typ:(Snarky_backendless.Typ.unit ()) main @@ -678,7 +700,7 @@ struct match disk_keys with | None -> lazy - (let id, _header, cs = Lazy.force disk_key_prover in + (let%map.Promise id, _header, cs = Lazy.force disk_key_prover in let digest = R1CS_constraint_system.digest cs in ( id , snark_keys_header @@ -686,7 +708,7 @@ struct (Md5.to_hex digest) , digest ) ) | Some (_, (_id, header, digest)) -> - Lazy.return (self_id, header, digest) + Lazy.return @@ Promise.return (self_id, header, digest) in let r = Common.time "wrap read or generate " (fun () -> @@ -700,21 +722,28 @@ struct in Timer.clock __LOC__ ; let wrap_vk = - Lazy.map wrap_vk ~f:(fun ((wrap_vk, _) as res) -> - let computed_domain_size = wrap_vk.index.domain.log_size_of_group in - let (Pow_2_roots_of_unity proposed_domain_size) = wrap_domains.h in - if computed_domain_size <> proposed_domain_size then - failwithf - "This circuit was compiled for proofs using the wrap domain of \ - size %d, but the actual wrap domain size for the circuit has \ - size %d. You should pass the ~override_wrap_domain argument to \ - set the correct domain size." - proposed_domain_size computed_domain_size () ; - res ) + Lazy.map wrap_vk + ~f: + (Promise.map ~f:(fun ((wrap_vk, _) as res) -> + let computed_domain_size = + wrap_vk.Verification_key.index.domain.log_size_of_group + in + let (Pow_2_roots_of_unity proposed_domain_size) = + wrap_domains.h + in + if computed_domain_size <> proposed_domain_size then + failwithf + "This circuit was compiled for proofs using the wrap domain \ + of size %d, but the actual wrap domain size for the \ + circuit has size %d. You should pass the \ + ~override_wrap_domain argument to set the correct domain \ + size." + proposed_domain_size computed_domain_size () ; + res ) ) in - accum_dirty (Lazy.map wrap_pk ~f:snd) ; - accum_dirty (Lazy.map wrap_vk ~f:snd) ; - let wrap_vk = Lazy.map wrap_vk ~f:fst in + accum_dirty (Lazy.map wrap_pk ~f:(Promise.map ~f:snd)) ; + accum_dirty (Lazy.map wrap_vk ~f:(Promise.map ~f:snd)) ; + let wrap_vk = Lazy.map wrap_vk ~f:(Promise.map ~f:fst) in let module S = Step.Make (Arg_var) (Arg_value) (struct @@ -745,21 +774,21 @@ struct Promise.t = fun (T b as branch_data) (step_pk, step_vk) -> let _, prev_vars_length = b.proofs_verified in - let step handler next_state = - let wrap_vk = Lazy.force wrap_vk in + let step ~proof_cache ~maxes handler next_state = + let%bind.Promise wrap_vk = Lazy.force wrap_vk in + let%bind.Promise step_pk = Lazy.force step_pk in S.f ?handler branch_data next_state ~prevs_length:prev_vars_length ~self ~step_domains ~self_dlog_plonk_index: ((* TODO *) Plonk_verification_key_evals.map ~f:(fun x -> [| x |]) wrap_vk.commitments ) - ~public_input ~auxiliary_typ ~feature_flags - (fst (Lazy.force step_pk)) - wrap_vk.index + ~public_input ~auxiliary_typ ~feature_flags ~proof_cache ~maxes + (fst step_pk) wrap_vk.index in - let step_vk = fst (Lazy.force step_vk) in let wrap ?handler next_state = - let wrap_vk = Lazy.force wrap_vk in + let%bind.Promise step_vk, _ = Lazy.force step_vk in + let%bind.Promise wrap_vk = Lazy.force wrap_vk in let%bind.Promise ( proof , return_value , auxiliary_value @@ -793,6 +822,8 @@ struct *) Some tweak_statement in + let%bind.Promise wrap_main = Lazy.force wrap_main in + let%bind.Promise wrap_pk = Lazy.force wrap_pk in Wrap.wrap ~proof_cache ~max_proofs_verified:Max_proofs_verified.n ~feature_flags ~actual_feature_flags:b.feature_flags full_signature.maxes wrap_requests ?tweak_statement @@ -801,9 +832,7 @@ struct ~f:(fun x -> [| x |]) wrap_vk.commitments ) wrap_main ~typ ~step_vk ~step_plonk_indices:(Lazy.force step_vks) - ~actual_wrap_domains - (fst (Lazy.force wrap_pk)) - proof + ~actual_wrap_domains (fst wrap_pk) proof in ( return_value , auxiliary_value @@ -849,10 +878,12 @@ struct ; max_proofs_verified ; public_input = typ ; wrap_key = - Lazy.map wrap_vk ~f:(fun x -> - Plonk_verification_key_evals.map (Verification_key.commitments x) - ~f:(fun x -> [| x |]) ) - ; wrap_vk = Lazy.map wrap_vk ~f:Verification_key.index + Lazy.map wrap_vk + ~f: + (Promise.map ~f:(fun x -> + Plonk_verification_key_evals.map + (Verification_key.commitments x) ~f:(fun x -> [| x |]) ) ) + ; wrap_vk = Lazy.map wrap_vk ~f:(Promise.map ~f:Verification_key.index) ; wrap_domains ; step_domains ; feature_flags diff --git a/src/lib/pickles/fix_domains.ml b/src/lib/pickles/fix_domains.ml index 32348ea46e9..835243fa5f1 100644 --- a/src/lib/pickles/fix_domains.ml +++ b/src/lib/pickles/fix_domains.ml @@ -17,7 +17,7 @@ let domains (type field gates) ?feature_flags .t ) (Spec.ETyp.T (typ, conv, _conv_inv)) (Spec.ETyp.T (return_typ, _ret_conv, ret_conv_inv)) main = - let main x () = ret_conv_inv (main (conv x)) in + let main x () = Promise.map ~f:ret_conv_inv (main (conv x)) in let domains2 sys = let open Domain in @@ -77,4 +77,9 @@ let domains (type field gates) ?feature_flags Pow_2_roots_of_unity Int.(max lookup_table_length_log2 (ceil_log2 rows)) } in - domains2 (Impl.constraint_system ~input_typ:typ ~return_typ main) + let constraint_builder = + Impl.constraint_system_manual ~input_typ:typ ~return_typ + in + let%map.Promise res = constraint_builder.run_circuit main in + let constraint_system = constraint_builder.finish_computation res in + domains2 constraint_system diff --git a/src/lib/pickles/fix_domains.mli b/src/lib/pickles/fix_domains.mli index 0ec9373180e..ad21f6dff5e 100644 --- a/src/lib/pickles/fix_domains.mli +++ b/src/lib/pickles/fix_domains.mli @@ -14,7 +14,7 @@ val domains : .t ) -> ('a, 'b, 'field) Import.Spec.ETyp.t -> ('c, 'd, 'field) Import.Spec.ETyp.t - -> ('a -> 'c) - -> Import.Domains.t + -> ('a -> 'c Promise.t) + -> Import.Domains.t Promise.t val rough_domains : Import.Domains.t diff --git a/src/lib/pickles/inductive_rule.ml b/src/lib/pickles/inductive_rule.ml index b1ad99e9e30..4eacd777939 100644 --- a/src/lib/pickles/inductive_rule.ml +++ b/src/lib/pickles/inductive_rule.ml @@ -114,7 +114,7 @@ type ( 'prev_vars ; prevs : ('prev_vars, 'prev_values, 'widths, 'heights) H4.T(Tag).t ; main : 'a_var main_input - -> ('prev_vars, 'widths, 'ret_var, 'auxiliary_var) main_return + -> ('prev_vars, 'widths, 'ret_var, 'auxiliary_var) main_return Promise.t ; feature_flags : bool Pickles_types.Plonk_types.Features.t } diff --git a/src/lib/pickles/inductive_rule.mli b/src/lib/pickles/inductive_rule.mli index e787e3a16f3..9a56bf875f6 100644 --- a/src/lib/pickles/inductive_rule.mli +++ b/src/lib/pickles/inductive_rule.mli @@ -112,7 +112,7 @@ type ( 'prev_vars Pickles_types.Hlist.H4.T(Tag).t ; main : 'a_var main_input - -> ('prev_vars, 'widths, 'ret_var, 'auxiliary_var) main_return + -> ('prev_vars, 'widths, 'ret_var, 'auxiliary_var) main_return Promise.t ; feature_flags : bool Pickles_types.Plonk_types.Features.t } diff --git a/src/lib/pickles/pickles.ml b/src/lib/pickles/pickles.ml index 1a45b1f6924..34141778200 100644 --- a/src/lib/pickles/pickles.ml +++ b/src/lib/pickles/pickles.ml @@ -219,21 +219,25 @@ module Make_str (_ : Wire_types.Concrete) = struct let to_input (t : t) = to_input ~field_of_int:Impls.Step.Field.Constant.of_int t - let of_compiled tag : t = + let of_compiled_promise tag : t Promise.t = let d = Types_map.lookup_compiled tag.Tag.id in + let%bind.Promise wrap_key = Lazy.force d.wrap_key in + let%map.Promise wrap_vk = Lazy.force d.wrap_vk in let actual_wrap_domain_size = Common.actual_wrap_domain_size - ~log_2_domain_size:(Lazy.force d.wrap_vk).domain.log_size_of_group + ~log_2_domain_size:wrap_vk.domain.log_size_of_group in - { wrap_vk = Some (Lazy.force d.wrap_vk) - ; wrap_index = - Plonk_verification_key_evals.map (Lazy.force d.wrap_key) - ~f:(fun x -> x.(0)) - ; max_proofs_verified = - Pickles_base.Proofs_verified.of_nat - (Nat.Add.n d.max_proofs_verified) - ; actual_wrap_domain_size - } + ( { wrap_vk = Some wrap_vk + ; wrap_index = + Plonk_verification_key_evals.map wrap_key ~f:(fun x -> x.(0)) + ; max_proofs_verified = + Pickles_base.Proofs_verified.of_nat + (Nat.Add.n d.max_proofs_verified) + ; actual_wrap_domain_size + } + : t ) + + let of_compiled tag = of_compiled_promise tag |> Promise.to_deferred module Max_width = Width.Max end @@ -409,10 +413,11 @@ module Make_str (_ : Wire_types.Concrete) = struct (fun { public_input = self } -> dummy_constraints () ; Field.Assert.equal self Field.zero ; - { previous_proof_statements = [] - ; public_output = () - ; auxiliary_output = () - } ) + Promise.return + { Inductive_rule.previous_proof_statements = [] + ; public_output = () + ; auxiliary_output = () + } ) } ] ) ) @@ -459,10 +464,11 @@ module Make_str (_ : Wire_types.Concrete) = struct ; main = (fun _ -> dummy_constraints () ; - { previous_proof_statements = [] - ; public_output = Field.zero - ; auxiliary_output = () - } ) + Promise.return + { Inductive_rule.previous_proof_statements = [] + ; public_output = Field.zero + ; auxiliary_output = () + } ) } ] ) ) @@ -536,15 +542,16 @@ module Make_str (_ : Wire_types.Concrete) = struct let proof_must_verify = Boolean.not is_base_case in let self_correct = Field.(equal (one + prev) self) in Boolean.Assert.any [ self_correct; is_base_case ] ; - { previous_proof_statements = - [ { public_input = prev - ; proof - ; proof_must_verify - } - ] - ; public_output = () - ; auxiliary_output = () - } ) + Promise.return + { Inductive_rule.previous_proof_statements = + [ { public_input = prev + ; proof + ; proof_must_verify + } + ] + ; public_output = () + ; auxiliary_output = () + } ) } ] ) ) @@ -653,19 +660,20 @@ module Make_str (_ : Wire_types.Concrete) = struct let proof_must_verify = Boolean.not is_base_case in let self_correct = Field.(equal (one + prev) self) in Boolean.Assert.any [ self_correct; is_base_case ] ; - { previous_proof_statements = - [ { public_input = no_recursive_input - ; proof = no_recursive_proof - ; proof_must_verify = Boolean.true_ - } - ; { public_input = prev - ; proof = prev_proof - ; proof_must_verify - } - ] - ; public_output = () - ; auxiliary_output = () - } ) + Promise.return + { Inductive_rule.previous_proof_statements = + [ { public_input = no_recursive_input + ; proof = no_recursive_proof + ; proof_must_verify = Boolean.true_ + } + ; { public_input = prev + ; proof = prev_proof + ; proof_must_verify + } + ] + ; public_output = () + ; auxiliary_output = () + } ) } ] ) ) @@ -789,19 +797,20 @@ module Make_str (_ : Wire_types.Concrete) = struct Field.( if_ is_base_case ~then_:zero ~else_:(one + prev)) in - { previous_proof_statements = - [ { public_input = no_recursive_input - ; proof = no_recursive_proof - ; proof_must_verify = Boolean.true_ - } - ; { public_input = prev - ; proof = prev_proof - ; proof_must_verify - } - ] - ; public_output = self - ; auxiliary_output = () - } ) + Promise.return + { Inductive_rule.previous_proof_statements = + [ { public_input = no_recursive_input + ; proof = no_recursive_proof + ; proof_must_verify = Boolean.true_ + } + ; { public_input = prev + ; proof = prev_proof + ; proof_must_verify + } + ] + ; public_output = self + ; auxiliary_output = () + } ) } ] ) ) @@ -878,10 +887,11 @@ module Make_str (_ : Wire_types.Concrete) = struct ; main = (fun { public_input = x } -> dummy_constraints () ; - { previous_proof_statements = [] - ; public_output = Field.(add one) x - ; auxiliary_output = () - } ) + Promise.return + { Inductive_rule.previous_proof_statements = [] + ; public_output = Field.(add one) x + ; auxiliary_output = () + } ) } ] ) ) @@ -942,10 +952,11 @@ module Make_str (_ : Wire_types.Concrete) = struct Step_main_inputs.Sponge.absorb sponge (`Field blinding_value) ; let result = Step_main_inputs.Sponge.squeeze sponge in - { previous_proof_statements = [] - ; public_output = result - ; auxiliary_output = blinding_value - } ) + Promise.return + { Inductive_rule.previous_proof_statements = [] + ; public_output = result + ; auxiliary_output = blinding_value + } ) } ] ) ) @@ -1020,19 +1031,20 @@ module Make_str (_ : Wire_types.Concrete) = struct As_prover.Ref.create (fun () -> Proof0.dummy Nat.N2.n Nat.N2.n Nat.N2.n ~domain_log2:15 ) in - { previous_proof_statements = - [ { public_input = () - ; proof = dummy_proof - ; proof_must_verify = Boolean.false_ - } - ; { public_input = () - ; proof = dummy_proof - ; proof_must_verify = Boolean.false_ - } - ] - ; public_output = () - ; auxiliary_output = () - } ) + Promise.return + { Inductive_rule.previous_proof_statements = + [ { public_input = () + ; proof = dummy_proof + ; proof_must_verify = Boolean.false_ + } + ; { public_input = () + ; proof = dummy_proof + ; proof_must_verify = Boolean.false_ + } + ] + ; public_output = () + ; auxiliary_output = () + } ) ; feature_flags = Plonk_types.Features.none_bool } @@ -1078,8 +1090,9 @@ module Make_str (_ : Wire_types.Concrete) = struct module Lazy_keys = struct type t = - (Impls.Step.Proving_key.t * Dirty.t) Lazy.t - * (Kimchi_bindings.Protocol.VerifierIndex.Fp.t * Dirty.t) Lazy.t + (Impls.Step.Proving_key.t * Dirty.t) Promise.t Lazy.t + * (Kimchi_bindings.Protocol.VerifierIndex.Fp.t * Dirty.t) Promise.t + Lazy.t (* TODO Think this is right.. *) end @@ -1146,7 +1159,7 @@ module Make_str (_ : Wire_types.Concrete) = struct Step_branch_data.t end in let proofs_verifieds = Vector.singleton 2 in - let (T inner_step_data as step_data) = + let%bind.Promise (T inner_step_data as step_data) = Step_branch_data.create ~index:0 ~feature_flags ~actual_feature_flags ~max_proofs_verified:Max_proofs_verified.n ~branches:Branches.n ~self ~public_input:(Input typ) @@ -1161,14 +1174,19 @@ module Make_str (_ : Wire_types.Concrete) = struct in let (T (typ, _conv, conv_inv)) = etyp in let main () () = - let res = inner_step_data.main ~step_domains () in + let%map.Promise res = inner_step_data.main ~step_domains () in Impls.Step.with_label "conv_inv" (fun () -> conv_inv res) in let open Impls.Step in let k_p = lazy - (let cs = - constraint_system ~input_typ:Typ.unit ~return_typ:typ main + (let%map.Promise cs = + let constraint_builder = + Impl.constraint_system_manual ~input_typ:Typ.unit + ~return_typ:typ + in + let%map.Promise res = constraint_builder.run_circuit main in + constraint_builder.finish_computation res in let cs_hash = Md5.to_hex (R1CS_constraint_system.digest cs) in ( Type_equal.Id.uid self.id @@ -1182,7 +1200,7 @@ module Make_str (_ : Wire_types.Concrete) = struct in let k_v = lazy - (let id, _header, index, cs = Lazy.force k_p in + (let%map.Promise id, _header, index, cs = Lazy.force k_p in let digest = R1CS_constraint_system.digest cs in ( id , snark_keys_header @@ -1202,8 +1220,10 @@ module Make_str (_ : Wire_types.Concrete) = struct in let step_vks = lazy - (Vector.map [ step_keypair ] ~f:(fun (_, vk) -> - Tick.Keypair.vk_commitments (fst (Lazy.force vk)) ) ) + (let%map.Promise step_vk = Lazy.force (snd step_keypair) in + Vector.map + [ (fst step_keypair, step_vk) ] + ~f:(fun (_, vk) -> Tick.Keypair.vk_commitments (fst vk)) ) in let wrap_main _ = let module SC' = SC in @@ -1268,11 +1288,14 @@ module Make_str (_ : Wire_types.Concrete) = struct let r = Common.time "wrap read or generate " (fun () -> Cache.Wrap.read_or_generate ~prev_challenges:2 [] - disk_key_prover disk_key_verifier typ Typ.unit main ) + (Lazy.map ~f:Promise.return disk_key_prover) + (Lazy.map ~f:Promise.return disk_key_verifier) + typ Typ.unit + (lazy (Promise.return main)) ) in (r, disk_key_verifier) in - let wrap_vk = Lazy.map wrap_vk ~f:fst in + let wrap_vk = Lazy.map wrap_vk ~f:(Promise.map ~f:fst) in let module S = Step.Make (A) (A_value) (Max_proofs_verified) in let prover = let f : @@ -1290,24 +1313,23 @@ module Make_str (_ : Wire_types.Concrete) = struct Requests.Wrap.create () in let _, prev_vars_length = b.proofs_verified in - let step = - let wrap_vk = Lazy.force wrap_vk in - S.f branch_data () ~feature_flags ~prevs_length:prev_vars_length - ~self ~public_input:(Input typ) + let step () = + let%bind.Promise step_pk = Lazy.force step_pk in + let%bind.Promise wrap_vk = Lazy.force wrap_vk in + S.f branch_data ~feature_flags ~prevs_length:prev_vars_length + ~self ~public_input:(Input typ) ~proof_cache:None + ~maxes:(module Maxes) ~auxiliary_typ:Impls.Step.Typ.unit ~step_domains ~self_dlog_plonk_index: ((* TODO *) Plonk_verification_key_evals.map ~f:(fun x -> [| x |]) wrap_vk.commitments ) - (fst (Lazy.force step_pk)) - wrap_vk.index + () (fst step_pk) wrap_vk.index in - let pairing_vk = fst (Lazy.force step_vk) in + let%bind.Promise pairing_vk, _ = Lazy.force step_vk in let wrap = let wrap_vk = Lazy.force wrap_vk in - let%bind.Promise proof, (), (), _ = - step ~proof_cache:None ~maxes:(module Maxes) - in + let%bind.Promise proof, (), (), _ = step () in let proof = { proof with statement = @@ -1460,7 +1482,7 @@ module Make_str (_ : Wire_types.Concrete) = struct public_input proof in let x_hat = O.(p_eval_1 o, p_eval_2 o) in - let step_vk, _ = Lazy.force step_vk in + let%bind.Promise step_vk, _ = Lazy.force step_vk in let next_statement : _ Types.Wrap.Statement.In_circuit.t = let scalar_chal f = Scalar_challenge.map ~f:Challenge.Constant.of_tick_field @@ -1778,6 +1800,8 @@ module Make_str (_ : Wire_types.Concrete) = struct } : _ P.Base.Wrap.t ) in + let%bind.Promise wrap_pk = Lazy.force wrap_pk in + let%bind.Promise wrap_vk = wrap_vk in wrap ~max_proofs_verified:Max_proofs_verified.n full_signature.maxes ~dlog_plonk_index: @@ -1787,8 +1811,7 @@ module Make_str (_ : Wire_types.Concrete) = struct wrap_main A_value.to_field_elements ~pairing_vk ~step_domains:b.domains ~pairing_plonk_indices:(Lazy.force step_vks) ~wrap_domains - (fst (Lazy.force wrap_pk)) - proof + (fst wrap_pk) proof in Proof.T { proof with @@ -1812,18 +1835,21 @@ module Make_str (_ : Wire_types.Concrete) = struct ; max_proofs_verified = (module Max_proofs_verified) ; public_input = typ ; wrap_key = - Lazy.map wrap_vk ~f:(fun x -> - (* TODO *) - Plonk_verification_key_evals.map - ~f:(fun x -> [| x |]) - (Verification_key.commitments x) ) - ; wrap_vk = Lazy.map wrap_vk ~f:Verification_key.index + Lazy.map wrap_vk + ~f: + (Promise.map ~f:(fun x -> + (* TODO *) + Plonk_verification_key_evals.map + ~f:(fun x -> [| x |]) + (Verification_key.commitments x) ) ) + ; wrap_vk = + Lazy.map wrap_vk ~f:(Promise.map ~f:Verification_key.index) ; wrap_domains ; step_domains } in Types_map.add_exn self data ; - (prover, wrap_vk, disk_key) + Promise.return (prover, wrap_vk, disk_key) end let step, wrap_vk, wrap_disk_key = M.compile diff --git a/src/lib/pickles/pickles_intf.mli b/src/lib/pickles/pickles_intf.mli index 83bac67504d..f3bcb90aab0 100644 --- a/src/lib/pickles/pickles_intf.mli +++ b/src/lib/pickles/pickles_intf.mli @@ -307,7 +307,9 @@ module type S = sig val typ : (Checked.t, t) Impls.Step.Typ.t - val of_compiled : _ Tag.t -> t + val of_compiled_promise : _ Tag.t -> t Promise.t + + val of_compiled : _ Tag.t -> t Deferred.t module Max_branches : Nat.Add.Intf diff --git a/src/lib/pickles/step.ml b/src/lib/pickles/step.ml index f1f6590524c..733c5fce009 100644 --- a/src/lib/pickles/step.ml +++ b/src/lib/pickles/step.ml @@ -124,7 +124,7 @@ struct -> _ array Plonk_verification_key_evals.t -> value -> (local_max_proofs_verified, local_max_proofs_verified) Proof.t - -> (var, value, local_max_proofs_verified, m) Tag.t + -> (var, value, local_max_proofs_verified, m) Types_map.Basic.t -> must_verify:bool -> [ `Sg of Tock.Curve.Affine.t ] * Unfinalized.Constant.t @@ -135,7 +135,7 @@ struct , m ) Per_proof_witness.Constant.No_app_state.t * [ `Actual_wrap_domain of int ] = - fun dlog_vk dlog_index app_state (T t) tag ~must_verify -> + fun dlog_vk dlog_index app_state (T t) data ~must_verify -> let t = { t with statement = @@ -146,7 +146,6 @@ struct } in let proof = Wrap_wire_proof.to_kimchi_proof t.proof in - let data = Types_map.lookup_basic tag in let plonk0 = t.statement.proof_state.deferred_values.plonk in let plonk = let domain = @@ -535,6 +534,20 @@ struct , witness , `Actual_wrap_domain dlog_vk.domain.log_size_of_group ) in + let%bind.Promise prevs = + let rec go : + type vars values ns ms. + (vars, values, ns, ms) H4.T(Tag).t + -> (vars, values, ns, ms) H4.T(Types_map.Basic).t Promise.t = function + | [] -> + Promise.return ([] : _ H4.T(Types_map.Basic).t) + | tag :: tags -> + let%bind.Promise data = Types_map.lookup_basic tag in + let%map.Promise rest = go tags in + (data :: rest : _ H4.T(Types_map.Basic).t) + in + go branch_data.rule.prevs + in let challenge_polynomial_commitments = ref None in let unfinalized_proofs = ref None in let statements_with_hashes = ref None in @@ -555,6 +568,7 @@ struct let[@warning "-4"] rec go : type vars values ns ms k. (vars, values, ns, ms) H4.T(Tag).t + -> (vars, values, ns, ms) H4.T(Types_map.Basic).t -> ( values , ns ) H2.T(Inductive_rule.Previous_proof_statement.Constant).t @@ -569,11 +583,12 @@ struct H3.T(Per_proof_witness.Constant.No_app_state).t * (ns, ns) H2.T(Proof).t * (int, k) Vector.t = - fun ts prev_proof_stmts l -> - match (ts, prev_proof_stmts, l) with - | [], [], Z -> + fun ts datas prev_proof_stmts l -> + match (ts, datas, prev_proof_stmts, l) with + | [], [], [], Z -> ([], [], [], [], [], [], []) | ( t :: ts + , data :: datas , { public_input = app_state ; proof = p ; proof_must_verify = must_verify @@ -583,13 +598,13 @@ struct let dlog_vk, dlog_index = if Type_equal.Id.same self.Tag.id t.id then (self_dlog_vk, self_dlog_plonk_index) - else - let d = Types_map.lookup_basic t in - (d.wrap_vk, d.wrap_key) + else (data.wrap_vk, data.wrap_key) in let `Sg sg, u, s, x, w, `Actual_wrap_domain domain = - expand_proof dlog_vk dlog_index app_state p t ~must_verify - and sgs, us, ss, xs, ws, ps, domains = go ts prev_proof_stmts l in + expand_proof dlog_vk dlog_index app_state p data ~must_verify + and sgs, us, ss, xs, ws, ps, domains = + go ts datas prev_proof_stmts l + in ( sg :: sgs , u :: us , s :: ss @@ -597,12 +612,12 @@ struct , w :: ws , p :: ps , domain :: domains ) - | _, _ :: _, _ -> + | _, _, _ :: _, _ -> . - | _, [], _ -> + | _, _, [], _ -> . in - go branch_data.rule.prevs prev_proof_requests prev_vars_length + go branch_data.rule.prevs prevs prev_proof_requests prev_vars_length in challenge_polynomial_commitments := Some challenge_polynomial_commitments' ; unfinalized_proofs := Some unfinalized_proofs' ; @@ -797,9 +812,9 @@ struct Impls.Step.generate_witness_manual ~handlers:[ handler ] ~input_typ:Impls.Step.Typ.unit ~return_typ:input () in - let res = + let%bind.Promise res = builder.run_circuit (fun () () -> - conv_inv (branch_data.main ~step_domains ()) ) + Promise.map ~f:conv_inv (branch_data.main ~step_domains ()) ) in let ( { Impls.Step.Proof_inputs.auxiliary_inputs; public_inputs } , next_statement_hashed ) = diff --git a/src/lib/pickles/step_branch_data.ml b/src/lib/pickles/step_branch_data.ml index d06021a365d..496a52ca552 100644 --- a/src/lib/pickles/step_branch_data.ml +++ b/src/lib/pickles/step_branch_data.ml @@ -42,6 +42,7 @@ type ( 'a_var , Impls.Step.Field.t , (Impls.Step.Field.t, 'max_proofs_verified) Vector.t ) Types.Step.Statement.t + Promise.t ; requests : (module Requests.Step.S with type statement = 'a_value @@ -148,7 +149,7 @@ let create |> unstage in Timer.clock __LOC__ ; - let own_domains = + let%map.Promise own_domains = let main = step ~step_domains: diff --git a/src/lib/pickles/step_branch_data.mli b/src/lib/pickles/step_branch_data.mli index 8fef1b7a732..8713efa9728 100644 --- a/src/lib/pickles/step_branch_data.mli +++ b/src/lib/pickles/step_branch_data.mli @@ -45,6 +45,7 @@ type ( 'a_var , (Impls.Step.Field.t, 'max_proofs_verified) Pickles_types.Vector.t ) Import.Types.Step.Statement.t + Promise.t ; requests : (module Requests.Step.S with type auxiliary_value = 'auxiliary_value @@ -123,3 +124,4 @@ val create : , 'g , 'h ) t + Promise.t diff --git a/src/lib/pickles/step_main.ml b/src/lib/pickles/step_main.ml index d5313005dca..7d92441bb12 100644 --- a/src/lib/pickles/step_main.ml +++ b/src/lib/pickles/step_main.ml @@ -176,7 +176,8 @@ let step_main : -> ( (Unfinalized.t, max_proofs_verified) Vector.t , Field.t , (Field.t, max_proofs_verified) Vector.t ) - Types.Step.Statement.t ) + Types.Step.Statement.t + Promise.t ) Staged.t = fun (module Req) max_proofs_verified ~self_branches ~local_signature ~local_signature_length ~local_branches ~local_branches_length @@ -266,24 +267,63 @@ let step_main : | Input_and_output (input_typ, output_typ) -> (input_typ, output_typ) in - let main () : _ Types.Step.Statement.t = + let main () : _ Types.Step.Statement.t Promise.t = let open Impls.Step in let logger = Internal_tracing_context_logger.get () in + let module Max_proofs_verified = ( val max_proofs_verified : Nat.Add.Intf + with type n = max_proofs_verified ) + in + let T = Max_proofs_verified.eq in + let app_state = exists input_typ ~request:(fun () -> Req.App_state) in + let module Optional_wrap_key = struct + type (_, _, _, _) t = + Step_main_inputs.Inner_curve.Constant.t array + Plonk_verification_key_evals.t + option + end in + (* Here, we prefetch the known wrap keys for all compiled rules. + These keys may resolve asynchronously due to key generation for other + pickles rules, but we want to preserve the single-threaded behavior of + pickles to maximize our chanes of successful debugging. + Hence, we preload here, and pass the values in as needed when we create + [datas] below. + *) + let%bind.Promise known_wrap_keys = + let rec go : + type a1 a2 n m. + (a1, a2, n, m) H4.T(Tag).t + -> (a1, a2, n, m) H4.T(Optional_wrap_key).t Promise.t = function + | [] -> + Promise.return ([] : _ H4.T(Optional_wrap_key).t) + | tag :: tags -> + let%bind.Promise opt_wrap_key = + match Type_equal.Id.same_witness self.id tag.id with + | Some T -> + Promise.return None + | None -> ( + match tag.kind with + | Compiled -> + let%map.Promise wrap_key = + Lazy.force + @@ (Types_map.lookup_compiled tag.id).wrap_key + in + Some wrap_key + | Side_loaded -> + Promise.return None ) + in + let%map.Promise rest = go tags in + (opt_wrap_key :: rest : _ H4.T(Optional_wrap_key).t) + in + go rule.prevs + in + let%map.Promise { Inductive_rule.previous_proof_statements + ; public_output = ret_var + ; auxiliary_output = auxiliary_var + } = + (* Run the application logic of the rule on the predecessor statements *) + with_label "rule_main" (fun () -> rule.main { public_input = app_state }) + in with_label "step_main" (fun () -> - let module Max_proofs_verified = ( val max_proofs_verified : Nat.Add.Intf - with type n = max_proofs_verified - ) - in - let T = Max_proofs_verified.eq in - let app_state = exists input_typ ~request:(fun () -> Req.App_state) in - let { Inductive_rule.previous_proof_statements - ; public_output = ret_var - ; auxiliary_output = auxiliary_var - } = - (* Run the application logic of the rule on the predecessor statements *) - with_label "rule_main" (fun () -> - rule.main { public_input = app_state } ) - in let () = exists Typ.unit ~request:(fun () -> let ret_value = As_prover.read output_typ ret_var in @@ -487,28 +527,35 @@ let step_main : ; feature_flags = basic.feature_flags } in - let module M = - H4.Map (Tag) (Types_map.For_step) - (struct - let f : - type a1 a2 n m. - (a1, a2, n, m) Tag.t - -> (a1, a2, n, m) Types_map.For_step.t = - fun tag -> + let rec go : + type a1 a2 n m. + (a1, a2, n, m) H4.T(Tag).t + -> (a1, a2, n, m) H4.T(Optional_wrap_key).t + -> (a1, a2, n, m) H4.T(Types_map.For_step).t = + fun tags optional_wrap_keys -> + match (tags, optional_wrap_keys) with + | [], [] -> + [] + | tag :: tags, optional_wrap_key :: optional_wrap_keys -> + let data = match Type_equal.Id.same_witness self.id tag.id with - | Some T -> - self_data | None -> ( match tag.kind with | Compiled -> - Types_map.For_step.of_compiled + Types_map.For_step + .of_compiled_with_known_wrap_key + ~wrap_key: + (Option.value_exn optional_wrap_key) (Types_map.lookup_compiled tag.id) | Side_loaded -> Types_map.For_step.of_side_loaded (Types_map.lookup_side_loaded tag.id) ) - end) + | Some T -> + self_data + in + data :: go tags optional_wrap_keys in - M.f rule.prevs + go rule.prevs known_wrap_keys in go prevs datas messages_for_next_wrap_proofs unfinalized_proofs previous_proof_statements proofs_verified ~actual_wrap_domains diff --git a/src/lib/pickles/step_main.mli b/src/lib/pickles/step_main.mli index 93d9b2bd847..faec66ceaec 100644 --- a/src/lib/pickles/step_main.mli +++ b/src/lib/pickles/step_main.mli @@ -55,5 +55,6 @@ val step_main : -> ( (Unfinalized.t, 'max_proofs_verified) Pickles_types.Vector.t , Impls.Step.Field.t , (Impls.Step.Field.t, 'max_proofs_verified) Pickles_types.Vector.t ) - Import.Types.Step.Statement.t ) + Import.Types.Step.Statement.t + Promise.t ) Core_kernel.Staged.t diff --git a/src/lib/pickles/types_map.ml b/src/lib/pickles/types_map.ml index 3496edd08f3..71886be392d 100644 --- a/src/lib/pickles/types_map.ml +++ b/src/lib/pickles/types_map.ml @@ -99,8 +99,9 @@ module Compiled = struct (* For each branch in this rule, how many predecessor proofs does it have? *) ; public_input : ('a_var, 'a_value) Impls.Step.Typ.t ; wrap_key : - Tick.Inner_curve.Affine.t array Plonk_verification_key_evals.t Lazy.t - ; wrap_vk : Impls.Wrap.Verification_key.t Lazy.t + Tick.Inner_curve.Affine.t array Plonk_verification_key_evals.t Promise.t + Lazy.t + ; wrap_vk : Impls.Wrap.Verification_key.t Promise.t Lazy.t ; wrap_domains : Domains.t ; step_domains : (Domains.t, 'branches) Vector.t ; feature_flags : Opt.Flag.t Plonk_types.Features.Full.t @@ -120,12 +121,14 @@ module Compiled = struct ; wrap_key ; feature_flags } = + let%bind.Promise wrap_key = Lazy.force wrap_key in + let%map.Promise wrap_vk = Lazy.force wrap_vk in { Basic.max_proofs_verified ; wrap_domains ; public_input ; branches = Vector.length step_domains - ; wrap_key = Lazy.force wrap_key - ; wrap_vk = Lazy.force wrap_vk + ; wrap_key + ; wrap_vk ; feature_flags } end @@ -174,12 +177,12 @@ module For_step = struct ; feature_flags } - let of_compiled + let of_compiled_with_known_wrap_key ~wrap_key ({ branches ; max_proofs_verified ; proofs_verifieds ; public_input - ; wrap_key + ; wrap_key = _ ; wrap_domains ; step_domains ; feature_flags @@ -192,12 +195,16 @@ module For_step = struct `Known (Vector.map proofs_verifieds ~f:Impls.Step.Field.of_int) ; public_input ; wrap_key = - Plonk_verification_key_evals.map (Lazy.force wrap_key) + Plonk_verification_key_evals.map wrap_key ~f:(Array.map ~f:Step_main_inputs.Inner_curve.constant) ; wrap_domain = `Known wrap_domains.h ; step_domains = `Known step_domains ; feature_flags } + + let of_compiled ({ wrap_key; _ } as t : _ Compiled.t) = + let%map.Promise wrap_key = Lazy.force wrap_key in + of_compiled_with_known_wrap_key ~wrap_key t end type t = @@ -230,13 +237,14 @@ let lookup_side_loaded : d let lookup_basic : - type var value n m. (var, value, n, m) Tag.t -> (var, value, n, m) Basic.t = + type var value n m. + (var, value, n, m) Tag.t -> (var, value, n, m) Basic.t Promise.t = fun t -> match t.kind with | Compiled -> Compiled.to_basic (lookup_compiled t.id) | Side_loaded -> - Side_loaded.to_basic (lookup_side_loaded t.id) + Promise.return @@ Side_loaded.to_basic (lookup_side_loaded t.id) let max_proofs_verified : type n1. (_, _, n1, _) Tag.t -> (module Nat.Add.Intf with type n = n1) = diff --git a/src/lib/pickles/types_map.mli b/src/lib/pickles/types_map.mli index 7377fce3206..d8711ed5338 100644 --- a/src/lib/pickles/types_map.mli +++ b/src/lib/pickles/types_map.mli @@ -71,8 +71,9 @@ module Compiled : sig ; wrap_key : Backend.Tick.Inner_curve.Affine.t array Pickles_types.Plonk_verification_key_evals.t + Promise.t Lazy.t - ; wrap_vk : Impls.Wrap.Verification_key.t Lazy.t + ; wrap_vk : Impls.Wrap.Verification_key.t Promise.t Lazy.t ; wrap_domains : Import.Domains.t ; step_domains : (Import.Domains.t, 'branches) Pickles_types.Vector.t ; feature_flags : Opt.Flag.t Plonk_types.Features.Full.t @@ -102,7 +103,14 @@ module For_step : sig val of_side_loaded : ('a, 'b, 'c, 'd) Side_loaded.t -> ('a, 'b, 'c, 'd) t - val of_compiled : ('a, 'b, 'c, 'd) Compiled.t -> ('a, 'b, 'c, 'd) t + val of_compiled_with_known_wrap_key : + wrap_key: + Step_main_inputs.Inner_curve.Constant.t array + Plonk_verification_key_evals.t + -> ('a, 'b, 'c, 'd) Compiled.t + -> ('a, 'b, 'c, 'd) t + + val of_compiled : ('a, 'b, 'c, 'd) Compiled.t -> ('a, 'b, 'c, 'd) t Promise.t end type t @@ -116,7 +124,7 @@ val lookup_side_loaded : ('var, 'value, 'n, 'm) Tag.id -> ('var, 'value, 'n, 'm) Side_loaded.t val lookup_basic : - ('var, 'value, 'n, 'm) Tag.t -> ('var, 'value, 'n, 'm) Basic.t + ('var, 'value, 'n, 'm) Tag.t -> ('var, 'value, 'n, 'm) Basic.t Promise.t val add_side_loaded : name:string diff --git a/src/lib/pickles/wrap_domains.ml b/src/lib/pickles/wrap_domains.ml index e8d27e2c20c..1229ef498ff 100644 --- a/src/lib/pickles/wrap_domains.ml +++ b/src/lib/pickles/wrap_domains.ml @@ -1,6 +1,5 @@ open Core_kernel open Pickles_types -open Import open Poly_types (* Compute the domains corresponding to wrap_main *) @@ -27,13 +26,14 @@ struct in let dummy_step_keys = lazy - (Vector.init num_choices ~f:(fun _ -> - let num_chunks = (* TODO *) 1 in - let g = - Array.init num_chunks ~f:(fun _ -> - Backend.Tock.Inner_curve.(to_affine_exn one) ) - in - Verification_key.dummy_step_commitments g ) ) + (Promise.return + (Vector.init num_choices ~f:(fun _ -> + let num_chunks = (* TODO *) 1 in + let g = + Array.init num_chunks ~f:(fun _ -> + Backend.Tock.Inner_curve.(to_affine_exn one) ) + in + Verification_key.dummy_step_commitments g ) ) ) in Timer.clock __LOC__ ; let srs = Backend.Tick.Keypair.load_urs () in @@ -42,12 +42,13 @@ struct dummy_step_keys dummy_step_widths dummy_step_domains max_proofs_verified in Timer.clock __LOC__ ; + let%bind.Promise main = Lazy.force main in let t = Fix_domains.domains (module Impls.Wrap) (Impls.Wrap.input ~feature_flags ()) (T (Snarky_backendless.Typ.unit (), Fn.id, Fn.id)) - main + (fun input -> Promise.return (main input)) in Timer.clock __LOC__ ; t @@ -57,12 +58,15 @@ struct Common.wrap_domains ~proofs_verified:(Nat.to_int (Nat.Add.n max_proofs_verified)) in - ( if debug then - let res' = - f_debug full_signature num_choices choices_length ~feature_flags - ~max_proofs_verified - in - [%test_eq: Domains.t] res res' ) ; + (*let%map.Promise () = + if debug then + let%map.Promise res' = + f_debug full_signature num_choices choices_length ~feature_flags + ~max_proofs_verified + in + [%test_eq: Domains.t] res res' + else Promise.return () + in*) res end [@@warning "-60"] diff --git a/src/lib/pickles/wrap_domains.mli b/src/lib/pickles/wrap_domains.mli index bf0ddbf4c66..b89d2d4de71 100644 --- a/src/lib/pickles/wrap_domains.mli +++ b/src/lib/pickles/wrap_domains.mli @@ -15,7 +15,7 @@ module Make -> ('e, 'b) Hlist.Length.t -> feature_flags:Opt.Flag.t Plonk_types.Features.Full.t -> max_proofs_verified:(module Nat.Add.Intf with type n = 'a) - -> Import.Domains.t + -> Import.Domains.t Promise.t val f : ('a, 'b, 'c) Full_signature.t @@ -23,6 +23,6 @@ module Make -> ('e, 'b) Hlist.Length.t -> feature_flags:Opt.Flag.t Plonk_types.Features.Full.t -> max_proofs_verified:(module Nat.Add.Intf with type n = 'a) - -> Import.Domains.Stable.V2.t + -> Import.Domains.t end [@@warning "-67"] diff --git a/src/lib/pickles/wrap_main.ml b/src/lib/pickles/wrap_main.ml index ccdfbe2b8d2..5c612dc2439 100644 --- a/src/lib/pickles/wrap_main.ml +++ b/src/lib/pickles/wrap_main.ml @@ -95,6 +95,7 @@ let wrap_main Wrap_verifier.index' , branches ) Vector.t + Promise.t Lazy.t ) (step_widths : (int, branches) Vector.t) (step_domains : (Domains.t, branches) Vector.t) ~srs (max_proofs_verified : @@ -112,7 +113,9 @@ let wrap_main , _ , _ ) Types.Wrap.Statement.In_circuit.t - -> unit ) = + -> unit ) + Promise.t + Lazy.t = Timer.clock __LOC__ ; let module Max_proofs_verified = ( val max_proofs_verified : Nat.Add.Intf with type n = max_proofs_verified ) @@ -129,314 +132,321 @@ let wrap_main full_signature in Timer.clock __LOC__ ; - let main - ({ proof_state = - { deferred_values = - { plonk - ; xi - ; combined_inner_product - ; b - ; branch_data - ; bulletproof_challenges - } - ; sponge_digest_before_evaluations - ; messages_for_next_wrap_proof = messages_for_next_wrap_proof_digest - } - ; messages_for_next_step_proof = _ - } : - ( _ - , _ - , _ Shifted_value.Type1.t - , _ - , _ - , _ - , _ - , _ - , _ - , _ - , Field.t ) - Types.Wrap.Statement.In_circuit.t ) = - let logger = Internal_tracing_context_logger.get () in - with_label __LOC__ (fun () -> - let which_branch' = - exists - (Typ.transport Field.typ ~there:Field.Constant.of_int - ~back:(fun _ -> failwith "unimplemented") ) - ~request:(fun () -> Req.Which_branch) - in - let which_branch = - Wrap_verifier.One_hot_vector.of_index which_branch' ~length:branches - in - let actual_proofs_verified_mask = - Util.ones_vector - (module Impl) - ~first_zero: - (Wrap_verifier.Pseudo.choose - (which_branch, step_widths) - ~f:Field.of_int ) - Max_proofs_verified.n - |> Vector.rev - in - let domain_log2 = - Wrap_verifier.Pseudo.choose - ( which_branch - , Vector.map ~f:(fun ds -> Domain.log2_size ds.h) step_domains ) - ~f:Field.of_int - in - let () = - with_label __LOC__ (fun () -> - (* Check that the branch_data public-input is correct *) - Branch_data.Checked.pack - (module Impl) - { proofs_verified_mask = - Vector.extend_front_exn actual_proofs_verified_mask Nat.N2.n - Boolean.false_ - ; domain_log2 - } - |> Field.Assert.equal branch_data ) - in - let prev_proof_state = - with_label __LOC__ (fun () -> - let open Types.Step.Proof_state in - let typ = - typ + let main = + let%map.Lazy step_keys = step_keys in + let%map.Promise step_keys = step_keys in + fun ({ proof_state = + { deferred_values = + { plonk + ; xi + ; combined_inner_product + ; b + ; branch_data + ; bulletproof_challenges + } + ; sponge_digest_before_evaluations + ; messages_for_next_wrap_proof = + messages_for_next_wrap_proof_digest + } + ; messages_for_next_step_proof = _ + } : + ( _ + , _ + , _ Shifted_value.Type1.t + , _ + , _ + , _ + , _ + , _ + , _ + , _ + , Field.t ) + Types.Wrap.Statement.In_circuit.t ) -> + let logger = Internal_tracing_context_logger.get () in + with_label __LOC__ (fun () -> + let which_branch' = + exists + (Typ.transport Field.typ ~there:Field.Constant.of_int + ~back:(fun _ -> failwith "unimplemented") ) + ~request:(fun () -> Req.Which_branch) + in + let which_branch = + Wrap_verifier.One_hot_vector.of_index which_branch' ~length:branches + in + let actual_proofs_verified_mask = + Util.ones_vector + (module Impl) + ~first_zero: + (Wrap_verifier.Pseudo.choose + (which_branch, step_widths) + ~f:Field.of_int ) + Max_proofs_verified.n + |> Vector.rev + in + let domain_log2 = + Wrap_verifier.Pseudo.choose + ( which_branch + , Vector.map ~f:(fun ds -> Domain.log2_size ds.h) step_domains ) + ~f:Field.of_int + in + let () = + with_label __LOC__ (fun () -> + (* Check that the branch_data public-input is correct *) + Branch_data.Checked.pack (module Impl) - ~assert_16_bits:(Wrap_verifier.assert_n_bits ~n:16) - (Vector.init Max_proofs_verified.n ~f:(fun _ -> - Plonk_types.Features.none ) ) - (Shifted_value.Type2.typ Field.typ) - in - exists typ ~request:(fun () -> Req.Proof_state) ) - in - let step_plonk_index = - with_label __LOC__ (fun () -> - Wrap_verifier.choose_key which_branch - (Vector.map (Lazy.force step_keys) - ~f: - (Plonk_verification_key_evals.Step.map - ~f:(Array.map ~f:Inner_curve.constant) ~f_opt:(function - | None -> - Opt.nothing - | Some x -> - Opt.just (Array.map ~f:Inner_curve.constant x) ) ) ) ) - in - let prev_step_accs = - with_label __LOC__ (fun () -> - exists (Vector.typ Inner_curve.typ Max_proofs_verified.n) - ~request:(fun () -> Req.Step_accs) ) - in - let old_bp_chals = - with_label __LOC__ (fun () -> - let typ = - let module T = - H1.Typ (Impls.Wrap) (Nat) (Challenges_vector) - (Challenges_vector.Constant) + { proofs_verified_mask = + Vector.extend_front_exn actual_proofs_verified_mask + Nat.N2.n Boolean.false_ + ; domain_log2 + } + |> Field.Assert.equal branch_data ) + in + let prev_proof_state = + with_label __LOC__ (fun () -> + let open Types.Step.Proof_state in + let typ = + typ + (module Impl) + ~assert_16_bits:(Wrap_verifier.assert_n_bits ~n:16) + (Vector.init Max_proofs_verified.n ~f:(fun _ -> + Plonk_types.Features.none ) ) + (Shifted_value.Type2.typ Field.typ) + in + exists typ ~request:(fun () -> Req.Proof_state) ) + in + let step_plonk_index = + with_label __LOC__ (fun () -> + Wrap_verifier.choose_key which_branch + (Vector.map step_keys + ~f: + (Plonk_verification_key_evals.Step.map + ~f:(Array.map ~f:Inner_curve.constant) + ~f_opt:(function + | None -> + Opt.nothing + | Some x -> + Opt.just (Array.map ~f:Inner_curve.constant x) ) ) ) ) + in + let prev_step_accs = + with_label __LOC__ (fun () -> + exists (Vector.typ Inner_curve.typ Max_proofs_verified.n) + ~request:(fun () -> Req.Step_accs) ) + in + let old_bp_chals = + with_label __LOC__ (fun () -> + let typ = + let module T = + H1.Typ (Impls.Wrap) (Nat) (Challenges_vector) + (Challenges_vector.Constant) + (struct + let f (type n) (n : n Nat.t) = + Vector.typ + (Vector.typ Field.typ Backend.Tock.Rounds.n) + n + end) + in + T.f Max_widths_by_slot.maxes + in + let module Z = H1.Zip (Nat) (Challenges_vector) in + let module M = + H1.Map + (H1.Tuple2 (Nat) (Challenges_vector)) + (E01 (Old_bulletproof_chals)) (struct - let f (type n) (n : n Nat.t) = - Vector.typ - (Vector.typ Field.typ Backend.Tock.Rounds.n) - n + let f (type n) + ((n, v) : n H1.Tuple2(Nat)(Challenges_vector).t) = + Old_bulletproof_chals.T (n, v) end) in - T.f Max_widths_by_slot.maxes - in - let module Z = H1.Zip (Nat) (Challenges_vector) in - let module M = - H1.Map - (H1.Tuple2 (Nat) (Challenges_vector)) - (E01 (Old_bulletproof_chals)) - (struct - let f (type n) - ((n, v) : n H1.Tuple2(Nat)(Challenges_vector).t) = - Old_bulletproof_chals.T (n, v) - end) - in - let module V = H1.To_vector (Old_bulletproof_chals) in - Z.f Max_widths_by_slot.maxes - (exists typ ~request:(fun () -> Req.Old_bulletproof_challenges)) - |> M.f - |> V.f Max_widths_by_slot.length ) - in - let new_bulletproof_challenges = - with_label __LOC__ (fun () -> - let evals = - let ty = + let module V = H1.To_vector (Old_bulletproof_chals) in + Z.f Max_widths_by_slot.maxes + (exists typ ~request:(fun () -> + Req.Old_bulletproof_challenges ) ) + |> M.f + |> V.f Max_widths_by_slot.length ) + in + let new_bulletproof_challenges = + with_label __LOC__ (fun () -> + let evals = let ty = - Plonk_types.All_evals.typ (module Impl) feature_flags + let ty = + Plonk_types.All_evals.typ (module Impl) feature_flags + in + Vector.typ ty Max_proofs_verified.n in - Vector.typ ty Max_proofs_verified.n + exists ty ~request:(fun () -> Req.Evals) in - exists ty ~request:(fun () -> Req.Evals) - in - let chals = - let wrap_domains = - let all_possible_domains = - Wrap_verifier.all_possible_domains () - in - let wrap_domain_indices = - exists (Vector.typ Field.typ Max_proofs_verified.n) - ~request:(fun () -> Req.Wrap_domain_indices) + let chals = + let wrap_domains = + let all_possible_domains = + Wrap_verifier.all_possible_domains () + in + let wrap_domain_indices = + exists (Vector.typ Field.typ Max_proofs_verified.n) + ~request:(fun () -> Req.Wrap_domain_indices) + in + Vector.map wrap_domain_indices ~f:(fun index -> + let which_branch = + Wrap_verifier.One_hot_vector.of_index index + ~length:Wrap_verifier.num_possible_domains + in + Wrap_verifier.Pseudo.Domain.to_domain ~shifts + ~domain_generator + (which_branch, all_possible_domains) ) in - Vector.map wrap_domain_indices ~f:(fun index -> - let which_branch = - Wrap_verifier.One_hot_vector.of_index index - ~length:Wrap_verifier.num_possible_domains + Vector.mapn + [ (* This is padded to max_proofs_verified for the benefit of wrapping with dummy unfinalized proofs *) + prev_proof_state.unfinalized_proofs + ; old_bp_chals + ; evals + ; wrap_domains + ] + ~f:(fun + [ { deferred_values + ; sponge_digest_before_evaluations + ; should_finalize + } + ; old_bulletproof_challenges + ; evals + ; wrap_domain + ] + -> + let sponge = + let s = Sponge.create sponge_params in + Sponge.absorb s sponge_digest_before_evaluations ; + s in - Wrap_verifier.Pseudo.Domain.to_domain ~shifts - ~domain_generator - (which_branch, all_possible_domains) ) - in - Vector.mapn - [ (* This is padded to max_proofs_verified for the benefit of wrapping with dummy unfinalized proofs *) - prev_proof_state.unfinalized_proofs - ; old_bp_chals - ; evals - ; wrap_domains - ] - ~f:(fun - [ { deferred_values - ; sponge_digest_before_evaluations - ; should_finalize - } - ; old_bulletproof_challenges - ; evals - ; wrap_domain - ] - -> - let sponge = - let s = Sponge.create sponge_params in - Sponge.absorb s sponge_digest_before_evaluations ; - s - in - (* the type of the local max proofs-verified depends on - which kind of step proof we are wrapping. *) - (* For each i in [0..max_proofs_verified-1], we have - max_local_max_proofs_verified, which is the largest - Local_max_proofs_verified which is the i^th inner proof of a step proof. + (* the type of the local max proofs-verified depends on + which kind of step proof we are wrapping. *) + (* For each i in [0..max_proofs_verified-1], we have + max_local_max_proofs_verified, which is the largest + Local_max_proofs_verified which is the i^th inner proof of a step proof. - Need to compute this value from the which_branch. - *) - let (T - ( _max_local_max_proofs_verified - , old_bulletproof_challenges ) ) = - old_bulletproof_challenges - in - let old_bulletproof_challenges = - Wrap_hack.Checked.pad_challenges + Need to compute this value from the which_branch. + *) + let (T + ( _max_local_max_proofs_verified + , old_bulletproof_challenges ) ) = old_bulletproof_challenges - in - let finalized, chals = - with_label __LOC__ (fun () -> - Wrap_verifier.finalize_other_proof - (module Wrap_hack.Padded_length) - ~domain:(wrap_domain :> _ Plonk_checks.plonk_domain) - ~sponge ~old_bulletproof_challenges deferred_values - evals ) - in - Boolean.(Assert.any [ finalized; not should_finalize ]) ; - chals ) - in - chals ) - in - let prev_statement = - let prev_messages_for_next_wrap_proof = - Vector.map2 prev_step_accs old_bp_chals - ~f:(fun sacc (T (max_local_max_proofs_verified, chals)) -> - Wrap_hack.Checked.hash_messages_for_next_wrap_proof - max_local_max_proofs_verified - { challenge_polynomial_commitment = sacc - ; old_bulletproof_challenges = chals - } ) + in + let old_bulletproof_challenges = + Wrap_hack.Checked.pad_challenges + old_bulletproof_challenges + in + let finalized, chals = + with_label __LOC__ (fun () -> + Wrap_verifier.finalize_other_proof + (module Wrap_hack.Padded_length) + ~domain: + (wrap_domain :> _ Plonk_checks.plonk_domain) + ~sponge ~old_bulletproof_challenges + deferred_values evals ) + in + Boolean.(Assert.any [ finalized; not should_finalize ]) ; + chals ) + in + chals ) + in + let prev_statement = + let prev_messages_for_next_wrap_proof = + Vector.map2 prev_step_accs old_bp_chals + ~f:(fun sacc (T (max_local_max_proofs_verified, chals)) -> + Wrap_hack.Checked.hash_messages_for_next_wrap_proof + max_local_max_proofs_verified + { challenge_polynomial_commitment = sacc + ; old_bulletproof_challenges = chals + } ) + in + { Types.Step.Statement.messages_for_next_wrap_proof = + prev_messages_for_next_wrap_proof + ; proof_state = prev_proof_state + } in - { Types.Step.Statement.messages_for_next_wrap_proof = - prev_messages_for_next_wrap_proof - ; proof_state = prev_proof_state - } - in - let openings_proof = - let shift = Shifts.tick1 in - exists - (Plonk_types.Openings.Bulletproof.typ - ( Typ.transport Wrap_verifier.Other_field.Packed.typ - ~there:(fun x -> - (* When storing, make it a shifted value *) - match - Shifted_value.Type1.of_field + let openings_proof = + let shift = Shifts.tick1 in + exists + (Plonk_types.Openings.Bulletproof.typ + ( Typ.transport Wrap_verifier.Other_field.Packed.typ + ~there:(fun x -> + (* When storing, make it a shifted value *) + match + Shifted_value.Type1.of_field + (module Backend.Tick.Field) + ~shift x + with + | Shifted_value x -> + x ) + ~back:(fun x -> + Shifted_value.Type1.to_field (module Backend.Tick.Field) - ~shift x - with - | Shifted_value x -> - x ) - ~back:(fun x -> - Shifted_value.Type1.to_field - (module Backend.Tick.Field) - ~shift (Shifted_value x) ) - (* When reading, unshift *) - |> Typ.transport_var - (* For the var, we just wrap the now shifted underlying value. *) - ~there:(fun (Shifted_value.Type1.Shifted_value x) -> x) - ~back:(fun x -> Shifted_value x) ) - Inner_curve.typ - ~length:(Nat.to_int Backend.Tick.Rounds.n) ) - ~request:(fun () -> Req.Openings_proof) - in - let ( sponge_digest_before_evaluations_actual - , (`Success bulletproof_success, bulletproof_challenges_actual) ) = - let messages = + ~shift (Shifted_value x) ) + (* When reading, unshift *) + |> Typ.transport_var + (* For the var, we just wrap the now shifted underlying value. *) + ~there:(fun (Shifted_value.Type1.Shifted_value x) -> x) + ~back:(fun x -> Shifted_value x) ) + Inner_curve.typ + ~length:(Nat.to_int Backend.Tick.Rounds.n) ) + ~request:(fun () -> Req.Openings_proof) + in + let ( sponge_digest_before_evaluations_actual + , (`Success bulletproof_success, bulletproof_challenges_actual) ) + = + let messages = + with_label __LOC__ (fun () -> + exists + (Plonk_types.Messages.typ + (module Impl) + Inner_curve.typ ~bool:Boolean.typ feature_flags + ~dummy:Inner_curve.Params.one + ~commitment_lengths:Commitment_lengths.default ) + ~request:(fun () -> Req.Messages) ) + in + let sponge = Wrap_verifier.Opt.create sponge_params in with_label __LOC__ (fun () -> - exists - (Plonk_types.Messages.typ - (module Impl) - Inner_curve.typ ~bool:Boolean.typ feature_flags - ~dummy:Inner_curve.Params.one - ~commitment_lengths:Commitment_lengths.default ) - ~request:(fun () -> Req.Messages) ) + [%log internal] "Wrap_verifier_incrementally_verify_proof" ; + let res = + Wrap_verifier.incrementally_verify_proof max_proofs_verified + ~actual_proofs_verified_mask ~step_domains + ~verification_key:step_plonk_index ~srs ~xi ~sponge + ~public_input: + (Array.map + (pack_statement Max_proofs_verified.n prev_statement) + ~f:(function + | `Field (Shifted_value x) -> + `Field (split_field x) + | `Packed_bits (x, n) -> + `Packed_bits (x, n) ) ) + ~sg_old:prev_step_accs + ~advice:{ b; combined_inner_product } + ~messages ~which_branch ~openings_proof ~plonk + in + [%log internal] "Wrap_verifier_incrementally_verify_proof_done" ; + res ) in - let sponge = Wrap_verifier.Opt.create sponge_params in with_label __LOC__ (fun () -> - [%log internal] "Wrap_verifier_incrementally_verify_proof" ; - let res = - Wrap_verifier.incrementally_verify_proof max_proofs_verified - ~actual_proofs_verified_mask ~step_domains - ~verification_key:step_plonk_index ~srs ~xi ~sponge - ~public_input: - (Array.map - (pack_statement Max_proofs_verified.n prev_statement) - ~f:(function - | `Field (Shifted_value x) -> - `Field (split_field x) - | `Packed_bits (x, n) -> - `Packed_bits (x, n) ) ) - ~sg_old:prev_step_accs - ~advice:{ b; combined_inner_product } - ~messages ~which_branch ~openings_proof ~plonk - in - [%log internal] "Wrap_verifier_incrementally_verify_proof_done" ; - res ) - in - with_label __LOC__ (fun () -> - Boolean.Assert.is_true bulletproof_success ) ; - with_label __LOC__ (fun () -> - Field.Assert.equal messages_for_next_wrap_proof_digest - (Wrap_hack.Checked.hash_messages_for_next_wrap_proof - Max_proofs_verified.n - { Types.Wrap.Proof_state.Messages_for_next_wrap_proof - .challenge_polynomial_commitment = - openings_proof.challenge_polynomial_commitment - ; old_bulletproof_challenges = new_bulletproof_challenges - } ) ) ; - with_label __LOC__ (fun () -> - Field.Assert.equal sponge_digest_before_evaluations - sponge_digest_before_evaluations_actual ) ; - Array.iter2_exn bulletproof_challenges_actual - (Vector.to_array bulletproof_challenges) - ~f:(fun - { prechallenge = { inner = x1 } } - ({ prechallenge = { inner = x2 } } : - _ SC.t Bulletproof_challenge.t ) - -> with_label __LOC__ (fun () -> Field.Assert.equal x1 x2) ) ; - () ) + Boolean.Assert.is_true bulletproof_success ) ; + with_label __LOC__ (fun () -> + Field.Assert.equal messages_for_next_wrap_proof_digest + (Wrap_hack.Checked.hash_messages_for_next_wrap_proof + Max_proofs_verified.n + { Types.Wrap.Proof_state.Messages_for_next_wrap_proof + .challenge_polynomial_commitment = + openings_proof.challenge_polynomial_commitment + ; old_bulletproof_challenges = new_bulletproof_challenges + } ) ) ; + with_label __LOC__ (fun () -> + Field.Assert.equal sponge_digest_before_evaluations + sponge_digest_before_evaluations_actual ) ; + Array.iter2_exn bulletproof_challenges_actual + (Vector.to_array bulletproof_challenges) + ~f:(fun + { prechallenge = { inner = x1 } } + ({ prechallenge = { inner = x2 } } : + _ SC.t Bulletproof_challenge.t ) + -> with_label __LOC__ (fun () -> Field.Assert.equal x1 x2) ) ; + () ) in Timer.clock __LOC__ ; ((module Req), main) diff --git a/src/lib/pickles/wrap_main.mli b/src/lib/pickles/wrap_main.mli index 04d903002bc..6cc4e3a7867 100644 --- a/src/lib/pickles/wrap_main.mli +++ b/src/lib/pickles/wrap_main.mli @@ -14,7 +14,8 @@ val wrap_main : Wrap_verifier.index' , 'branches ) Pickles_types.Vector.t - Core_kernel.Lazy.t + Promise.t + Lazy.t -> (int, 'branches) Pickles_types.Vector.t -> (Import.Domains.t, 'branches) Pickles_types.Vector.t -> srs:Kimchi_bindings.Protocol.SRS.Fp.t @@ -42,3 +43,5 @@ val wrap_main : , Wrap_main_inputs.Impl.Field.t ) Import.Types.Wrap.Statement.In_circuit.t -> unit ) + Promise.t + Lazy.t From d975e6184a5d3f5389457cc342481129fde7355f Mon Sep 17 00:00:00 2001 From: mrmr1993 Date: Sun, 11 Feb 2024 13:53:17 +0000 Subject: [PATCH 03/45] Pickles compiles! --- src/lib/pickles/compile.ml | 124 +++++++++++--------- src/lib/pickles/compile.mli | 22 +++- src/lib/pickles/pickles.ml | 169 ++++++++++++++++----------- src/lib/pickles/pickles_intf.mli | 11 +- src/lib/pickles/step.ml | 4 +- src/lib/pickles/step.mli | 3 +- src/lib/pickles/step_branch_data.ml | 32 ++++- src/lib/pickles/step_branch_data.mli | 6 +- src/lib/pickles/step_main.ml | 33 ++++-- src/lib/pickles/types_map.ml | 22 +++- src/lib/pickles/types_map.mli | 8 +- src/lib/pickles/wrap_domains.ml | 3 +- src/lib/pickles/wrap_main.ml | 14 ++- src/lib/pickles/wrap_main.mli | 2 +- 14 files changed, 288 insertions(+), 165 deletions(-) diff --git a/src/lib/pickles/compile.ml b/src/lib/pickles/compile.ml index 7c159695229..12f160b4373 100644 --- a/src/lib/pickles/compile.ml +++ b/src/lib/pickles/compile.ml @@ -90,9 +90,13 @@ module type Proof_intf = sig type t - val verification_key : Verification_key.t Lazy.t + val verification_key_promise : Verification_key.t Promise.t Lazy.t - val id : Cache.Wrap.Key.Verification.t Lazy.t + val verification_key : Verification_key.t Deferred.t Lazy.t + + val id_promise : Cache.Wrap.Key.Verification.t Promise.t Lazy.t + + val id : Cache.Wrap.Key.Verification.t Deferred.t Lazy.t val verify : (statement * t) list -> unit Or_error.t Deferred.t @@ -125,7 +129,7 @@ type ('max_proofs_verified, 'branches, 'prev_varss) wrap_main_generic = Promise.t Lazy.t -> (int, 'branches) Pickles_types.Vector.t - -> (Import.Domains.t, 'branches) Pickles_types.Vector.t + -> (Import.Domains.t Promise.t, 'branches) Pickles_types.Vector.t -> (module Pickles_types.Nat.Add.Intf with type n = 'max_proofs_verified) -> ('max_proofs_verified, 'max_local_max_proofs_verifieds) Requests.Wrap.t * ( ( ( Impls.Wrap.Field.t @@ -512,19 +516,18 @@ struct let module V = H4.To_vector (Int) in V.f prev_varss_length (M.f choices) in - let%bind.Promise step_data = + let step_data = let i = ref 0 in Timer.clock __LOC__ ; let rec f : type a b c d. - (a, b, c, d) H4.T(IR).t -> (a, b, c, d) H4.T(Branch_data).t Promise.t - = function + (a, b, c, d) H4.T(IR).t -> (a, b, c, d) H4.T(Branch_data).t = function | [] -> - Promise.return ([] : _ H4.T(Branch_data).t) + [] | rule :: rules -> - let%bind.Promise first = + let first = Timer.clock __LOC__ ; - let%map.Promise res = + let res = Common.time "make step data" (fun () -> Step_branch_data.create ~index:!i ~feature_flags ~actual_feature_flags:rule.feature_flags @@ -535,20 +538,22 @@ struct in Timer.clock __LOC__ ; incr i ; res in - let%map.Promise rest = f rules in - (first :: rest : _ H4.T(Branch_data).t) + first :: f rules in f choices in Timer.clock __LOC__ ; let step_domains = + let module DomainsPromise = struct + type t = Domains.t Promise.t + end in let module M = - H4.Map (Branch_data) (E04 (Domains)) + H4.Map (Branch_data) (E04 (DomainsPromise)) (struct let f (T b : _ Branch_data.t) = b.domains end) in - let module V = H4.To_vector (Domains) in + let module V = H4.To_vector (DomainsPromise) in V.f prev_varss_length (M.f step_data) in let cache_handle = ref (Lazy.return (Promise.return `Cache_hit)) in @@ -903,20 +908,25 @@ module Side_loaded = struct let to_input (t : t) = to_input ~field_of_int:Impls.Step.Field.Constant.of_int t - let of_compiled tag : t = + let of_compiled_promise tag : t Promise.t = let d = Types_map.lookup_compiled tag.Tag.id in + let%bind.Promise wrap_key = Lazy.force d.wrap_key in + let%map.Promise wrap_vk = Lazy.force d.wrap_vk in let actual_wrap_domain_size = Common.actual_wrap_domain_size - ~log_2_domain_size:(Lazy.force d.wrap_vk).domain.log_size_of_group + ~log_2_domain_size:wrap_vk.domain.log_size_of_group in - { wrap_vk = Some (Lazy.force d.wrap_vk) - ; wrap_index = - Plonk_verification_key_evals.map (Lazy.force d.wrap_key) ~f:(fun x -> - x.(0) ) - ; max_proofs_verified = - Pickles_base.Proofs_verified.of_nat (Nat.Add.n d.max_proofs_verified) - ; actual_wrap_domain_size - } + ( { wrap_vk = Some wrap_vk + ; wrap_index = + Plonk_verification_key_evals.map wrap_key ~f:(fun x -> x.(0)) + ; max_proofs_verified = + Pickles_base.Proofs_verified.of_nat + (Nat.Add.n d.max_proofs_verified) + ; actual_wrap_domain_size + } + : t ) + + let of_compiled tag = of_compiled_promise tag |> Promise.to_deferred module Max_width = Width.Max end @@ -1152,18 +1162,22 @@ let compile_with_wrap_main_override_promise : include Max_local_max_proofs_verified end) - let id = wrap_disk_key + let id_promise = wrap_disk_key - let verification_key = wrap_vk + let id = Lazy.map ~f:Promise.to_deferred wrap_disk_key + + let verification_key_promise = wrap_vk + + let verification_key = Lazy.map ~f:Promise.to_deferred wrap_vk let verify_promise ts = + let%bind.Promise verification_key = Lazy.force verification_key_promise in verify_promise ( module struct include Max_proofs_verified end ) (module Value) - (Lazy.force verification_key) - ts + verification_key ts let verify ts = verify_promise ts |> Promise.to_deferred end in @@ -1217,7 +1231,7 @@ let wrap_main_dummy_override _ _ _ _ _ _ _ = assert_r1cs x y z done in - (requests, wrap_main) + (requests, Lazy.return @@ Promise.return @@ wrap_main) module Make_adversarial_test (M : sig val tweak_statement : @@ -1309,19 +1323,20 @@ struct As_prover.Ref.create (fun () -> Proof.dummy Nat.N2.n Nat.N2.n Nat.N2.n ~domain_log2:15 ) in - { previous_proof_statements = - [ { public_input = () - ; proof = dummy_proof - ; proof_must_verify = Boolean.false_ - } - ; { public_input = () - ; proof = dummy_proof - ; proof_must_verify = Boolean.false_ - } - ] - ; public_output = () - ; auxiliary_output = () - } ) + Promise.return + { Inductive_rule.previous_proof_statements = + [ { public_input = () + ; proof = dummy_proof + ; proof_must_verify = Boolean.false_ + } + ; { public_input = () + ; proof = dummy_proof + ; proof_must_verify = Boolean.false_ + } + ] + ; public_output = () + ; auxiliary_output = () + } ) ; feature_flags = Plonk_types.Features.none_bool } @@ -1397,19 +1412,20 @@ struct let proof = exists (Typ.Internal.ref ()) ~request:(fun () -> Proof) in - { previous_proof_statements = - [ { public_input = () - ; proof - ; proof_must_verify = Boolean.true_ - } - ; { public_input = () - ; proof - ; proof_must_verify = Boolean.true_ - } - ] - ; public_output = () - ; auxiliary_output = () - } ) + Promise.return + { Inductive_rule.previous_proof_statements = + [ { public_input = () + ; proof + ; proof_must_verify = Boolean.true_ + } + ; { public_input = () + ; proof + ; proof_must_verify = Boolean.true_ + } + ] + ; public_output = () + ; auxiliary_output = () + } ) } ] ) ) diff --git a/src/lib/pickles/compile.mli b/src/lib/pickles/compile.mli index 0e6a145f804..5682d7e2830 100644 --- a/src/lib/pickles/compile.mli +++ b/src/lib/pickles/compile.mli @@ -35,9 +35,13 @@ module type Proof_intf = sig type t - val verification_key : Verification_key.t Lazy.t + val verification_key_promise : Verification_key.t Promise.t Lazy.t - val id : Cache.Wrap.Key.Verification.t Lazy.t + val verification_key : Verification_key.t Deferred.t Lazy.t + + val id_promise : Cache.Wrap.Key.Verification.t Promise.t Lazy.t + + val id : Cache.Wrap.Key.Verification.t Deferred.t Lazy.t val verify : (statement * t) list -> unit Or_error.t Deferred.t @@ -87,7 +91,9 @@ module Side_loaded : sig val typ : (Checked.t, t) Impls.Step.Typ.t - val of_compiled : _ Tag.t -> t + val of_compiled_promise : _ Tag.t -> t Promise.t + + val of_compiled : _ Tag.t -> t Deferred.t module Max_branches : Nat.Add.Intf @@ -159,9 +165,10 @@ type ('max_proofs_verified, 'branches, 'prev_varss) wrap_main_generic = Wrap_verifier.index' , 'branches ) Vector.t + Promise.t Lazy.t -> (int, 'branches) Pickles_types.Vector.t - -> (Import.Domains.t, 'branches) Pickles_types.Vector.t + -> (Import.Domains.t Promise.t, 'branches) Pickles_types.Vector.t -> (module Pickles_types.Nat.Add.Intf with type n = 'max_proofs_verified) -> ('max_proofs_verified, 'max_local_max_proofs_verifieds) Requests.Wrap.t * ( ( ( Impls.Wrap.Field.t @@ -190,6 +197,8 @@ type ('max_proofs_verified, 'branches, 'prev_varss) wrap_main_generic = , Impls.Wrap.Field.t ) Composition_types.Wrap.Statement.t -> unit ) + Promise.t + Lazy.t (** An override for wrap_main, which allows for adversarial testing with an 'invalid' pickles statement by passing a dummy proof. *) @@ -344,9 +353,10 @@ val wrap_main_dummy_override : Wrap_verifier.index' , 'branches ) Vector.t + Promise.t Lazy.t -> (int, 'branches) Pickles_types.Vector.t - -> (Import.Domains.t, 'branches) Pickles_types.Vector.t + -> (Import.Domains.t Promise.t, 'branches) Pickles_types.Vector.t -> (module Pickles_types.Nat.Add.Intf with type n = 'max_proofs_verified) -> ('max_proofs_verified, 'max_local_max_proofs_verifieds) Requests.Wrap.t * ( ( ( Impls.Wrap.Field.t @@ -374,6 +384,8 @@ val wrap_main_dummy_override : , Impls.Wrap.Field.t ) Composition_types.Wrap.Statement.t -> unit ) + Promise.t + Lazy.t module Make_adversarial_test : functor (_ : sig diff --git a/src/lib/pickles/pickles.ml b/src/lib/pickles/pickles.ml index 34141778200..f9208ddb8fd 100644 --- a/src/lib/pickles/pickles.ml +++ b/src/lib/pickles/pickles.ml @@ -1159,7 +1159,7 @@ module Make_str (_ : Wire_types.Concrete) = struct Step_branch_data.t end in let proofs_verifieds = Vector.singleton 2 in - let%bind.Promise (T inner_step_data as step_data) = + let (T inner_step_data as step_data) = Step_branch_data.create ~index:0 ~feature_flags ~actual_feature_flags ~max_proofs_verified:Max_proofs_verified.n ~branches:Branches.n ~self ~public_input:(Input typ) @@ -1849,7 +1849,7 @@ module Make_str (_ : Wire_types.Concrete) = struct } in Types_map.add_exn self data ; - Promise.return (prover, wrap_vk, disk_key) + (prover, wrap_vk, disk_key) end let step, wrap_vk, wrap_disk_key = M.compile @@ -1863,11 +1863,11 @@ module Make_str (_ : Wire_types.Concrete) = struct let verification_key = wrap_vk let verify ts = + let%bind.Promise verification_key = Lazy.force verification_key in verify_promise (module Max_proofs_verified) (module A_value) - (Lazy.force verification_key) - ts + verification_key ts let _statement (T p : t) = p.statement.messages_for_next_step_proof.app_state @@ -1916,19 +1916,20 @@ module Make_str (_ : Wire_types.Concrete) = struct exists (Typ.Internal.ref ()) ~request:(fun () -> Proof ) in - { previous_proof_statements = - [ { public_input = () - ; proof - ; proof_must_verify = Boolean.true_ - } - ; { public_input = () - ; proof - ; proof_must_verify = Boolean.true_ - } - ] - ; public_output = () - ; auxiliary_output = () - } ) + Promise.return + { Inductive_rule.previous_proof_statements = + [ { public_input = () + ; proof + ; proof_must_verify = Boolean.true_ + } + ; { public_input = () + ; proof + ; proof_must_verify = Boolean.true_ + } + ] + ; public_output = () + ; auxiliary_output = () + } ) } ] ) ) @@ -2054,10 +2055,11 @@ module Make_str (_ : Wire_types.Concrete) = struct (fun { public_input = self } -> dummy_constraints () ; Field.Assert.equal self Field.zero ; - { previous_proof_statements = [] - ; public_output = () - ; auxiliary_output = () - } ) + Promise.return + { Inductive_rule.previous_proof_statements = [] + ; public_output = () + ; auxiliary_output = () + } ) } ] ) ) @@ -2105,10 +2107,11 @@ module Make_str (_ : Wire_types.Concrete) = struct (fun { public_input = self } -> dummy_constraints () ; Field.Assert.equal self Field.zero ; - { previous_proof_statements = [] - ; public_output = () - ; auxiliary_output = () - } ) + Promise.return + { Inductive_rule.previous_proof_statements = [] + ; public_output = () + ; auxiliary_output = () + } ) } ] ) ) @@ -2157,10 +2160,11 @@ module Make_str (_ : Wire_types.Concrete) = struct (fun { public_input = self } -> dummy_constraints () ; Field.Assert.equal self Field.zero ; - { previous_proof_statements = [] - ; public_output = () - ; auxiliary_output = () - } ) + Promise.return + { Inductive_rule.previous_proof_statements = [] + ; public_output = () + ; auxiliary_output = () + } ) } ] ) ) @@ -2254,15 +2258,16 @@ module Make_str (_ : Wire_types.Concrete) = struct let is_base_case = Field.equal Field.zero self in let self_correct = Field.(equal (one + prev) self) in Boolean.Assert.any [ self_correct; is_base_case ] ; - { previous_proof_statements = - [ { public_input = prev - ; proof - ; proof_must_verify = Boolean.true_ - } - ] - ; public_output = () - ; auxiliary_output = () - } ) + Promise.return + { Inductive_rule.previous_proof_statements = + [ { public_input = prev + ; proof + ; proof_must_verify = Boolean.true_ + } + ] + ; public_output = () + ; auxiliary_output = () + } ) } ] ) ) @@ -2272,13 +2277,16 @@ module Make_str (_ : Wire_types.Concrete) = struct let (), (), b1 = Common.time "b1" (fun () -> Promise.block_on_async_exn (fun () -> + let%bind.Promise vk = + Side_loaded.Verification_key.of_compiled_promise + No_recursion.tag + in step ~handler: (handler No_recursion.example_input (Side_loaded.Proof.of_proof No_recursion.example_proof ) - (Side_loaded.Verification_key.of_compiled - No_recursion.tag ) ) + vk ) Field.Constant.one ) ) in Or_error.ok_exn @@ -2290,13 +2298,16 @@ module Make_str (_ : Wire_types.Concrete) = struct let (), (), b2 = Common.time "b2" (fun () -> Promise.block_on_async_exn (fun () -> + let%bind.Promise vk = + Side_loaded.Verification_key.of_compiled_promise + Fake_1_recursion.tag + in step ~handler: (handler Fake_1_recursion.example_input (Side_loaded.Proof.of_proof Fake_1_recursion.example_proof ) - (Side_loaded.Verification_key.of_compiled - Fake_1_recursion.tag ) ) + vk ) Field.Constant.one ) ) in Or_error.ok_exn @@ -2308,13 +2319,16 @@ module Make_str (_ : Wire_types.Concrete) = struct let (), (), b3 = Common.time "b3" (fun () -> Promise.block_on_async_exn (fun () -> + let%bind.Promise vk = + Side_loaded.Verification_key.of_compiled_promise + Fake_2_recursion.tag + in step ~handler: (handler Fake_2_recursion.example_input (Side_loaded.Proof.of_proof Fake_2_recursion.example_proof ) - (Side_loaded.Verification_key.of_compiled - Fake_2_recursion.tag ) ) + vk ) Field.Constant.one ) ) in Or_error.ok_exn @@ -2402,10 +2416,11 @@ module Make_str (_ : Wire_types.Concrete) = struct (fun { public_input = self } -> dummy_constraints () ; Field.Assert.equal self Field.zero ; - { previous_proof_statements = [] - ; public_output = () - ; auxiliary_output = () - } ) + Promise.return + { Inductive_rule.previous_proof_statements = [] + ; public_output = () + ; auxiliary_output = () + } ) } ] ) ) @@ -2453,10 +2468,11 @@ module Make_str (_ : Wire_types.Concrete) = struct (fun { public_input = self } -> dummy_constraints () ; Field.Assert.equal self Field.zero ; - { previous_proof_statements = [] - ; public_output = () - ; auxiliary_output = () - } ) + Promise.return + { Inductive_rule.previous_proof_statements = [] + ; public_output = () + ; auxiliary_output = () + } ) } ] ) ) @@ -2505,10 +2521,11 @@ module Make_str (_ : Wire_types.Concrete) = struct (fun { public_input = self } -> dummy_constraints () ; Field.Assert.equal self Field.zero ; - { previous_proof_statements = [] - ; public_output = () - ; auxiliary_output = () - } ) + Promise.return + { Inductive_rule.previous_proof_statements = [] + ; public_output = () + ; auxiliary_output = () + } ) } ] ) ) @@ -2605,15 +2622,16 @@ module Make_str (_ : Wire_types.Concrete) = struct let is_base_case = Field.equal Field.zero self in let self_correct = Field.(equal (one + prev) self) in Boolean.Assert.any [ self_correct; is_base_case ] ; - { previous_proof_statements = - [ { public_input = prev - ; proof - ; proof_must_verify = Boolean.true_ - } - ] - ; public_output = () - ; auxiliary_output = () - } ) + Promise.return + { Inductive_rule.previous_proof_statements = + [ { public_input = prev + ; proof + ; proof_must_verify = Boolean.true_ + } + ] + ; public_output = () + ; auxiliary_output = () + } ) } ] ) ) @@ -2623,13 +2641,16 @@ module Make_str (_ : Wire_types.Concrete) = struct let (), (), b1 = Common.time "b1" (fun () -> Promise.block_on_async_exn (fun () -> + let%bind.Promise vk = + Side_loaded.Verification_key.of_compiled_promise + No_recursion.tag + in step ~handler: (handler No_recursion.example_input (Side_loaded.Proof.of_proof No_recursion.example_proof ) - (Side_loaded.Verification_key.of_compiled - No_recursion.tag ) ) + vk ) Field.Constant.one ) ) in Or_error.ok_exn @@ -2641,13 +2662,16 @@ module Make_str (_ : Wire_types.Concrete) = struct let (), (), b2 = Common.time "b2" (fun () -> Promise.block_on_async_exn (fun () -> + let%bind.Promise vk = + Side_loaded.Verification_key.of_compiled_promise + Fake_1_recursion.tag + in step ~handler: (handler Fake_1_recursion.example_input (Side_loaded.Proof.of_proof Fake_1_recursion.example_proof ) - (Side_loaded.Verification_key.of_compiled - Fake_1_recursion.tag ) ) + vk ) Field.Constant.one ) ) in Or_error.ok_exn @@ -2659,13 +2683,16 @@ module Make_str (_ : Wire_types.Concrete) = struct let (), (), b3 = Common.time "b3" (fun () -> Promise.block_on_async_exn (fun () -> + let%bind.Promise vk = + Side_loaded.Verification_key.of_compiled_promise + Fake_2_recursion.tag + in step ~handler: (handler Fake_2_recursion.example_input (Side_loaded.Proof.of_proof Fake_2_recursion.example_proof ) - (Side_loaded.Verification_key.of_compiled - Fake_2_recursion.tag ) ) + vk ) Field.Constant.one ) ) in Or_error.ok_exn diff --git a/src/lib/pickles/pickles_intf.mli b/src/lib/pickles/pickles_intf.mli index f3bcb90aab0..b36c01fe32d 100644 --- a/src/lib/pickles/pickles_intf.mli +++ b/src/lib/pickles/pickles_intf.mli @@ -66,9 +66,13 @@ module type S = sig type t - val verification_key : Verification_key.t Lazy.t + val verification_key_promise : Verification_key.t Promise.t Lazy.t - val id : Verification_key.Id.t Lazy.t + val verification_key : Verification_key.t Deferred.t Lazy.t + + val id_promise : Verification_key.Id.t Promise.t Lazy.t + + val id : Verification_key.Id.t Deferred.t Lazy.t val verify : (statement * t) list -> unit Or_error.t Deferred.t @@ -235,6 +239,7 @@ module type S = sig ; main : 'a_var main_input -> ('prev_vars, 'widths, 'ret_var, 'auxiliary_var) main_return + Promise.t ; feature_flags : bool Pickles_types.Plonk_types.Features.t } end @@ -273,7 +278,7 @@ module type S = sig module Cache_handle : sig type t - val generate_or_load : t -> Dirty.t + val generate_or_load : t -> Dirty.t Promise.t end module Storables : sig diff --git a/src/lib/pickles/step.ml b/src/lib/pickles/step.ml index 733c5fce009..cf122da2e14 100644 --- a/src/lib/pickles/step.ml +++ b/src/lib/pickles/step.ml @@ -804,7 +804,9 @@ struct Impls.Step.input ~proofs_verified:Max_proofs_verified.n ~wrap_rounds:Tock.Rounds.n in - let { Domains.h } = Vector.nth_exn step_domains branch_data.index in + let%bind.Promise { Domains.h } = + Vector.nth_exn step_domains branch_data.index + in ksprintf Common.time "step-prover %d (%d)" branch_data.index (Domain.size h) (fun () -> [%log internal] "Step_generate_witness_conv" ; diff --git a/src/lib/pickles/step.mli b/src/lib/pickles/step.mli index 54a2271ed40..647c3acd4e5 100644 --- a/src/lib/pickles/step.mli +++ b/src/lib/pickles/step.mli @@ -30,7 +30,8 @@ module Make and type ns = 'max_local_max_proof_verifieds ) -> prevs_length:('prev_vars, 'prevs_length) Pickles_types.Hlist.Length.t -> self:('a, 'b, 'c, 'd) Tag.t - -> step_domains:(Import.Domains.t, 'self_branches) Pickles_types.Vector.t + -> step_domains: + (Import.Domains.t Promise.t, 'self_branches) Pickles_types.Vector.t -> feature_flags:Opt.Flag.t Plonk_types.Features.Full.t -> self_dlog_plonk_index: Backend.Tick.Inner_curve.Affine.t array diff --git a/src/lib/pickles/step_branch_data.ml b/src/lib/pickles/step_branch_data.ml index 496a52ca552..7f0d4d3c9d0 100644 --- a/src/lib/pickles/step_branch_data.ml +++ b/src/lib/pickles/step_branch_data.ml @@ -22,7 +22,7 @@ type ( 'a_var 'proofs_verified Nat.t * ('prev_vars, 'proofs_verified) Hlist.Length.t ; index : int ; lte : ('proofs_verified, 'max_proofs_verified) Nat.Lte.t - ; domains : Domains.t + ; domains : Domains.t Promise.t ; rule : ( 'prev_vars , 'prev_values @@ -36,7 +36,7 @@ type ( 'a_var , 'auxiliary_value ) Inductive_rule.t ; main : - step_domains:(Domains.t, 'branches) Vector.t + step_domains:(Domains.t Promise.t, 'branches) Vector.t -> unit -> ( (Unfinalized.t, 'max_proofs_verified) Vector.t , Impls.Step.Field.t @@ -133,6 +133,16 @@ let create in Timer.clock __LOC__ ; let step ~step_domains = + let%map.Promise step_domains = + let%map.Promise () = + (* Wait for promises to resolve. *) + Vector.fold ~init:(Promise.return ()) step_domains + ~f:(fun acc step_domain -> + let%bind.Promise _ = step_domain in + acc ) + in + Vector.map ~f:(fun x -> Option.value_exn @@ Promise.peek x) step_domains + in Step_main.step_main requests (Nat.Add.create max_proofs_verified) rule @@ -148,12 +158,26 @@ let create ~local_branches_length ~lte ~self |> unstage in + (* Rebinding trick: we set off the outer promise immediately, as we would if + it was not there, and then we chain off the promise in the main function + itself. + Notably, this ensures that we don't initialize on every call of the + function, which is the whole idea of the [stage] above. + *) + let step ~step_domains = + let step = step ~step_domains in + fun () -> + let%bind.Promise step = step in + step () + in + (* Now that we've triggered the promise computation, we rebind. *) Timer.clock __LOC__ ; - let%map.Promise own_domains = + let own_domains = let main = step ~step_domains: - (Vector.init branches ~f:(fun _ -> Fix_domains.rough_domains)) + (Vector.init branches ~f:(fun _ -> + Promise.return Fix_domains.rough_domains ) ) in let etyp = Impls.Step.input ~proofs_verified:max_proofs_verified diff --git a/src/lib/pickles/step_branch_data.mli b/src/lib/pickles/step_branch_data.mli index 8713efa9728..5e84fd6cad5 100644 --- a/src/lib/pickles/step_branch_data.mli +++ b/src/lib/pickles/step_branch_data.mli @@ -23,7 +23,7 @@ type ( 'a_var * ('prev_vars, 'proofs_verified) Pickles_types.Hlist.Length.t ; index : int ; lte : ('proofs_verified, 'max_proofs_verified) Pickles_types.Nat.Lte.t - ; domains : Import.Domains.t + ; domains : Import.Domains.t Promise.t ; rule : ( 'prev_vars , 'prev_values @@ -38,7 +38,8 @@ type ( 'a_var Inductive_rule.t (* Main functions to compute *) ; main : - step_domains:(Import.Domains.t, 'branches) Pickles_types.Vector.t + step_domains: + (Import.Domains.t Promise.t, 'branches) Pickles_types.Vector.t -> unit -> ( (Unfinalized.t, 'max_proofs_verified) Pickles_types.Vector.t , Impls.Step.Field.t @@ -124,4 +125,3 @@ val create : , 'g , 'h ) t - Promise.t diff --git a/src/lib/pickles/step_main.ml b/src/lib/pickles/step_main.ml index 7d92441bb12..cad6dc74da4 100644 --- a/src/lib/pickles/step_main.ml +++ b/src/lib/pickles/step_main.ml @@ -276,9 +276,10 @@ let step_main : let T = Max_proofs_verified.eq in let app_state = exists input_typ ~request:(fun () -> Req.App_state) in let module Optional_wrap_key = struct - type (_, _, _, _) t = - Step_main_inputs.Inner_curve.Constant.t array - Plonk_verification_key_evals.t + type (_, _, _, 'branches) t = + ( Step_main_inputs.Inner_curve.Constant.t array + Plonk_verification_key_evals.t + * (Import.Domains.t, 'branches) Vector.t ) option end in (* Here, we prefetch the known wrap keys for all compiled rules. @@ -303,11 +304,21 @@ let step_main : | None -> ( match tag.kind with | Compiled -> - let%map.Promise wrap_key = - Lazy.force - @@ (Types_map.lookup_compiled tag.id).wrap_key + let compiled = Types_map.lookup_compiled tag.id in + let%map.Promise wrap_key = Lazy.force @@ compiled.wrap_key + and step_domains = + let%map.Promise () = + (* Wait for promises to resolve. *) + Vector.fold ~init:(Promise.return ()) + compiled.step_domains ~f:(fun acc step_domain -> + let%bind.Promise _ = step_domain in + acc ) + in + Vector.map + ~f:(fun x -> Option.value_exn @@ Promise.peek x) + compiled.step_domains in - Some wrap_key + Some (wrap_key, step_domains) | Side_loaded -> Promise.return None ) in @@ -542,10 +553,12 @@ let step_main : | None -> ( match tag.kind with | Compiled -> + let wrap_key, step_domains = + Option.value_exn optional_wrap_key + in Types_map.For_step - .of_compiled_with_known_wrap_key - ~wrap_key: - (Option.value_exn optional_wrap_key) + .of_compiled_with_known_wrap_key ~wrap_key + ~step_domains (Types_map.lookup_compiled tag.id) | Side_loaded -> Types_map.For_step.of_side_loaded diff --git a/src/lib/pickles/types_map.ml b/src/lib/pickles/types_map.ml index 71886be392d..abfe7b8e96d 100644 --- a/src/lib/pickles/types_map.ml +++ b/src/lib/pickles/types_map.ml @@ -103,7 +103,7 @@ module Compiled = struct Lazy.t ; wrap_vk : Impls.Wrap.Verification_key.t Promise.t Lazy.t ; wrap_domains : Domains.t - ; step_domains : (Domains.t, 'branches) Vector.t + ; step_domains : (Domains.t Promise.t, 'branches) Vector.t ; feature_flags : Opt.Flag.t Plonk_types.Features.Full.t } @@ -177,14 +177,14 @@ module For_step = struct ; feature_flags } - let of_compiled_with_known_wrap_key ~wrap_key + let of_compiled_with_known_wrap_key ~wrap_key ~step_domains ({ branches ; max_proofs_verified ; proofs_verifieds ; public_input ; wrap_key = _ ; wrap_domains - ; step_domains + ; step_domains = _ ; feature_flags ; wrap_vk = _ } : @@ -202,9 +202,19 @@ module For_step = struct ; feature_flags } - let of_compiled ({ wrap_key; _ } as t : _ Compiled.t) = - let%map.Promise wrap_key = Lazy.force wrap_key in - of_compiled_with_known_wrap_key ~wrap_key t + let of_compiled ({ wrap_key; step_domains; _ } as t : _ Compiled.t) = + let%map.Promise wrap_key = Lazy.force wrap_key + and step_domains = + let%map.Promise () = + (* Wait for promises to resolve. *) + Vector.fold ~init:(Promise.return ()) step_domains + ~f:(fun acc step_domain -> + let%bind.Promise _ = step_domain in + acc ) + in + Vector.map ~f:(fun x -> Option.value_exn @@ Promise.peek x) step_domains + in + of_compiled_with_known_wrap_key ~wrap_key ~step_domains t end type t = diff --git a/src/lib/pickles/types_map.mli b/src/lib/pickles/types_map.mli index d8711ed5338..fd7aa2a4f7a 100644 --- a/src/lib/pickles/types_map.mli +++ b/src/lib/pickles/types_map.mli @@ -75,7 +75,8 @@ module Compiled : sig Lazy.t ; wrap_vk : Impls.Wrap.Verification_key.t Promise.t Lazy.t ; wrap_domains : Import.Domains.t - ; step_domains : (Import.Domains.t, 'branches) Pickles_types.Vector.t + ; step_domains : + (Import.Domains.t Promise.t, 'branches) Pickles_types.Vector.t ; feature_flags : Opt.Flag.t Plonk_types.Features.Full.t } end @@ -107,8 +108,9 @@ module For_step : sig wrap_key: Step_main_inputs.Inner_curve.Constant.t array Plonk_verification_key_evals.t - -> ('a, 'b, 'c, 'd) Compiled.t - -> ('a, 'b, 'c, 'd) t + -> step_domains:(Import.Domains.t, 'branches) Vector.t + -> ('a, 'b, 'c, 'branches) Compiled.t + -> ('a, 'b, 'c, 'branches) t val of_compiled : ('a, 'b, 'c, 'd) Compiled.t -> ('a, 'b, 'c, 'd) t Promise.t end diff --git a/src/lib/pickles/wrap_domains.ml b/src/lib/pickles/wrap_domains.ml index 1229ef498ff..9398fdb3bc0 100644 --- a/src/lib/pickles/wrap_domains.ml +++ b/src/lib/pickles/wrap_domains.ml @@ -18,7 +18,8 @@ struct ~max_proofs_verified = let num_choices = Hlist.Length.to_nat choices_length in let dummy_step_domains = - Vector.init num_choices ~f:(fun _ -> Fix_domains.rough_domains) + Vector.init num_choices ~f:(fun _ -> + Promise.return Fix_domains.rough_domains ) in let dummy_step_widths = Vector.init num_choices ~f:(fun _ -> diff --git a/src/lib/pickles/wrap_main.ml b/src/lib/pickles/wrap_main.ml index 5c612dc2439..f15daa3156b 100644 --- a/src/lib/pickles/wrap_main.ml +++ b/src/lib/pickles/wrap_main.ml @@ -97,7 +97,7 @@ let wrap_main Vector.t Promise.t Lazy.t ) (step_widths : (int, branches) Vector.t) - (step_domains : (Domains.t, branches) Vector.t) ~srs + (step_domains : (Domains.t Promise.t, branches) Vector.t) ~srs (max_proofs_verified : (module Nat.Add.Intf with type n = max_proofs_verified) ) : (max_proofs_verified, max_local_max_proofs_verifieds) Requests.Wrap.t @@ -134,7 +134,17 @@ let wrap_main Timer.clock __LOC__ ; let main = let%map.Lazy step_keys = step_keys in - let%map.Promise step_keys = step_keys in + let%bind.Promise step_keys = step_keys in + let%map.Promise step_domains = + let%map.Promise () = + (* Wait for promises to resolve. *) + Vector.fold ~init:(Promise.return ()) step_domains + ~f:(fun acc step_domain -> + let%bind.Promise _ = step_domain in + acc ) + in + Vector.map ~f:(fun x -> Option.value_exn @@ Promise.peek x) step_domains + in fun ({ proof_state = { deferred_values = { plonk diff --git a/src/lib/pickles/wrap_main.mli b/src/lib/pickles/wrap_main.mli index 6cc4e3a7867..7d0d9059f15 100644 --- a/src/lib/pickles/wrap_main.mli +++ b/src/lib/pickles/wrap_main.mli @@ -17,7 +17,7 @@ val wrap_main : Promise.t Lazy.t -> (int, 'branches) Pickles_types.Vector.t - -> (Import.Domains.t, 'branches) Pickles_types.Vector.t + -> (Import.Domains.t Promise.t, 'branches) Pickles_types.Vector.t -> srs:Kimchi_bindings.Protocol.SRS.Fp.t -> (module Pickles_types.Nat.Add.Intf with type n = 'max_proofs_verified) -> ('max_proofs_verified, 'max_local_max_proofs_verifieds) Requests.Wrap.t From 38a78488d13bf95de94d3f90b6da637a47a27a6b Mon Sep 17 00:00:00 2001 From: mrmr1993 Date: Sun, 11 Feb 2024 14:59:04 +0000 Subject: [PATCH 04/45] Fixup default `Pickles.compile` to use non-async inductive rules --- src/lib/pickles/compile.ml | 9 +-- src/lib/pickles/compile.mli | 2 +- src/lib/pickles/inductive_rule.ml | 87 +++++++++++++------------ src/lib/pickles/inductive_rule.mli | 94 ++++++++++++++++------------ src/lib/pickles/pickles.ml | 24 ++++++- src/lib/pickles/pickles_intf.mli | 55 +++++++++------- src/lib/pickles/step_branch_data.ml | 4 +- src/lib/pickles/step_branch_data.mli | 4 +- src/lib/pickles/step_main.ml | 2 +- src/lib/pickles/step_main.mli | 2 +- 10 files changed, 170 insertions(+), 113 deletions(-) diff --git a/src/lib/pickles/compile.ml b/src/lib/pickles/compile.ml index 12f160b4373..12399ad0e28 100644 --- a/src/lib/pickles/compile.ml +++ b/src/lib/pickles/compile.ml @@ -259,7 +259,8 @@ module Make (Auxiliary_value : T0) = struct module IR = - Inductive_rule.T (Arg_var) (Arg_value) (Ret_var) (Ret_value) (Auxiliary_var) + Inductive_rule.Promise.T (Arg_var) (Arg_value) (Ret_var) (Ret_value) + (Auxiliary_var) (Auxiliary_value) module HIR = H4.T (IR) @@ -1035,7 +1036,7 @@ let compile_with_wrap_main_override_promise : , ret_value , auxiliary_var , auxiliary_value ) - H4_6.T(Inductive_rule).t ) + H4_6.T(Inductive_rule.Promise).t ) -> unit -> (var, value, max_proofs_verified, branches) Tag.t * Cache_handle.t @@ -1115,7 +1116,7 @@ let compile_with_wrap_main_override_promise : , ret_value , auxiliary_var , auxiliary_value ) - H4_6.T(Inductive_rule).t + H4_6.T(Inductive_rule.Promise).t -> (v1ss, v2ss, wss, hss) H4.T(M.IR).t = function | [] -> [] @@ -1314,7 +1315,7 @@ struct ; fork = None } - let rule self : _ Inductive_rule.t = + let rule self : _ Inductive_rule.Promise.t = { identifier = "main" ; prevs = [ self; self ] ; main = diff --git a/src/lib/pickles/compile.mli b/src/lib/pickles/compile.mli index 5682d7e2830..5ad22212770 100644 --- a/src/lib/pickles/compile.mli +++ b/src/lib/pickles/compile.mli @@ -324,7 +324,7 @@ val compile_with_wrap_main_override_promise : , 'ret_value , 'auxiliary_var , 'auxiliary_value ) - H4_6.T(Inductive_rule).t ) + H4_6.T(Inductive_rule.Promise).t ) -> unit -> ('var, 'value, 'max_proofs_verified, 'branches) Tag.t * Cache_handle.t diff --git a/src/lib/pickles/inductive_rule.ml b/src/lib/pickles/inductive_rule.ml index 4eacd777939..98ad65497de 100644 --- a/src/lib/pickles/inductive_rule.ml +++ b/src/lib/pickles/inductive_rule.ml @@ -63,7 +63,11 @@ type ('prev_vars, 'widths, 'public_output, 'auxiliary_output) main_return = *) } -(** This type models an "inductive rule". It includes +module Make (M : sig + type _ t +end) = +struct + (** This type models an "inductive rule". It includes - the list of previous statements which this one assumes - the snarky main function @@ -99,43 +103,48 @@ type ('prev_vars, 'widths, 'public_output, 'auxiliary_output) main_return = auxiliary data, to be returned to the prover but not exposed in the public input. *) -type ( 'prev_vars - , 'prev_values - , 'widths - , 'heights - , 'a_var - , 'a_value - , 'ret_var - , 'ret_value - , 'auxiliary_var - , 'auxiliary_value ) - t = - { identifier : string - ; prevs : ('prev_vars, 'prev_values, 'widths, 'heights) H4.T(Tag).t - ; main : - 'a_var main_input - -> ('prev_vars, 'widths, 'ret_var, 'auxiliary_var) main_return Promise.t - ; feature_flags : bool Pickles_types.Plonk_types.Features.t - } + type ( 'prev_vars + , 'prev_values + , 'widths + , 'heights + , 'a_var + , 'a_value + , 'ret_var + , 'ret_value + , 'auxiliary_var + , 'auxiliary_value ) + t = + { identifier : string + ; prevs : ('prev_vars, 'prev_values, 'widths, 'heights) H4.T(Tag).t + ; main : + 'a_var main_input + -> ('prev_vars, 'widths, 'ret_var, 'auxiliary_var) main_return M.t + ; feature_flags : bool Pickles_types.Plonk_types.Features.t + } -module T - (Statement : T0) - (Statement_value : T0) - (Return_var : T0) - (Return_value : T0) - (Auxiliary_var : T0) - (Auxiliary_value : T0) = -struct - type nonrec ('prev_vars, 'prev_values, 'widths, 'heights) t = - ( 'prev_vars - , 'prev_values - , 'widths - , 'heights - , Statement.t - , Statement_value.t - , Return_var.t - , Return_value.t - , Auxiliary_var.t - , Auxiliary_value.t ) - t + module T + (Statement : T0) + (Statement_value : T0) + (Return_var : T0) + (Return_value : T0) + (Auxiliary_var : T0) + (Auxiliary_value : T0) = + struct + type nonrec ('prev_vars, 'prev_values, 'widths, 'heights) t = + ( 'prev_vars + , 'prev_values + , 'widths + , 'heights + , Statement.t + , Statement_value.t + , Return_var.t + , Return_value.t + , Auxiliary_var.t + , Auxiliary_value.t ) + t + end end + +module Promise = Make (Promise) +module Deferred = Make (Async_kernel.Deferred) +include Make (Id) diff --git a/src/lib/pickles/inductive_rule.mli b/src/lib/pickles/inductive_rule.mli index 9a56bf875f6..198589684fd 100644 --- a/src/lib/pickles/inductive_rule.mli +++ b/src/lib/pickles/inductive_rule.mli @@ -56,7 +56,10 @@ type ('prev_vars, 'widths, 'public_output, 'auxiliary_output) main_return = *) } -(** This type models an "inductive rule". It includes +module Make (M : sig + type _ t +end) : sig + (** This type models an "inductive rule". It includes - the list of previous statements which this one assumes - the snarky main function @@ -92,47 +95,58 @@ type ('prev_vars, 'widths, 'public_output, 'auxiliary_output) main_return = auxiliary data, to be returned to the prover but not exposed in the public input. *) -type ( 'prev_vars - , 'prev_values - , 'widths - , 'heights - , 'a_var - , 'a_value - , 'ret_var - , 'ret_value - , 'auxiliary_var - , 'auxiliary_value ) - t = - { identifier : string - ; prevs : + type ( 'prev_vars + , 'prev_values + , 'widths + , 'heights + , 'a_var + , 'a_value + , 'ret_var + , 'ret_value + , 'auxiliary_var + , 'auxiliary_value ) + t = + { identifier : string + ; prevs : + ( 'prev_vars + , 'prev_values + , 'widths + , 'heights ) + Pickles_types.Hlist.H4.T(Tag).t + ; main : + 'a_var main_input + -> ('prev_vars, 'widths, 'ret_var, 'auxiliary_var) main_return M.t + ; feature_flags : bool Pickles_types.Plonk_types.Features.t + } + + module T + (Statement : Pickles_types.Poly_types.T0) + (Statement_value : Pickles_types.Poly_types.T0) + (Return_var : Pickles_types.Poly_types.T0) + (Return_value : Pickles_types.Poly_types.T0) + (Auxiliary_var : Pickles_types.Poly_types.T0) + (Auxiliary_value : Pickles_types.Poly_types.T0) : sig + type nonrec ('prev_vars, 'prev_values, 'widths, 'heights) t = ( 'prev_vars , 'prev_values , 'widths - , 'heights ) - Pickles_types.Hlist.H4.T(Tag).t - ; main : - 'a_var main_input - -> ('prev_vars, 'widths, 'ret_var, 'auxiliary_var) main_return Promise.t - ; feature_flags : bool Pickles_types.Plonk_types.Features.t - } + , 'heights + , Statement.t + , Statement_value.t + , Return_var.t + , Return_value.t + , Auxiliary_var.t + , Auxiliary_value.t ) + t + end +end + +module Promise : sig + include module type of Make (Promise) +end -module T - (Statement : Pickles_types.Poly_types.T0) - (Statement_value : Pickles_types.Poly_types.T0) - (Return_var : Pickles_types.Poly_types.T0) - (Return_value : Pickles_types.Poly_types.T0) - (Auxiliary_var : Pickles_types.Poly_types.T0) - (Auxiliary_value : Pickles_types.Poly_types.T0) : sig - type nonrec ('prev_vars, 'prev_values, 'widths, 'heights) t = - ( 'prev_vars - , 'prev_values - , 'widths - , 'heights - , Statement.t - , Statement_value.t - , Return_var.t - , Return_value.t - , Auxiliary_var.t - , Auxiliary_value.t ) - t +module Deferred : sig + include module type of Make (Async_kernel.Deferred) end + +include module type of Make (Pickles_types.Hlist.Id) diff --git a/src/lib/pickles/pickles.ml b/src/lib/pickles/pickles.ml index f9208ddb8fd..28022752095 100644 --- a/src/lib/pickles/pickles.ml +++ b/src/lib/pickles/pickles.ml @@ -324,6 +324,25 @@ module Make_str (_ : Wire_types.Concrete) = struct let compile ?self ?cache ?storables ?proof_cache ?disk_keys ?override_wrap_domain ~public_input ~auxiliary_typ ~branches ~max_proofs_verified ~name ~constraint_constants ~choices () = + let choices ~self = + let choices = choices ~self in + let rec go : + type a b c d e f g h i j. + (a, b, c, d, e, f, g, h, i, j) H4_6.T(Inductive_rule).t + -> (a, b, c, d, e, f, g, h, i, j) H4_6.T(Inductive_rule.Promise).t = + function + | [] -> + [] + | { identifier; prevs; main; feature_flags } :: rest -> + { identifier + ; prevs + ; main = (fun x -> Promise.return (main x)) + ; feature_flags + } + :: go rest + in + go choices + in let self, cache_handle, proof_module, provers = compile_promise ?self ?cache ?storables ?proof_cache ?disk_keys ?override_wrap_domain ~public_input ~auxiliary_typ ~branches @@ -1021,7 +1040,7 @@ module Make_str (_ : Wire_types.Concrete) = struct let tagname = "" in Tag.create ~kind:Compiled tagname - let rule : _ Inductive_rule.t = + let rule : _ Inductive_rule.Promise.t = let open Impls.Step in { identifier = "main" ; prevs = [ tag; tag ] @@ -1049,7 +1068,8 @@ module Make_str (_ : Wire_types.Concrete) = struct } module M = struct - module IR = Inductive_rule.T (A) (A_value) (A) (A_value) (A) (A_value) + module IR = + Inductive_rule.Promise.T (A) (A_value) (A) (A_value) (A) (A_value) let max_local_max_proofs_verifieds ~self (type n) (module Max_proofs_verified : Nat.Intf with type n = n) branches diff --git a/src/lib/pickles/pickles_intf.mli b/src/lib/pickles/pickles_intf.mli index b36c01fe32d..df69484311b 100644 --- a/src/lib/pickles/pickles_intf.mli +++ b/src/lib/pickles/pickles_intf.mli @@ -187,7 +187,10 @@ module type S = sig *) } - (** This type models an "inductive rule". It includes + module Make (M : sig + type _ t + end) : sig + (** This type models an "inductive rule". It includes - the list of previous statements which this one assumes - the snarky main function @@ -223,25 +226,35 @@ module type S = sig auxiliary data, to be returned to the prover but not exposed in the public input. *) - type ( 'prev_vars - , 'prev_values - , 'widths - , 'heights - , 'a_var - , 'a_value - , 'ret_var - , 'ret_value - , 'auxiliary_var - , 'auxiliary_value ) - t = - { identifier : string - ; prevs : ('prev_vars, 'prev_values, 'widths, 'heights) H4.T(Tag).t - ; main : - 'a_var main_input - -> ('prev_vars, 'widths, 'ret_var, 'auxiliary_var) main_return - Promise.t - ; feature_flags : bool Pickles_types.Plonk_types.Features.t - } + type ( 'prev_vars + , 'prev_values + , 'widths + , 'heights + , 'a_var + , 'a_value + , 'ret_var + , 'ret_value + , 'auxiliary_var + , 'auxiliary_value ) + t = + { identifier : string + ; prevs : ('prev_vars, 'prev_values, 'widths, 'heights) H4.T(Tag).t + ; main : + 'a_var main_input + -> ('prev_vars, 'widths, 'ret_var, 'auxiliary_var) main_return M.t + ; feature_flags : bool Pickles_types.Plonk_types.Features.t + } + end + + module Promise : sig + include module type of Make (Promise) + end + + module Deferred : sig + include module type of Make (Deferred) + end + + include module type of Make (Id) end val verify_promise : @@ -411,7 +424,7 @@ module type S = sig , 'ret_value , 'auxiliary_var , 'auxiliary_value ) - H4_6.T(Inductive_rule).t ) + H4_6.T(Inductive_rule.Promise).t ) -> unit -> ('var, 'value, 'max_proofs_verified, 'branches) Tag.t * Cache_handle.t diff --git a/src/lib/pickles/step_branch_data.ml b/src/lib/pickles/step_branch_data.ml index 7f0d4d3c9d0..0aaaaba8d01 100644 --- a/src/lib/pickles/step_branch_data.ml +++ b/src/lib/pickles/step_branch_data.ml @@ -34,7 +34,7 @@ type ( 'a_var , 'ret_value , 'auxiliary_var , 'auxiliary_value ) - Inductive_rule.t + Inductive_rule.Promise.t ; main : step_domains:(Domains.t Promise.t, 'branches) Vector.t -> unit @@ -85,7 +85,7 @@ let create , ret_var , ret_value ) Inductive_rule.public_input ) ~auxiliary_typ _var_to_field_elements - _value_to_field_elements (rule : _ Inductive_rule.t) = + _value_to_field_elements (rule : _ Inductive_rule.Promise.t) = Timer.clock __LOC__ ; let module HT = H4.T (Tag) in let (T (self_width, proofs_verified)) = HT.length rule.prevs in diff --git a/src/lib/pickles/step_branch_data.mli b/src/lib/pickles/step_branch_data.mli index 5e84fd6cad5..4ac5366284c 100644 --- a/src/lib/pickles/step_branch_data.mli +++ b/src/lib/pickles/step_branch_data.mli @@ -35,7 +35,7 @@ type ( 'a_var , 'ret_value , 'auxiliary_var , 'auxiliary_value ) - Inductive_rule.t + Inductive_rule.Promise.t (* Main functions to compute *) ; main : step_domains: @@ -111,7 +111,7 @@ val create : , 'ret_value , 'a , 'b ) - Inductive_rule.t + Inductive_rule.Promise.t -> ( 'a_var , 'a_value , 'ret_var diff --git a/src/lib/pickles/step_main.ml b/src/lib/pickles/step_main.ml index cad6dc74da4..447022b3115 100644 --- a/src/lib/pickles/step_main.ml +++ b/src/lib/pickles/step_main.ml @@ -171,7 +171,7 @@ let step_main : , ret_value , auxiliary_var , auxiliary_value ) - Inductive_rule.t + Inductive_rule.Promise.t -> ( unit -> ( (Unfinalized.t, max_proofs_verified) Vector.t , Field.t diff --git a/src/lib/pickles/step_main.mli b/src/lib/pickles/step_main.mli index faec66ceaec..c6f348d9f01 100644 --- a/src/lib/pickles/step_main.mli +++ b/src/lib/pickles/step_main.mli @@ -50,7 +50,7 @@ val step_main : , 'ret_value , 'auxiliary_var , 'auxiliary_value ) - Inductive_rule.t + Inductive_rule.Promise.t -> ( unit -> ( (Unfinalized.t, 'max_proofs_verified) Pickles_types.Vector.t , Impls.Step.Field.t From 08165dffa0f77ba4f3fa30c9e41974f7d76c66a4 Mon Sep 17 00:00:00 2001 From: mrmr1993 Date: Sun, 11 Feb 2024 15:02:59 +0000 Subject: [PATCH 05/45] Add `Pickles.compile_async` --- src/lib/pickles/pickles.ml | 45 ++++++++++++++++++++++++++ src/lib/pickles/pickles_intf.mli | 55 ++++++++++++++++++++++++++++++++ 2 files changed, 100 insertions(+) diff --git a/src/lib/pickles/pickles.ml b/src/lib/pickles/pickles.ml index 28022752095..18dbff19748 100644 --- a/src/lib/pickles/pickles.ml +++ b/src/lib/pickles/pickles.ml @@ -361,6 +361,51 @@ module Make_str (_ : Wire_types.Concrete) = struct in (self, cache_handle, proof_module, adjust_provers provers) + let compile_async ?self ?cache ?storables ?proof_cache ?disk_keys + ?override_wrap_domain ~public_input ~auxiliary_typ ~branches + ~max_proofs_verified ~name ~constraint_constants ~choices () = + let choices ~self = + let choices = choices ~self in + let rec go : + type a b c d e f g h i j. + (a, b, c, d, e, f, g, h, i, j) H4_6.T(Inductive_rule.Deferred).t + -> (a, b, c, d, e, f, g, h, i, j) H4_6.T(Inductive_rule.Promise).t = + function + | [] -> + [] + | { identifier; prevs; main; feature_flags } :: rest -> + { identifier + ; prevs + ; main = + (fun x -> + Promise.create (fun callback -> + Deferred.don't_wait_for + (let%map res = main x in + callback res ) ) ) + ; feature_flags + } + :: go rest + in + go choices + in + let self, cache_handle, proof_module, provers = + compile_promise ?self ?cache ?storables ?proof_cache ?disk_keys + ?override_wrap_domain ~public_input ~auxiliary_typ ~branches + ~max_proofs_verified ~name ~constraint_constants ~choices () + in + let rec adjust_provers : + type a1 a2 a3 s1 s2_inner. + (a1, a2, a3, s1, s2_inner Promise.t) H3_2.T(Prover).t + -> (a1, a2, a3, s1, s2_inner Deferred.t) H3_2.T(Prover).t = function + | [] -> + [] + | prover :: tl -> + (fun ?handler public_input -> + Promise.to_deferred (prover ?handler public_input) ) + :: adjust_provers tl + in + (self, cache_handle, proof_module, adjust_provers provers) + module Provers = H3_2.T (Prover) module Proof0 = Proof diff --git a/src/lib/pickles/pickles_intf.mli b/src/lib/pickles/pickles_intf.mli index df69484311b..41c8bf36a1f 100644 --- a/src/lib/pickles/pickles_intf.mli +++ b/src/lib/pickles/pickles_intf.mli @@ -495,4 +495,59 @@ module type S = sig * ('max_proofs_verified, 'max_proofs_verified) Proof.t ) Deferred.t ) H3_2.T(Prover).t + + (** This compiles a series of inductive rules defining a set into a proof + system for proving membership in that set, with a prover corresponding + to each inductive rule. *) + val compile_async : + ?self:('var, 'value, 'max_proofs_verified, 'branches) Tag.t + -> ?cache:Key_cache.Spec.t list + -> ?storables:Storables.t + -> ?proof_cache:Proof_cache.t + -> ?disk_keys: + (Cache.Step.Key.Verification.t, 'branches) Vector.t + * Cache.Wrap.Key.Verification.t + -> ?override_wrap_domain:Pickles_base.Proofs_verified.t + -> public_input: + ( 'var + , 'value + , 'a_var + , 'a_value + , 'ret_var + , 'ret_value ) + Inductive_rule.public_input + -> auxiliary_typ:('auxiliary_var, 'auxiliary_value) Impls.Step.Typ.t + -> branches:(module Nat.Intf with type n = 'branches) + -> max_proofs_verified: + (module Nat.Add.Intf with type n = 'max_proofs_verified) + -> name:string + -> constraint_constants:Snark_keys_header.Constraint_constants.t + -> choices: + ( self:('var, 'value, 'max_proofs_verified, 'branches) Tag.t + -> ( 'prev_varss + , 'prev_valuess + , 'widthss + , 'heightss + , 'a_var + , 'a_value + , 'ret_var + , 'ret_value + , 'auxiliary_var + , 'auxiliary_value ) + H4_6.T(Inductive_rule.Deferred).t ) + -> unit + -> ('var, 'value, 'max_proofs_verified, 'branches) Tag.t + * Cache_handle.t + * (module Proof_intf + with type t = ('max_proofs_verified, 'max_proofs_verified) Proof.t + and type statement = 'value ) + * ( 'prev_valuess + , 'widthss + , 'heightss + , 'a_value + , ( 'ret_value + * 'auxiliary_value + * ('max_proofs_verified, 'max_proofs_verified) Proof.t ) + Deferred.t ) + H3_2.T(Prover).t end From 95d6757b5203b4a7ad9900dd0655d7851861c264 Mon Sep 17 00:00:00 2001 From: mrmr1993 Date: Mon, 12 Feb 2024 17:51:12 +0000 Subject: [PATCH 06/45] Fixup compilation errors (nearly) --- src/app/zkapps_examples/zkapps_examples.ml | 12 ++++++------ src/lib/mina_base/util/mina_base_util.ml | 22 ++++++++++++++++++++++ src/lib/verifier/dummy.ml | 4 ++-- src/lib/verifier/prod.ml | 5 +++-- 4 files changed, 33 insertions(+), 10 deletions(-) diff --git a/src/app/zkapps_examples/zkapps_examples.ml b/src/app/zkapps_examples/zkapps_examples.ml index 739f21c7df8..834c6acb895 100644 --- a/src/app/zkapps_examples/zkapps_examples.ml +++ b/src/app/zkapps_examples/zkapps_examples.ml @@ -590,7 +590,7 @@ let compile : , Zkapp_statement.t , return_type Prover_value.t * auxiliary_var , return_type * auxiliary_value ) - H4_6.T(Pickles.Inductive_rule).t = function + H4_6.T(Pickles.Inductive_rule.Deferred).t = function | [] -> [] | { identifier; prevs; main; feature_flags } :: choices -> @@ -599,8 +599,8 @@ let compile : ; feature_flags ; main = (fun { Pickles.Inductive_rule.public_input = () } -> - let vk_hash = - exists Field.typ ~compute:(fun () -> + let%map.Deferred vk_hash = + Mina_base_util.exists_deferred Field.typ ~compute:(fun () -> Lazy.force @@ Option.value_exn !vk_hash ) in let { Pickles.Inductive_rule.previous_proof_statements @@ -612,7 +612,7 @@ let compile : let public_output, account_update_tree = to_account_update account_update_under_construction in - { previous_proof_statements + { Pickles.Inductive_rule.previous_proof_statements ; public_output ; auxiliary_output = (account_update_tree, auxiliary_output) } ) @@ -622,7 +622,7 @@ let compile : go (choices ~self) in let tag, cache_handle, proof, provers = - Pickles.compile () ?self ?cache ?disk_keys ?override_wrap_domain + Pickles.compile_async () ?self ?cache ?disk_keys ?override_wrap_domain ~public_input:(Output Zkapp_statement.typ) ~auxiliary_typ:Typ.(Prover_value.typ () * auxiliary_typ) ~branches ~max_proofs_verified ~name ~constraint_constants ~choices @@ -631,7 +631,7 @@ let compile : vk_hash := Some ( lazy - ( Zkapp_account.digest_vk + ( Deferred.map ~f:Zkapp_account.digest_vk @@ Pickles.Side_loaded.Verification_key.of_compiled tag ) ) in let provers = diff --git a/src/lib/mina_base/util/mina_base_util.ml b/src/lib/mina_base/util/mina_base_util.ml index 9241f54e195..19d76b3b6fb 100644 --- a/src/lib/mina_base/util/mina_base_util.ml +++ b/src/lib/mina_base/util/mina_base_util.ml @@ -17,3 +17,25 @@ let split_last_exn = function [] -> failwith "split_last: Empty list" | x :: xs -> go [] x xs let two_to_the i = Bignum_bigint.(pow (of_int 2) (of_int i)) + +(* TODO: This shouldn't live here.. *) +let exists_deferred ?request:req ?compute typ = + let open Snark_params.Tick.Run in + let open Async_kernel in + (* Set up a full Ivar, in case we are generating the constraint system. *) + let deferred = ref (Ivar.create_full ()) in + (* Request or compute the [Deferred.t] value. *) + let requested = exists ?request:req ?compute (Typ.Internal.ref ()) in + as_prover (fun () -> + (* If we are generating the witness, create a new Ivar.. *) + deferred := Ivar.create () ; + (* ..and fill it when the value we want to read resolves. *) + Deferred.upon (As_prover.Ref.get requested) (fun _ -> + Ivar.fill !deferred () ) ) ; + (* Await the [Deferred.t] if we're generating the witness, otherwise we + immediately bind over the filled Ivar and continue. + *) + Deferred.map (Ivar.read !deferred) ~f:(fun () -> + (* Retrieve the value by peeking in the known-resolved deferred. *) + exists typ ~compute:(fun () -> + Option.value_exn @@ Deferred.peek @@ As_prover.Ref.get requested ) ) diff --git a/src/lib/verifier/dummy.ml b/src/lib/verifier/dummy.ml index 4ee2cc1b12b..c9d4ba7b560 100644 --- a/src/lib/verifier/dummy.ml +++ b/src/lib/verifier/dummy.ml @@ -8,7 +8,7 @@ type t = ; verify_blockchain_snarks : Blockchain_snark.Blockchain.t list -> unit Or_error.t Or_error.t Deferred.t - ; verification_key : Pickles.Verification_key.t Lazy.t + ; verification_key : Pickles.Verification_key.t Deferred.t Lazy.t ; verify_transaction_snarks : (Ledger_proof.Prod.t * Mina_base.Sok_message.t) list -> unit Or_error.t Or_error.t Deferred.t @@ -163,7 +163,7 @@ let verify_transaction_snarks { verify_transaction_snarks; _ } ts = let get_blockchain_verification_key { verification_key; _ } = Deferred.Or_error.try_with ~here:[%here] (fun () -> - Deferred.return @@ Lazy.force verification_key ) + Lazy.force verification_key ) let toggle_internal_tracing _ _ = Deferred.Or_error.ok_unit diff --git a/src/lib/verifier/prod.ml b/src/lib/verifier/prod.ml index ef33349a2dd..1a516780552 100644 --- a/src/lib/verifier/prod.ml +++ b/src/lib/verifier/prod.ml @@ -53,7 +53,8 @@ module Worker_state = struct val verify_transaction_snarks : (Transaction_snark.t * Sok_message.t) list -> unit Or_error.t Deferred.t - val get_blockchain_verification_key : unit -> Pickles.Verification_key.t + val get_blockchain_verification_key : + unit -> Pickles.Verification_key.t Deferred.t val toggle_internal_tracing : bool -> unit @@ -314,7 +315,7 @@ module Worker = struct let get_blockchain_verification_key (w : Worker_state.t) () = let (module M) = Worker_state.get w in - Deferred.return (M.get_blockchain_verification_key ()) + M.get_blockchain_verification_key () let toggle_internal_tracing (w : Worker_state.t) enabled = let (module M) = Worker_state.get w in From 0c59d1f87f9dfcd26303ddbf0516ab815ef38e0d Mon Sep 17 00:00:00 2001 From: Gregor Date: Tue, 20 Feb 2024 12:50:55 +0100 Subject: [PATCH 07/45] make test_fix_domains compile --- .../optional_custom_gates/test_fix_domains.ml | 38 +++++++------------ 1 file changed, 14 insertions(+), 24 deletions(-) diff --git a/src/lib/pickles/test/optional_custom_gates/test_fix_domains.ml b/src/lib/pickles/test/optional_custom_gates/test_fix_domains.ml index 2c4304dae3b..b7a1e290c78 100644 --- a/src/lib/pickles/test/optional_custom_gates/test_fix_domains.ml +++ b/src/lib/pickles/test/optional_custom_gates/test_fix_domains.ml @@ -15,6 +15,16 @@ let etyp_unit = Composition_types.Spec.ETyp.T (Snarky_backendless.Typ.unit (), Core_kernel.Fn.id, Core_kernel.Fn.id) +let log2_size ~feature_flags main = + let main' () = main () ; Promise.return () in + let domains = + Promise.block_on_async_exn (fun () -> + Pickles__Fix_domains.domains ~feature_flags + (module Pickles.Impls.Step) + etyp_unit etyp_unit main' ) + in + Pickles_base.Domain.log2_size domains.h + let test_fix_domains_with_runtime_table_cfgs () = let table_sizes = [ [ 1 ]; [ 1; 1 ]; [ 1; 10; 42; 36 ] ] in (* Log2 value *) @@ -36,12 +46,7 @@ let test_fix_domains_with_runtime_table_cfgs () = (AddRuntimeTableCfg { id = Int32.of_int i; first_column }) ) table_sizes in - let domains = - Pickles__Fix_domains.domains ~feature_flags - (module Pickles.Impls.Step) - etyp_unit etyp_unit main - in - let log2_size = Pickles_base.Domain.log2_size domains.h in + let log2_size = log2_size ~feature_flags main in log2_size = exp_output ) table_sizes exp_output ) @@ -83,12 +88,7 @@ let test_fix_domains_with_runtime_table_cfgs_and_fixed_lookup_tables () = ) ) rt_cfgs_table_sizes in - let domains = - Pickles__Fix_domains.domains ~feature_flags - (module Pickles.Impls.Step) - etyp_unit etyp_unit main - in - let log2_size = Pickles_base.Domain.log2_size domains.h in + let log2_size = log2_size ~feature_flags main in log2_size = exp_output ) (List.combine fixed_table_sizes rt_cfgs_table_sizes) exp_outputs ) @@ -123,12 +123,7 @@ let test_fix_domains_with_runtime_table_cfgs_and_fixed_lookup_tables_sharing_id in add_constraint (AddRuntimeTableCfg { id; first_column }) in - let domains = - Pickles__Fix_domains.domains ~feature_flags - (module Pickles.Impls.Step) - etyp_unit etyp_unit main - in - let log2_size = Pickles_base.Domain.log2_size domains.h in + let log2_size = log2_size ~feature_flags main in log2_size = exp_output ) (List.combine fixed_lt_sizes rt_cfg_sizes) exp_outputs ) @@ -158,12 +153,7 @@ let test_fix_domains_with_fixed_lookup_tables () = { id = Int32.of_int i; data = [| indexes; values |] } ) ) table_sizes in - let domains = - Pickles__Fix_domains.domains ~feature_flags - (module Pickles.Impls.Step) - etyp_unit etyp_unit main - in - let log2_size = Pickles_base.Domain.log2_size domains.h in + let log2_size = log2_size ~feature_flags main in log2_size = exp_output ) table_sizes exp_output ) From 7463898230f7473b7221763eade2813eaecbea75 Mon Sep 17 00:00:00 2001 From: Gregor Date: Tue, 20 Feb 2024 15:49:50 +0100 Subject: [PATCH 08/45] adapt tx snark intf --- src/lib/transaction_snark/transaction_snark.ml | 5 +++-- src/lib/transaction_snark/transaction_snark_intf.ml | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/lib/transaction_snark/transaction_snark.ml b/src/lib/transaction_snark/transaction_snark.ml index 41f5058cf95..1ab89ddd0dd 100644 --- a/src/lib/transaction_snark/transaction_snark.ml +++ b/src/lib/transaction_snark/transaction_snark.ml @@ -3335,9 +3335,9 @@ module Make_str (A : Wire_types.Concrete) = struct val verify : (t * Sok_message.t) list -> unit Or_error.t Async.Deferred.t - val id : Pickles.Verification_key.Id.t Lazy.t + val id : Pickles.Verification_key.Id.t Async.Deferred.t Lazy.t - val verification_key : Pickles.Verification_key.t Lazy.t + val verification_key : Pickles.Verification_key.t Async.Deferred.t Lazy.t val verify_against_digest : t -> unit Or_error.t Async.Deferred.t @@ -4194,6 +4194,7 @@ module Make_str (A : Wire_types.Concrete) = struct ((), (), Pickles.Side_loaded.Proof.of_proof proof) in let vk = Pickles.Side_loaded.Verification_key.of_compiled tag in + let vk = Async.Thread_safe.block_on_async_exn (fun () -> vk) in ( `VK (With_hash.of_data ~hash_data:Zkapp_account.digest_vk vk) , `Prover trivial_prover ) diff --git a/src/lib/transaction_snark/transaction_snark_intf.ml b/src/lib/transaction_snark/transaction_snark_intf.ml index 2e63b0510c9..d55581c0c43 100644 --- a/src/lib/transaction_snark/transaction_snark_intf.ml +++ b/src/lib/transaction_snark/transaction_snark_intf.ml @@ -51,9 +51,9 @@ module type Full = sig val verify : (t * Sok_message.t) list -> unit Or_error.t Async.Deferred.t - val id : Pickles.Verification_key.Id.t Lazy.t + val id : Pickles.Verification_key.Id.t Async.Deferred.t Lazy.t - val verification_key : Pickles.Verification_key.t Lazy.t + val verification_key : Pickles.Verification_key.t Async.Deferred.t Lazy.t val verify_against_digest : t -> unit Or_error.t Async.Deferred.t From 1d7f4c27fa0439b8b825a535e4a3a843bfba3279 Mon Sep 17 00:00:00 2001 From: Gregor Date: Tue, 20 Feb 2024 15:54:45 +0100 Subject: [PATCH 09/45] adapt some tests --- .../transaction_snark_test_access_permission.ml | 4 +++- .../test/multisig_account/multisig_account.ml | 1 + src/lib/transaction_snark/test/ring_sig.ml | 1 + src/lib/transaction_snark/transaction_snark.ml | 1 + 4 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/lib/transaction_snark/test/access_permission/transaction_snark_test_access_permission.ml b/src/lib/transaction_snark/test/access_permission/transaction_snark_test_access_permission.ml index 504c3713878..f6d5c2d6e64 100644 --- a/src/lib/transaction_snark/test/access_permission/transaction_snark_test_access_permission.ml +++ b/src/lib/transaction_snark/test/access_permission/transaction_snark_test_access_permission.ml @@ -33,7 +33,9 @@ let%test_module "Access permission tests" = module P = (val p_module) - let vk = Pickles.Side_loaded.Verification_key.of_compiled tag + let vk = + let vk = Pickles.Side_loaded.Verification_key.of_compiled tag in + Async.Thread_safe.block_on_async_exn (fun () -> vk) let vk_hash = Mina_base.Verification_key_wire.digest_vk vk diff --git a/src/lib/transaction_snark/test/multisig_account/multisig_account.ml b/src/lib/transaction_snark/test/multisig_account/multisig_account.ml index 73e5892d449..ed590199008 100644 --- a/src/lib/transaction_snark/test/multisig_account/multisig_account.ml +++ b/src/lib/transaction_snark/test/multisig_account/multisig_account.ml @@ -265,6 +265,7 @@ let%test_module "multisig_account" = ] ) in let vk = Pickles.Side_loaded.Verification_key.of_compiled tag in + let vk = Async.Thread_safe.block_on_async_exn (fun () -> vk) in let { Mina_transaction_logic.For_tests.Transaction_spec.fee ; sender = sender, sender_nonce ; receiver = multisig_account_pk diff --git a/src/lib/transaction_snark/test/ring_sig.ml b/src/lib/transaction_snark/test/ring_sig.ml index 49787c15b4d..957d2287624 100644 --- a/src/lib/transaction_snark/test/ring_sig.ml +++ b/src/lib/transaction_snark/test/ring_sig.ml @@ -133,6 +133,7 @@ let%test_unit "ring-signature zkapp tx with 3 zkapp_command" = ~choices:(fun ~self:_ -> [ ring_sig_rule ring_member_pks ]) in let vk = Pickles.Side_loaded.Verification_key.of_compiled tag in + let vk = Async.Thread_safe.block_on_async_exn (fun () -> vk) in ( if debug_mode then Binable.to_string (module Side_loaded_verification_key.Stable.V2) vk |> Base64.encode_exn ~alphabet:Base64.uri_safe_alphabet diff --git a/src/lib/transaction_snark/transaction_snark.ml b/src/lib/transaction_snark/transaction_snark.ml index 1ab89ddd0dd..c4e7bff755a 100644 --- a/src/lib/transaction_snark/transaction_snark.ml +++ b/src/lib/transaction_snark/transaction_snark.ml @@ -4194,6 +4194,7 @@ module Make_str (A : Wire_types.Concrete) = struct ((), (), Pickles.Side_loaded.Proof.of_proof proof) in let vk = Pickles.Side_loaded.Verification_key.of_compiled tag in + (* we just block on the Deferred since this is only used for tests *) let vk = Async.Thread_safe.block_on_async_exn (fun () -> vk) in ( `VK (With_hash.of_data ~hash_data:Zkapp_account.digest_vk vk) , `Prover trivial_prover ) From 5a99295a2d6c60af0b9369d1ff751c37e02687bc Mon Sep 17 00:00:00 2001 From: Gregor Date: Tue, 20 Feb 2024 15:55:24 +0100 Subject: [PATCH 10/45] adapt some methods which already return a Deferred --- .../genesis_ledger_helper.ml | 13 +- src/lib/genesis_proof/genesis_proof.ml | 14 +- src/lib/prover/prover.ml | 278 +++++++++--------- 3 files changed, 153 insertions(+), 152 deletions(-) diff --git a/src/lib/genesis_ledger_helper/genesis_ledger_helper.ml b/src/lib/genesis_ledger_helper/genesis_ledger_helper.ml index fe363d11b10..e1bd238635b 100644 --- a/src/lib/genesis_ledger_helper/genesis_ledger_helper.ml +++ b/src/lib/genesis_ledger_helper/genesis_ledger_helper.ml @@ -733,15 +733,16 @@ module Genesis_proof = struct let b, id = match (inputs.blockchain_proof_system_id, inputs.proof_level) with | Some id, _ -> - (None, id) + (None, Deferred.return id) | None, Full -> let ((_, (module B)) as b) = Genesis_proof.blockchain_snark_state inputs in (Some b, Lazy.force B.Proof.id) | _ -> - (None, Pickles.Verification_key.Id.dummy ()) + (None, Deferred.return @@ Pickles.Verification_key.Id.dummy ()) in + let%bind id = id in let base_hash = Base_hash.create ~id ~state_hash: @@ -771,7 +772,7 @@ module Genesis_proof = struct let%bind found_proof = match%bind find_file ~logger ~base_hash ~genesis_dir with | Some file -> ( - match%map load file with + match%bind load file with | Ok genesis_proof -> let b = lazy @@ -791,10 +792,10 @@ module Genesis_proof = struct Lazy.force @@ Genesis_proof.digests (module T) (module B) ) in - let blockchain_proof_system_id = + let%map blockchain_proof_system_id = match inputs.blockchain_proof_system_id with | Some id -> - id + Deferred.return id | None -> let _, (module B) = Lazy.force b in Lazy.force B.Proof.id @@ -821,7 +822,7 @@ module Genesis_proof = struct [ ("path", `String file) ; ("error", Error_json.error_to_yojson err) ] ; - None ) + return None ) | None -> return None in diff --git a/src/lib/genesis_proof/genesis_proof.ml b/src/lib/genesis_proof/genesis_proof.ml index 0b7b50044fb..a7bfe26c634 100644 --- a/src/lib/genesis_proof/genesis_proof.ml +++ b/src/lib/genesis_proof/genesis_proof.ml @@ -219,7 +219,11 @@ let blockchain_snark_state (inputs : Inputs.t) : ((module T), (module B)) let create_values txn b (t : Inputs.t) = - let%map.Async.Deferred (), (), genesis_proof = base_proof b t in + let%bind.Async.Deferred (), (), genesis_proof = base_proof b t in + let%map.Async.Deferred blockchain_proof_system_id = + let (module B) = b in + Lazy.force B.Proof.id + in { runtime_config = t.runtime_config ; constraint_constants = t.constraint_constants ; proof_level = t.proof_level @@ -230,13 +234,7 @@ let create_values txn b (t : Inputs.t) = ; consensus_constants = t.consensus_constants ; protocol_state_with_hashes = t.protocol_state_with_hashes ; constraint_system_digests = digests txn b - ; proof_data = - Some - { blockchain_proof_system_id = - (let (module B) = b in - Lazy.force B.Proof.id ) - ; genesis_proof - } + ; proof_data = Some { blockchain_proof_system_id; genesis_proof } } let create_values_no_proof (t : Inputs.t) = diff --git a/src/lib/prover/prover.ml b/src/lib/prover/prover.ml index 19ef29032fa..d4886b13fa3 100644 --- a/src/lib/prover/prover.ml +++ b/src/lib/prover/prover.ml @@ -83,144 +83,146 @@ module Worker_state = struct , Lazy.force Proof.transaction_dummy ) let create { logger; proof_level; constraint_constants; _ } : t Deferred.t = - Deferred.return - (let m = - match proof_level with - | Genesis_constants.Proof_level.Full -> - ( module struct - module T = Transaction_snark.Make (struct - let constraint_constants = constraint_constants - - let proof_level = proof_level - end) - - module B = Blockchain_snark.Blockchain_snark_state.Make (struct - let tag = T.tag - - let constraint_constants = constraint_constants - - let proof_level = proof_level - end) - - let (_ : Pickles.Dirty.t) = - Pickles.Cache_handle.generate_or_load B.cache_handle - - let extend_blockchain (chain : Blockchain.t) - (next_state : Protocol_state.Value.t) - (block : Snark_transition.value) (t : Ledger_proof.t option) - state_for_handler pending_coinbase = - Internal_tracing.Context_call.with_call_id - @@ fun () -> - [%log internal] "Prover_extend_blockchain" ; - let%map.Async.Deferred res = - Deferred.Or_error.try_with ~here:[%here] (fun () -> - let txn_snark_statement, txn_snark_proof = - ledger_proof_opt next_state t - in - Internal_tracing.Context_logger.with_logger (Some logger) - @@ fun () -> - let%map.Async.Deferred (), (), proof = - B.step - ~handler: - (Consensus.Data.Prover_state.handler - ~constraint_constants state_for_handler - ~pending_coinbase ) - { transition = block - ; prev_state = - Blockchain_snark.Blockchain.state chain - ; prev_state_proof = - Blockchain_snark.Blockchain.proof chain - ; txn_snark = txn_snark_statement - ; txn_snark_proof - } - next_state - in - Blockchain_snark.Blockchain.create ~state:next_state - ~proof ) - in - [%log internal] "Prover_extend_blockchain_done" ; - Or_error.iter_error res ~f:(fun e -> - [%log error] - ~metadata:[ ("error", Error_json.error_to_yojson e) ] - "Prover threw an error while extending block: $error" ) ; - res - - let verify state proof = - Internal_tracing.Context_call.with_call_id - @@ fun () -> - [%log internal] "Prover_verify" ; - let%map result = B.Proof.verify [ (state, proof) ] in - [%log internal] "Prover_verify_done" ; - result - - let toggle_internal_tracing enabled = - don't_wait_for - @@ Internal_tracing.toggle ~logger - (if enabled then `Enabled else `Disabled) - - let set_itn_logger_data ~daemon_port = - Itn_logger.set_data ~process_kind:"prover" ~daemon_port - end : S ) - | Check -> - ( module struct - module Transaction_snark = Transaction_snark - - let extend_blockchain (chain : Blockchain.t) - (next_state : Protocol_state.Value.t) - (block : Snark_transition.value) (t : Ledger_proof.t option) - state_for_handler pending_coinbase = - let t, _proof = ledger_proof_opt next_state t in - let res = - Blockchain_snark.Blockchain_snark_state.check ~proof_level - ~constraint_constants - { transition = block - ; prev_state = Blockchain_snark.Blockchain.state chain - ; prev_state_proof = - Lazy.force Mina_base.Proof.blockchain_dummy - ; txn_snark = t - ; txn_snark_proof = - Lazy.force Mina_base.Proof.transaction_dummy - } - ~handler: - (Consensus.Data.Prover_state.handler state_for_handler - ~constraint_constants ~pending_coinbase ) - next_state - |> Or_error.map ~f:(fun () -> - Blockchain_snark.Blockchain.create ~state:next_state - ~proof:(Lazy.force Mina_base.Proof.blockchain_dummy) ) - in - Or_error.iter_error res ~f:(fun e -> - [%log error] - ~metadata:[ ("error", Error_json.error_to_yojson e) ] - "Prover threw an error while extending block: $error" ) ; - Async.Deferred.return res - - let verify _state _proof = Deferred.return (Ok ()) - - let toggle_internal_tracing _ = () - - let set_itn_logger_data ~daemon_port:_ = () - end : S ) - | None -> - ( module struct - module Transaction_snark = Transaction_snark - - let extend_blockchain _chain next_state _block _ledger_proof - _state_for_handler _pending_coinbase = - Deferred.return - @@ Ok - (Blockchain_snark.Blockchain.create - ~proof:(Lazy.force Mina_base.Proof.blockchain_dummy) - ~state:next_state ) - - let verify _ _ = Deferred.return (Ok ()) - - let toggle_internal_tracing _ = () - - let set_itn_logger_data ~daemon_port:_ = () - end : S ) - in - m ) + let m = + match proof_level with + | Genesis_constants.Proof_level.Full -> + let module T = Transaction_snark.Make (struct + let constraint_constants = constraint_constants + + let proof_level = proof_level + end) in + let module B = Blockchain_snark.Blockchain_snark_state.Make (struct + let tag = T.tag + + let constraint_constants = constraint_constants + + let proof_level = proof_level + end) in + let%map.Async.Deferred (_ : Pickles.Dirty.t) = + Pickles.Cache_handle.generate_or_load B.cache_handle + |> Promise.to_deferred + in + + ( module struct + module T = T + module B = B + + let extend_blockchain (chain : Blockchain.t) + (next_state : Protocol_state.Value.t) + (block : Snark_transition.value) (t : Ledger_proof.t option) + state_for_handler pending_coinbase = + Internal_tracing.Context_call.with_call_id + @@ fun () -> + [%log internal] "Prover_extend_blockchain" ; + let%map.Async.Deferred res = + Deferred.Or_error.try_with ~here:[%here] (fun () -> + let txn_snark_statement, txn_snark_proof = + ledger_proof_opt next_state t + in + Internal_tracing.Context_logger.with_logger (Some logger) + @@ fun () -> + let%map.Async.Deferred (), (), proof = + B.step + ~handler: + (Consensus.Data.Prover_state.handler + ~constraint_constants state_for_handler + ~pending_coinbase ) + { transition = block + ; prev_state = Blockchain_snark.Blockchain.state chain + ; prev_state_proof = + Blockchain_snark.Blockchain.proof chain + ; txn_snark = txn_snark_statement + ; txn_snark_proof + } + next_state + in + Blockchain_snark.Blockchain.create ~state:next_state ~proof ) + in + [%log internal] "Prover_extend_blockchain_done" ; + Or_error.iter_error res ~f:(fun e -> + [%log error] + ~metadata:[ ("error", Error_json.error_to_yojson e) ] + "Prover threw an error while extending block: $error" ) ; + res + + let verify state proof = + Internal_tracing.Context_call.with_call_id + @@ fun () -> + [%log internal] "Prover_verify" ; + let%map result = B.Proof.verify [ (state, proof) ] in + [%log internal] "Prover_verify_done" ; + result + + let toggle_internal_tracing enabled = + don't_wait_for + @@ Internal_tracing.toggle ~logger + (if enabled then `Enabled else `Disabled) + + let set_itn_logger_data ~daemon_port = + Itn_logger.set_data ~process_kind:"prover" ~daemon_port + end : S ) + | Check -> + Deferred.return + ( module struct + module Transaction_snark = Transaction_snark + + let extend_blockchain (chain : Blockchain.t) + (next_state : Protocol_state.Value.t) + (block : Snark_transition.value) (t : Ledger_proof.t option) + state_for_handler pending_coinbase = + let t, _proof = ledger_proof_opt next_state t in + let res = + Blockchain_snark.Blockchain_snark_state.check ~proof_level + ~constraint_constants + { transition = block + ; prev_state = Blockchain_snark.Blockchain.state chain + ; prev_state_proof = + Lazy.force Mina_base.Proof.blockchain_dummy + ; txn_snark = t + ; txn_snark_proof = + Lazy.force Mina_base.Proof.transaction_dummy + } + ~handler: + (Consensus.Data.Prover_state.handler state_for_handler + ~constraint_constants ~pending_coinbase ) + next_state + |> Or_error.map ~f:(fun () -> + Blockchain_snark.Blockchain.create ~state:next_state + ~proof:(Lazy.force Mina_base.Proof.blockchain_dummy) ) + in + Or_error.iter_error res ~f:(fun e -> + [%log error] + ~metadata:[ ("error", Error_json.error_to_yojson e) ] + "Prover threw an error while extending block: $error" ) ; + Async.Deferred.return res + + let verify _state _proof = Deferred.return (Ok ()) + + let toggle_internal_tracing _ = () + + let set_itn_logger_data ~daemon_port:_ = () + end : S ) + | None -> + Deferred.return + ( module struct + module Transaction_snark = Transaction_snark + + let extend_blockchain _chain next_state _block _ledger_proof + _state_for_handler _pending_coinbase = + Deferred.return + @@ Ok + (Blockchain_snark.Blockchain.create + ~proof:(Lazy.force Mina_base.Proof.blockchain_dummy) + ~state:next_state ) + + let verify _ _ = Deferred.return (Ok ()) + + let toggle_internal_tracing _ = () + + let set_itn_logger_data ~daemon_port:_ = () + end : S ) + in + m let get = Fn.id end From 8ea7cfd12741b11aef6eeb659fa081eafc95f8c8 Mon Sep 17 00:00:00 2001 From: Gregor Date: Tue, 20 Feb 2024 16:40:23 +0100 Subject: [PATCH 11/45] fixup zkapp examples build --- .../print_blockchain_snark_vk.ml | 3 ++- src/app/test_executive/verification_key_update.ml | 8 ++++++-- src/app/zkapps_examples/test/actions/actions.ml | 4 +++- src/app/zkapps_examples/test/add_events/add_events.ml | 4 +++- src/app/zkapps_examples/test/big_circuit/big_circuit.ml | 3 ++- src/app/zkapps_examples/test/calls/calls.ml | 4 +++- src/app/zkapps_examples/test/empty_update/empty_update.ml | 4 +++- .../test/initialize_state/initialize_state.ml | 4 +++- src/app/zkapps_examples/test/tokens/tokens.ml | 4 +++- src/app/zkapps_examples/tokens/zkapps_tokens.ml | 6 ++++++ src/app/zkapps_examples/tokens/zkapps_tokens.mli | 2 +- 11 files changed, 35 insertions(+), 11 deletions(-) diff --git a/src/app/print_blockchain_snark_vk/print_blockchain_snark_vk.ml b/src/app/print_blockchain_snark_vk/print_blockchain_snark_vk.ml index c111eacabd3..24492fd7b95 100644 --- a/src/app/print_blockchain_snark_vk/print_blockchain_snark_vk.ml +++ b/src/app/print_blockchain_snark_vk/print_blockchain_snark_vk.ml @@ -36,6 +36,7 @@ let () = (Time.Span.to_string_hum (Time.diff after before)) let () = - Lazy.force Blockchain_snark_instance.Proof.verification_key + Async.Thread_safe.block_on_async_exn (fun () -> + Lazy.force Blockchain_snark_instance.Proof.verification_key ) |> Pickles.Verification_key.to_yojson |> Yojson.Safe.to_string |> Format.print_string diff --git a/src/app/test_executive/verification_key_update.ml b/src/app/test_executive/verification_key_update.ml index cefdb987952..608c16ba2e5 100644 --- a/src/app/test_executive/verification_key_update.ml +++ b/src/app/test_executive/verification_key_update.ml @@ -143,8 +143,12 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct constraint_constants ) ~choices:(fun ~self:_ -> [ Trivial_rule2.rule ]) in - let vk1 = Pickles.Side_loaded.Verification_key.of_compiled tag1 in - let vk2 = Pickles.Side_loaded.Verification_key.of_compiled tag2 in + let%bind.Async.Deferred vk1 = + Pickles.Side_loaded.Verification_key.of_compiled tag1 + in + let%bind.Async.Deferred vk2 = + Pickles.Side_loaded.Verification_key.of_compiled tag2 + in let%bind.Async.Deferred account_update1, _ = trivial_prover1 ~handler:Trivial_rule1.handler () in diff --git a/src/app/zkapps_examples/test/actions/actions.ml b/src/app/zkapps_examples/test/actions/actions.ml index d49f985b8ac..4e70f82d863 100644 --- a/src/app/zkapps_examples/test/actions/actions.ml +++ b/src/app/zkapps_examples/test/actions/actions.ml @@ -36,7 +36,9 @@ let%test_module "Actions test" = module P = (val p_module) - let vk = Pickles.Side_loaded.Verification_key.of_compiled tag + let vk = + Async.Thread_safe.block_on_async_exn (fun () -> + Pickles.Side_loaded.Verification_key.of_compiled tag ) module Deploy_account_update = struct let account_update_body : Account_update.Body.t = diff --git a/src/app/zkapps_examples/test/add_events/add_events.ml b/src/app/zkapps_examples/test/add_events/add_events.ml index 1c1fda7cec8..149332996de 100644 --- a/src/app/zkapps_examples/test/add_events/add_events.ml +++ b/src/app/zkapps_examples/test/add_events/add_events.ml @@ -36,7 +36,9 @@ let%test_module "Add events test" = module P = (val p_module) - let vk = Pickles.Side_loaded.Verification_key.of_compiled tag + let vk = + Async.Thread_safe.block_on_async_exn (fun () -> + Pickles.Side_loaded.Verification_key.of_compiled tag ) module Deploy_account_update = struct let account_update_body : Account_update.Body.t = diff --git a/src/app/zkapps_examples/test/big_circuit/big_circuit.ml b/src/app/zkapps_examples/test/big_circuit/big_circuit.ml index 7e0243402fe..de34f927e29 100644 --- a/src/app/zkapps_examples/test/big_circuit/big_circuit.ml +++ b/src/app/zkapps_examples/test/big_circuit/big_circuit.ml @@ -31,7 +31,8 @@ let tag, _cache, _p_module, Pickles.Provers.[ prover ] = [ Zkapps_big_circuit.rule ~num_constraints pk_compressed ] ) let vk : Side_loaded_verification_key.t = - Pickles.Side_loaded.Verification_key.of_compiled tag + Async.Thread_safe.block_on_async_exn (fun () -> + Pickles.Side_loaded.Verification_key.of_compiled tag ) let vk_hash = Mina_base.Verification_key_wire.digest_vk vk diff --git a/src/app/zkapps_examples/test/calls/calls.ml b/src/app/zkapps_examples/test/calls/calls.ml index 2a22ee3571a..8111b47ce86 100644 --- a/src/app/zkapps_examples/test/calls/calls.ml +++ b/src/app/zkapps_examples/test/calls/calls.ml @@ -91,7 +91,9 @@ let%test_module "Composability test" = module P = (val p_module) - let vk = Pickles.Side_loaded.Verification_key.of_compiled tag + let vk = + Async.Thread_safe.block_on_async_exn (fun () -> + Pickles.Side_loaded.Verification_key.of_compiled tag ) module Deploy_account_update = struct let account_update_body : Account_update.Body.t = diff --git a/src/app/zkapps_examples/test/empty_update/empty_update.ml b/src/app/zkapps_examples/test/empty_update/empty_update.ml index dbfc64f9eb1..89b8e4b8f92 100644 --- a/src/app/zkapps_examples/test/empty_update/empty_update.ml +++ b/src/app/zkapps_examples/test/empty_update/empty_update.ml @@ -28,7 +28,9 @@ let tag, _, p_module, Pickles.Provers.[ prover ] = module P = (val p_module) -let vk = Pickles.Side_loaded.Verification_key.of_compiled tag +let vk = + Async.Thread_safe.block_on_async_exn (fun () -> + Pickles.Side_loaded.Verification_key.of_compiled tag ) let account_update, () = Async.Thread_safe.block_on_async_exn prover diff --git a/src/app/zkapps_examples/test/initialize_state/initialize_state.ml b/src/app/zkapps_examples/test/initialize_state/initialize_state.ml index 229ec2ade03..c44a901fd79 100644 --- a/src/app/zkapps_examples/test/initialize_state/initialize_state.ml +++ b/src/app/zkapps_examples/test/initialize_state/initialize_state.ml @@ -40,7 +40,9 @@ let%test_module "Initialize state test" = module P = (val p_module) - let vk = Pickles.Side_loaded.Verification_key.of_compiled tag + let vk = + Async.Thread_safe.block_on_async_exn (fun () -> + Pickles.Side_loaded.Verification_key.of_compiled tag ) module Deploy_account_update = struct let account_update_body : Account_update.Body.t = diff --git a/src/app/zkapps_examples/test/tokens/tokens.ml b/src/app/zkapps_examples/test/tokens/tokens.ml index 0eae247a307..fbab323fd0d 100644 --- a/src/app/zkapps_examples/test/tokens/tokens.ml +++ b/src/app/zkapps_examples/test/tokens/tokens.ml @@ -42,7 +42,9 @@ let%test_module "Tokens test" = let owned_token_id = Account_id.derive_token_id ~owner:account_id - let vk = Lazy.force Zkapps_tokens.vk + let vk = + Async.Thread_safe.block_on_async_exn (fun () -> + Lazy.force Zkapps_tokens.vk ) let mint_to_keys = gen_keys () diff --git a/src/app/zkapps_examples/tokens/zkapps_tokens.ml b/src/app/zkapps_examples/tokens/zkapps_tokens.ml index e152b04718c..81927db8eb1 100644 --- a/src/app/zkapps_examples/tokens/zkapps_tokens.ml +++ b/src/app/zkapps_examples/tokens/zkapps_tokens.ml @@ -691,8 +691,14 @@ module P = struct let verification_key = Lazy.bind p_module ~f:(fun (module P : Proof_intf) -> P.verification_key) + (* this is annoying *) + let verification_key_promise = lazy (failwith "not implemented") + let id = Lazy.bind p_module ~f:(fun (module P : Proof_intf) -> P.id) + (* this is annoying *) + let id_promise = lazy (failwith "not implemented") + let verify statements = let module P : Proof_intf = (val Lazy.force p_module) in P.verify statements diff --git a/src/app/zkapps_examples/tokens/zkapps_tokens.mli b/src/app/zkapps_examples/tokens/zkapps_tokens.mli index 61132585305..7ee5eaa9406 100644 --- a/src/app/zkapps_examples/tokens/zkapps_tokens.mli +++ b/src/app/zkapps_examples/tokens/zkapps_tokens.mli @@ -12,7 +12,7 @@ val tag : Pickles.Tag.t Lazy.t -val vk : Pickles.Side_loaded.Verification_key.t Lazy.t +val vk : Pickles.Side_loaded.Verification_key.t Async.Deferred.t Lazy.t module P : Pickles.Proof_intf From 40c10bbf44e5af0dfb9dc6e437eed6e4be9e86cf Mon Sep 17 00:00:00 2001 From: Gregor Date: Tue, 20 Feb 2024 16:41:19 +0100 Subject: [PATCH 12/45] missing dune file change --- src/lib/prover/dune | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lib/prover/dune b/src/lib/prover/dune index 9b425c25139..473c3a0c899 100644 --- a/src/lib/prover/dune +++ b/src/lib/prover/dune @@ -50,6 +50,7 @@ kimchi_backend.pasta kimchi_backend.pasta.basic mina_wire_types + promise ) (preprocessor_deps "../../config.mlh") (instrumentation (backend bisect_ppx)) From 8e75713be50b2827f86c980080215f1ae5cc2054 Mon Sep 17 00:00:00 2001 From: Gregor Date: Thu, 22 Feb 2024 17:29:53 +0100 Subject: [PATCH 13/45] progress: fix_domains runs circuits in order --- src/lib/pickles/cache.ml | 3 +- src/lib/pickles/cache.mli | 2 +- src/lib/pickles/compile.ml | 46 ++++++++++++++++++++-------- src/lib/pickles/pickles.ml | 32 +++++++++++++++---- src/lib/pickles/step.ml | 5 ++- src/lib/pickles/step.mli | 2 +- src/lib/pickles/step_branch_data.ml | 31 +++---------------- src/lib/pickles/step_branch_data.mli | 4 +-- 8 files changed, 72 insertions(+), 53 deletions(-) diff --git a/src/lib/pickles/cache.ml b/src/lib/pickles/cache.ml index 33d2c8c061b..1ecadd1cc4a 100644 --- a/src/lib/pickles/cache.ml +++ b/src/lib/pickles/cache.ml @@ -92,7 +92,7 @@ module Step = struct header path ) ) let read_or_generate ~prev_challenges cache ?(s_p = storable) k_p - ?(s_v = vk_storable) k_v typ return_typ main = + ?(s_v = vk_storable) k_v typ return_typ main_promise = let open Impls.Step in let pk = lazy @@ -107,6 +107,7 @@ module Step = struct | Error _e -> let%map.Promise r = Common.time "stepkeygen" (fun () -> + let%bind.Promise main = main_promise in let%map.Promise constraint_system = let constraint_builder = constraint_system_manual ~input_typ:typ ~return_typ diff --git a/src/lib/pickles/cache.mli b/src/lib/pickles/cache.mli index e16d7dc0a74..6f8b401bb57 100644 --- a/src/lib/pickles/cache.mli +++ b/src/lib/pickles/cache.mli @@ -45,7 +45,7 @@ module Step : sig -> Key.Verification.t Promise.t Lazy.t -> ('a, 'b) Impls.Step.Typ.t -> ('c, 'd) Impls.Step.Typ.t - -> ('a -> unit -> 'c Promise.t) + -> ('a -> unit -> 'c Promise.t) Promise.t -> ( Impls.Step.Proving_key.t * ([> `Cache_hit | `Generated_something | `Locally_generated ] as 'e) ) Promise.t diff --git a/src/lib/pickles/compile.ml b/src/lib/pickles/compile.ml index 12399ad0e28..95bda22cc5c 100644 --- a/src/lib/pickles/compile.ml +++ b/src/lib/pickles/compile.ml @@ -305,7 +305,7 @@ struct * (Kimchi_bindings.Protocol.VerifierIndex.Fp.t * Dirty.t) Promise.t Lazy.t end - let log_step main _typ name index = + let _log_step main _typ name index = let module Constraints = Snarky_log.Constraints (Impls.Step.Internal_Basic) in let log = let weight = @@ -522,10 +522,11 @@ struct Timer.clock __LOC__ ; let rec f : type a b c d. - (a, b, c, d) H4.T(IR).t -> (a, b, c, d) H4.T(Branch_data).t = function - | [] -> + (a, b, c, d) H4.T(IR).t * unit Promise.t + -> (a, b, c, d) H4.T(Branch_data).t = function + | [], _ -> [] - | rule :: rules -> + | rule :: rules, chain_to -> let first = Timer.clock __LOC__ ; let res = @@ -535,13 +536,15 @@ struct ~max_proofs_verified:Max_proofs_verified.n ~branches:Branches.n ~self ~public_input ~auxiliary_typ Arg_var.to_field_elements Arg_value.to_field_elements rule - ~wrap_domains ~proofs_verifieds ) + ~wrap_domains ~proofs_verifieds ~chain_to ) in Timer.clock __LOC__ ; incr i ; res in - first :: f rules + let (T b) = first in + let chain_to = Promise.map b.domains ~f:(fun _ -> ()) in + first :: f (rules, chain_to) in - f choices + f (choices, Promise.return ()) in Timer.clock __LOC__ ; let step_domains = @@ -557,6 +560,18 @@ struct let module V = H4.To_vector (DomainsPromise) in V.f prev_varss_length (M.f step_data) in + + let all_step_domains = + let%map.Promise () = + (* Wait for promises to resolve. *) + Vector.fold ~init:(Promise.return ()) step_domains + ~f:(fun acc step_domain -> + let%bind.Promise _ = step_domain in + acc ) + in + Vector.map ~f:(fun x -> Option.value_exn @@ Promise.peek x) step_domains + in + let cache_handle = ref (Lazy.return (Promise.return `Cache_hit)) in let accum_dirty t = cache_handle := Cache_handle.(!cache_handle + t) in Timer.clock __LOC__ ; @@ -573,11 +588,15 @@ struct let f (T b : _ Branch_data.t) = let (T (typ, _conv, conv_inv)) = etyp in - let main () () = - let%map.Promise res = b.main ~step_domains () in - Impls.Step.with_label "conv_inv" (fun () -> conv_inv res) + let main_promise : (unit -> unit -> _ Promise.t) Promise.t = + let%map.Promise step_domains = all_step_domains in + let main () () = + let%map.Promise res = b.main ~step_domains () in + Impls.Step.with_label "conv_inv" (fun () -> conv_inv res) + in + main in - let () = if true then log_step main typ name b.index in + (* let () = if true then log_step main typ name b.index in *) let open Impls.Step in (* HACK: TODO docs *) if return_early_digest_exception then failwith "TODO: Delete me" ; @@ -585,6 +604,7 @@ struct let k_p = lazy (let%map.Promise cs = + let%bind.Promise main = main_promise in let constraint_builder = Impl.constraint_system_manual ~input_typ:Typ.unit ~return_typ:typ @@ -631,7 +651,7 @@ struct ~prev_challenges:(Nat.to_int (fst b.proofs_verified)) cache ~s_p:step_storable k_p ~s_v:step_vk_storable k_v (Snarky_backendless.Typ.unit ()) - typ main ) + typ main_promise ) in accum_dirty (Lazy.map pk ~f:(Promise.map ~f:snd)) ; accum_dirty (Lazy.map vk ~f:(Promise.map ~f:snd)) ; @@ -784,7 +804,7 @@ struct let%bind.Promise wrap_vk = Lazy.force wrap_vk in let%bind.Promise step_pk = Lazy.force step_pk in S.f ?handler branch_data next_state ~prevs_length:prev_vars_length - ~self ~step_domains + ~self ~step_domains:all_step_domains ~self_dlog_plonk_index: ((* TODO *) Plonk_verification_key_evals.map ~f:(fun x -> [| x |]) diff --git a/src/lib/pickles/pickles.ml b/src/lib/pickles/pickles.ml index 18dbff19748..e952ed3c317 100644 --- a/src/lib/pickles/pickles.ml +++ b/src/lib/pickles/pickles.ml @@ -1229,23 +1229,42 @@ module Make_str (_ : Wire_types.Concrete) = struct ~actual_feature_flags ~max_proofs_verified:Max_proofs_verified.n ~branches:Branches.n ~self ~public_input:(Input typ) ~auxiliary_typ:typ A.to_field_elements A_value.to_field_elements - rule ~wrap_domains ~proofs_verifieds + rule ~wrap_domains ~proofs_verifieds ~chain_to:(Promise.return ()) + (* TODO? *) in let step_domains = Vector.singleton inner_step_data.domains in + let all_step_domains = + let%map.Promise () = + (* Wait for promises to resolve. *) + Vector.fold ~init:(Promise.return ()) step_domains + ~f:(fun acc step_domain -> + let%bind.Promise _ = step_domain in + acc ) + in + Vector.map + ~f:(fun x -> Option.value_exn @@ Promise.peek x) + step_domains + in + let step_keypair = let etyp = Impls.Step.input ~proofs_verified:Max_proofs_verified.n ~wrap_rounds:Tock.Rounds.n in let (T (typ, _conv, conv_inv)) = etyp in - let main () () = - let%map.Promise res = inner_step_data.main ~step_domains () in - Impls.Step.with_label "conv_inv" (fun () -> conv_inv res) + let main_promise : (unit -> unit -> _ Promise.t) Promise.t = + let%map.Promise step_domains = all_step_domains in + let main () () = + let%map.Promise res = inner_step_data.main ~step_domains () in + Impls.Step.with_label "conv_inv" (fun () -> conv_inv res) + in + main in let open Impls.Step in let k_p = lazy (let%map.Promise cs = + let%bind.Promise main = main_promise in let constraint_builder = Impl.constraint_system_manual ~input_typ:Typ.unit ~return_typ:typ @@ -1281,7 +1300,7 @@ module Make_str (_ : Wire_types.Concrete) = struct (Nat.to_int (fst inner_step_data.proofs_verified)) [] k_p k_v (Snarky_backendless.Typ.unit ()) - typ main + typ main_promise in let step_vks = lazy @@ -1384,7 +1403,8 @@ module Make_str (_ : Wire_types.Concrete) = struct S.f branch_data ~feature_flags ~prevs_length:prev_vars_length ~self ~public_input:(Input typ) ~proof_cache:None ~maxes:(module Maxes) - ~auxiliary_typ:Impls.Step.Typ.unit ~step_domains + ~auxiliary_typ:Impls.Step.Typ.unit + ~step_domains:all_step_domains ~self_dlog_plonk_index: ((* TODO *) Plonk_verification_key_evals.map ~f:(fun x -> [| x |]) diff --git a/src/lib/pickles/step.ml b/src/lib/pickles/step.ml index cf122da2e14..160494f82f6 100644 --- a/src/lib/pickles/step.ml +++ b/src/lib/pickles/step.ml @@ -804,9 +804,8 @@ struct Impls.Step.input ~proofs_verified:Max_proofs_verified.n ~wrap_rounds:Tock.Rounds.n in - let%bind.Promise { Domains.h } = - Vector.nth_exn step_domains branch_data.index - in + let%bind.Promise step_domains = step_domains in + let { Domains.h } = Vector.nth_exn step_domains branch_data.index in ksprintf Common.time "step-prover %d (%d)" branch_data.index (Domain.size h) (fun () -> [%log internal] "Step_generate_witness_conv" ; diff --git a/src/lib/pickles/step.mli b/src/lib/pickles/step.mli index 647c3acd4e5..83a16dd1f64 100644 --- a/src/lib/pickles/step.mli +++ b/src/lib/pickles/step.mli @@ -31,7 +31,7 @@ module Make -> prevs_length:('prev_vars, 'prevs_length) Pickles_types.Hlist.Length.t -> self:('a, 'b, 'c, 'd) Tag.t -> step_domains: - (Import.Domains.t Promise.t, 'self_branches) Pickles_types.Vector.t + (Import.Domains.t, 'self_branches) Pickles_types.Vector.t Promise.t -> feature_flags:Opt.Flag.t Plonk_types.Features.Full.t -> self_dlog_plonk_index: Backend.Tick.Inner_curve.Affine.t array diff --git a/src/lib/pickles/step_branch_data.ml b/src/lib/pickles/step_branch_data.ml index 0aaaaba8d01..377cb09ab03 100644 --- a/src/lib/pickles/step_branch_data.ml +++ b/src/lib/pickles/step_branch_data.ml @@ -36,7 +36,7 @@ type ( 'a_var , 'auxiliary_value ) Inductive_rule.Promise.t ; main : - step_domains:(Domains.t Promise.t, 'branches) Vector.t + step_domains:(Domains.t, 'branches) Vector.t -> unit -> ( (Unfinalized.t, 'max_proofs_verified) Vector.t , Impls.Step.Field.t @@ -85,7 +85,8 @@ let create , ret_var , ret_value ) Inductive_rule.public_input ) ~auxiliary_typ _var_to_field_elements - _value_to_field_elements (rule : _ Inductive_rule.Promise.t) = + _value_to_field_elements ~(chain_to : unit Promise.t) + (rule : _ Inductive_rule.Promise.t) = Timer.clock __LOC__ ; let module HT = H4.T (Tag) in let (T (self_width, proofs_verified)) = HT.length rule.prevs in @@ -133,16 +134,6 @@ let create in Timer.clock __LOC__ ; let step ~step_domains = - let%map.Promise step_domains = - let%map.Promise () = - (* Wait for promises to resolve. *) - Vector.fold ~init:(Promise.return ()) step_domains - ~f:(fun acc step_domain -> - let%bind.Promise _ = step_domain in - acc ) - in - Vector.map ~f:(fun x -> Option.value_exn @@ Promise.peek x) step_domains - in Step_main.step_main requests (Nat.Add.create max_proofs_verified) rule @@ -158,32 +149,20 @@ let create ~local_branches_length ~lte ~self |> unstage in - (* Rebinding trick: we set off the outer promise immediately, as we would if - it was not there, and then we chain off the promise in the main function - itself. - Notably, this ensures that we don't initialize on every call of the - function, which is the whole idea of the [stage] above. - *) - let step ~step_domains = - let step = step ~step_domains in - fun () -> - let%bind.Promise step = step in - step () - in (* Now that we've triggered the promise computation, we rebind. *) Timer.clock __LOC__ ; let own_domains = let main = step ~step_domains: - (Vector.init branches ~f:(fun _ -> - Promise.return Fix_domains.rough_domains ) ) + (Vector.init branches ~f:(fun _ -> Fix_domains.rough_domains)) in let etyp = Impls.Step.input ~proofs_verified:max_proofs_verified ~wrap_rounds:Backend.Tock.Rounds.n (* TODO *) in + let%bind.Promise () = chain_to in Fix_domains.domains ~feature_flags:actual_feature_flags (module Impls.Step) (T (Snarky_backendless.Typ.unit (), Fn.id, Fn.id)) diff --git a/src/lib/pickles/step_branch_data.mli b/src/lib/pickles/step_branch_data.mli index 4ac5366284c..34a3ce2ce99 100644 --- a/src/lib/pickles/step_branch_data.mli +++ b/src/lib/pickles/step_branch_data.mli @@ -38,8 +38,7 @@ type ( 'a_var Inductive_rule.Promise.t (* Main functions to compute *) ; main : - step_domains: - (Import.Domains.t Promise.t, 'branches) Pickles_types.Vector.t + step_domains:(Import.Domains.t, 'branches) Pickles_types.Vector.t -> unit -> ( (Unfinalized.t, 'max_proofs_verified) Pickles_types.Vector.t , Impls.Step.Field.t @@ -101,6 +100,7 @@ val create : -> auxiliary_typ:('a, 'b) Impls.Step.Typ.t -> 'c -> 'd + -> chain_to:unit Promise.t -> ( 'e , 'f , 'g From d3e025beef1a160feb1801b7de3da1e92122545b Mon Sep 17 00:00:00 2001 From: Gregor Date: Thu, 22 Feb 2024 18:17:14 +0100 Subject: [PATCH 14/45] work towards running circuits for key generation in sequence --- src/lib/pickles/compile.ml | 166 +++++++++++++++++++------------------ 1 file changed, 87 insertions(+), 79 deletions(-) diff --git a/src/lib/pickles/compile.ml b/src/lib/pickles/compile.ml index 95bda22cc5c..1462434b874 100644 --- a/src/lib/pickles/compile.ml +++ b/src/lib/pickles/compile.ml @@ -579,87 +579,95 @@ struct let disk_keys = Option.map disk_keys ~f:(fun (xs, _) -> Vector.to_array xs) in - let module M = - H4.Map (Branch_data) (E04 (Lazy_keys)) - (struct - let etyp = - Impls.Step.input ~proofs_verified:Max_proofs_verified.n - ~wrap_rounds:Tock.Rounds.n - - let f (T b : _ Branch_data.t) = - let (T (typ, _conv, conv_inv)) = etyp in - let main_promise : (unit -> unit -> _ Promise.t) Promise.t = - let%map.Promise step_domains = all_step_domains in - let main () () = - let%map.Promise res = b.main ~step_domains () in - Impls.Step.with_label "conv_inv" (fun () -> conv_inv res) - in - main - in - (* let () = if true then log_step main typ name b.index in *) - let open Impls.Step in - (* HACK: TODO docs *) - if return_early_digest_exception then failwith "TODO: Delete me" ; - - let k_p = - lazy - (let%map.Promise cs = - let%bind.Promise main = main_promise in - let constraint_builder = - Impl.constraint_system_manual ~input_typ:Typ.unit - ~return_typ:typ - in - let%map.Promise res = - constraint_builder.run_circuit main - in - constraint_builder.finish_computation res - in - let cs_hash = - Md5.to_hex (R1CS_constraint_system.digest cs) - in - ( Type_equal.Id.uid self.id - , snark_keys_header - { type_ = "step-proving-key" - ; identifier = name ^ "-" ^ b.rule.identifier - } - cs_hash - , b.index - , cs ) ) - in - let k_v = - match disk_keys with - | Some ks -> - Lazy.return (Promise.return ks.(b.index)) - | None -> - lazy - (let%map.Promise id, _header, index, cs = - Lazy.force k_p - in - let digest = R1CS_constraint_system.digest cs in - ( id - , snark_keys_header - { type_ = "step-verification-key" - ; identifier = name ^ "-" ^ b.rule.identifier - } - (Md5.to_hex digest) - , index - , digest ) ) - in - let ((pk, vk) as res) = - Common.time "step read or generate" (fun () -> - Cache.Step.read_or_generate - ~prev_challenges:(Nat.to_int (fst b.proofs_verified)) - cache ~s_p:step_storable k_p ~s_v:step_vk_storable k_v - (Snarky_backendless.Typ.unit ()) - typ main_promise ) - in - accum_dirty (Lazy.map pk ~f:(Promise.map ~f:snd)) ; - accum_dirty (Lazy.map vk ~f:(Promise.map ~f:snd)) ; - res - end) + let etyp = + Impls.Step.input ~proofs_verified:Max_proofs_verified.n + ~wrap_rounds:Tock.Rounds.n + in + let create_keys (T b : _ Branch_data.t) chain_to = + let (T (typ, _conv, conv_inv)) = etyp in + let main_promise : (unit -> unit -> _ Promise.t) Promise.t = + let%bind.Promise step_domains = all_step_domains in + let%map.Promise () = chain_to in + let main () () = + let%map.Promise res = b.main ~step_domains () in + Impls.Step.with_label "conv_inv" (fun () -> conv_inv res) + in + main + in + (* let () = if true then log_step main typ name b.index in *) + let open Impls.Step in + (* HACK: TODO docs *) + if return_early_digest_exception then failwith "TODO: Delete me" ; + + let stop_waiting_for_circuit = ref (fun () -> ()) in + let next_chain_to = + Promise.create (fun resolve -> stop_waiting_for_circuit := resolve) + in + + let k_p = + lazy + (let%map.Promise cs = + let%bind.Promise main = main_promise in + let constraint_builder = + Impl.constraint_system_manual ~input_typ:Typ.unit + ~return_typ:typ + in + let%map.Promise res = constraint_builder.run_circuit main in + constraint_builder.finish_computation res + in + !stop_waiting_for_circuit () ; + let cs_hash = Md5.to_hex (R1CS_constraint_system.digest cs) in + ( Type_equal.Id.uid self.id + , snark_keys_header + { type_ = "step-proving-key" + ; identifier = name ^ "-" ^ b.rule.identifier + } + cs_hash + , b.index + , cs ) ) + in + let k_v = + match disk_keys with + | Some ks -> + Lazy.return (Promise.return ks.(b.index)) + | None -> + lazy + (let%map.Promise id, _header, index, cs = Lazy.force k_p in + let digest = R1CS_constraint_system.digest cs in + ( id + , snark_keys_header + { type_ = "step-verification-key" + ; identifier = name ^ "-" ^ b.rule.identifier + } + (Md5.to_hex digest) + , index + , digest ) ) + in + let ((pk, vk) as res) = + Common.time "step read or generate" (fun () -> + Cache.Step.read_or_generate (* ~stop_waiting_for_circuit *) + ~prev_challenges:(Nat.to_int (fst b.proofs_verified)) + cache ~s_p:step_storable k_p ~s_v:step_vk_storable k_v + (Snarky_backendless.Typ.unit ()) + typ main_promise ) + in + accum_dirty (Lazy.map pk ~f:(Promise.map ~f:snd)) ; + accum_dirty (Lazy.map vk ~f:(Promise.map ~f:snd)) ; + (res, next_chain_to) in - M.f step_data + let rec f : + type a b c d. + (a, b, c, d) H4.T(Branch_data).t * unit Promise.t + -> (a, b, c, d) H4.T(E04(Lazy_keys)).t = function + | [], _ -> + [] + | data :: rest, chain_to -> + let first, chain_to = create_keys data chain_to in + first :: f (rest, chain_to) + in + f (step_data, Promise.return ()) in + Timer.clock __LOC__ ; let step_vks = let module V = H4.To_vector (Lazy_keys) in From eebd9a84fe683535b3f55faceaf71597d832d426 Mon Sep 17 00:00:00 2001 From: Gregor Date: Thu, 22 Feb 2024 21:41:36 +0100 Subject: [PATCH 15/45] run stuff in sequence, but for some reason it doesn't work yet --- src/lib/pickles/cache.ml | 20 +++++++------ src/lib/pickles/cache.mli | 5 +++- src/lib/pickles/compile.ml | 57 +++++++++++++++++++++++--------------- src/lib/pickles/pickles.ml | 1 + 4 files changed, 50 insertions(+), 33 deletions(-) diff --git a/src/lib/pickles/cache.ml b/src/lib/pickles/cache.ml index 1ecadd1cc4a..34672d1fff9 100644 --- a/src/lib/pickles/cache.ml +++ b/src/lib/pickles/cache.ml @@ -91,8 +91,9 @@ module Step = struct (Kimchi_bindings.Protocol.VerifierIndex.Fp.write (Some true) x) header path ) ) - let read_or_generate ~prev_challenges cache ?(s_p = storable) k_p - ?(s_v = vk_storable) k_v typ return_typ main_promise = + let read_or_generate ~(run_in_sequence : (unit -> _ Promise.t) -> _ Promise.t) + ~prev_challenges cache ?(s_p = storable) k_p ?(s_v = vk_storable) k_v typ + return_typ main_promise = let open Impls.Step in let pk = lazy @@ -109,13 +110,14 @@ module Step = struct Common.time "stepkeygen" (fun () -> let%bind.Promise main = main_promise in let%map.Promise constraint_system = - let constraint_builder = - constraint_system_manual ~input_typ:typ ~return_typ - in - let%map.Promise res = - constraint_builder.run_circuit main - in - constraint_builder.finish_computation res + run_in_sequence (fun () -> + let constraint_builder = + constraint_system_manual ~input_typ:typ ~return_typ + in + let%map.Promise res = + constraint_builder.run_circuit main + in + constraint_builder.finish_computation res ) in constraint_system |> Keypair.generate ~prev_challenges ) in diff --git a/src/lib/pickles/cache.mli b/src/lib/pickles/cache.mli index 6f8b401bb57..40a35febb9c 100644 --- a/src/lib/pickles/cache.mli +++ b/src/lib/pickles/cache.mli @@ -37,7 +37,10 @@ module Step : sig val vk_storable : vk_storable val read_or_generate : - prev_challenges:int + run_in_sequence: + ( (unit -> Backend.Tick.R1CS_constraint_system.t Promise.t) + -> Backend.Tick.R1CS_constraint_system.t Promise.t ) + -> prev_challenges:int -> Key_cache.Spec.t list -> ?s_p:storable -> Key.Proving.t Promise.t Lazy.t diff --git a/src/lib/pickles/compile.ml b/src/lib/pickles/compile.ml index 1462434b874..f1bc3ca0ef7 100644 --- a/src/lib/pickles/compile.ml +++ b/src/lib/pickles/compile.ml @@ -250,6 +250,23 @@ module Storables = struct } end +let create_lock () = + let lock = ref (Promise.return ()) in + let open Promise.Let_syntax in + let run_in_sequence (f : unit -> 'a Promise.t) : 'a Promise.t = + (* await the current lock *) + let%bind () = !lock in + (* create a new lock *) + let unlock = ref (fun () -> ()) in + lock := Promise.create (fun resolve -> unlock := resolve) ; + (* run the function and unlock *) + try + let%map res = f () in + !unlock () ; res + with exn -> !unlock () ; raise exn + in + run_in_sequence + module Make (Arg_var : Statement_var_intf) (Arg_value : Statement_value_intf) @@ -571,6 +588,7 @@ struct in Vector.map ~f:(fun x -> Option.value_exn @@ Promise.peek x) step_domains in + let run_in_sequence = create_lock () in let cache_handle = ref (Lazy.return (Promise.return `Cache_hit)) in let accum_dirty t = cache_handle := Cache_handle.(!cache_handle + t) in @@ -583,11 +601,10 @@ struct Impls.Step.input ~proofs_verified:Max_proofs_verified.n ~wrap_rounds:Tock.Rounds.n in - let create_keys (T b : _ Branch_data.t) chain_to = + let create_keys (T b : _ Branch_data.t) = let (T (typ, _conv, conv_inv)) = etyp in let main_promise : (unit -> unit -> _ Promise.t) Promise.t = - let%bind.Promise step_domains = all_step_domains in - let%map.Promise () = chain_to in + let%map.Promise step_domains = all_step_domains in let main () () = let%map.Promise res = b.main ~step_domains () in Impls.Step.with_label "conv_inv" (fun () -> conv_inv res) @@ -599,23 +616,18 @@ struct (* HACK: TODO docs *) if return_early_digest_exception then failwith "TODO: Delete me" ; - let stop_waiting_for_circuit = ref (fun () -> ()) in - let next_chain_to = - Promise.create (fun resolve -> stop_waiting_for_circuit := resolve) - in - let k_p = lazy (let%map.Promise cs = let%bind.Promise main = main_promise in - let constraint_builder = - Impl.constraint_system_manual ~input_typ:Typ.unit - ~return_typ:typ - in - let%map.Promise res = constraint_builder.run_circuit main in - constraint_builder.finish_computation res + run_in_sequence (fun () -> + let constraint_builder = + Impl.constraint_system_manual ~input_typ:Typ.unit + ~return_typ:typ + in + let%map.Promise res = constraint_builder.run_circuit main in + constraint_builder.finish_computation res ) in - !stop_waiting_for_circuit () ; let cs_hash = Md5.to_hex (R1CS_constraint_system.digest cs) in ( Type_equal.Id.uid self.id , snark_keys_header @@ -645,7 +657,7 @@ struct in let ((pk, vk) as res) = Common.time "step read or generate" (fun () -> - Cache.Step.read_or_generate (* ~stop_waiting_for_circuit *) + Cache.Step.read_or_generate ~run_in_sequence ~prev_challenges:(Nat.to_int (fst b.proofs_verified)) cache ~s_p:step_storable k_p ~s_v:step_vk_storable k_v (Snarky_backendless.Typ.unit ()) @@ -653,19 +665,18 @@ struct in accum_dirty (Lazy.map pk ~f:(Promise.map ~f:snd)) ; accum_dirty (Lazy.map vk ~f:(Promise.map ~f:snd)) ; - (res, next_chain_to) + res in let rec f : type a b c d. - (a, b, c, d) H4.T(Branch_data).t * unit Promise.t + (a, b, c, d) H4.T(Branch_data).t -> (a, b, c, d) H4.T(E04(Lazy_keys)).t = function - | [], _ -> + | [] -> [] - | data :: rest, chain_to -> - let first, chain_to = create_keys data chain_to in - first :: f (rest, chain_to) + | data :: rest -> + create_keys data :: f rest in - f (step_data, Promise.return ()) + f step_data in Timer.clock __LOC__ ; diff --git a/src/lib/pickles/pickles.ml b/src/lib/pickles/pickles.ml index e952ed3c317..f5d342510b2 100644 --- a/src/lib/pickles/pickles.ml +++ b/src/lib/pickles/pickles.ml @@ -1296,6 +1296,7 @@ module Make_str (_ : Wire_types.Concrete) = struct , digest ) ) in Cache.Step.read_or_generate + ~run_in_sequence:(fun f -> f ()) ~prev_challenges: (Nat.to_int (fst inner_step_data.proofs_verified)) [] k_p k_v From 114a1d9aee1a675ab186bc879d53db978c2a3b96 Mon Sep 17 00:00:00 2001 From: Gregor Date: Fri, 23 Feb 2024 09:07:48 +0100 Subject: [PATCH 16/45] fix locking --- src/lib/pickles/compile.ml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/lib/pickles/compile.ml b/src/lib/pickles/compile.ml index f1bc3ca0ef7..5417a204659 100644 --- a/src/lib/pickles/compile.ml +++ b/src/lib/pickles/compile.ml @@ -254,12 +254,13 @@ let create_lock () = let lock = ref (Promise.return ()) in let open Promise.Let_syntax in let run_in_sequence (f : unit -> 'a Promise.t) : 'a Promise.t = - (* await the current lock *) - let%bind () = !lock in - (* create a new lock *) + (* acquire the lock *) + let existing_lock = !lock in let unlock = ref (fun () -> ()) in lock := Promise.create (fun resolve -> unlock := resolve) ; - (* run the function and unlock *) + (* await the existing lock *) + let%bind () = existing_lock in + (* run the function and release the lock *) try let%map res = f () in !unlock () ; res From 402b277e4d2b663868399aa3172dc014528802d1 Mon Sep 17 00:00:00 2001 From: Gregor Date: Fri, 23 Feb 2024 10:09:40 +0100 Subject: [PATCH 17/45] simplify: reuse promise for entire domain vector --- src/lib/pickles/compile.ml | 30 +++++++++++++++++------------- src/lib/pickles/compile.mli | 2 +- src/lib/pickles/wrap_domains.ml | 4 ++-- src/lib/pickles/wrap_main.ml | 15 +++------------ src/lib/pickles/wrap_main.mli | 2 +- 5 files changed, 24 insertions(+), 29 deletions(-) diff --git a/src/lib/pickles/compile.ml b/src/lib/pickles/compile.ml index 5417a204659..236b0aceed7 100644 --- a/src/lib/pickles/compile.ml +++ b/src/lib/pickles/compile.ml @@ -129,7 +129,7 @@ type ('max_proofs_verified, 'branches, 'prev_varss) wrap_main_generic = Promise.t Lazy.t -> (int, 'branches) Pickles_types.Vector.t - -> (Import.Domains.t Promise.t, 'branches) Pickles_types.Vector.t + -> (Import.Domains.t, 'branches) Pickles_types.Vector.t Promise.t -> (module Pickles_types.Nat.Add.Intf with type n = 'max_proofs_verified) -> ('max_proofs_verified, 'max_local_max_proofs_verifieds) Requests.Wrap.t * ( ( ( Impls.Wrap.Field.t @@ -252,6 +252,7 @@ end let create_lock () = let lock = ref (Promise.return ()) in + let open Promise.Let_syntax in let run_in_sequence (f : unit -> 'a Promise.t) : 'a Promise.t = (* acquire the lock *) @@ -268,6 +269,18 @@ let create_lock () = in run_in_sequence +(* turn a vector of promises into a promise of a vector *) +let promise_all (type a n) (vec : (a Promise.t, n) Vector.t) : + (a, n) Vector.t Promise.t = + let open Promise.Let_syntax in + let%map () = + (* Wait for promises to resolve. *) + Vector.fold ~init:(Promise.return ()) vec ~f:(fun acc el -> + let%bind _ = el in + acc ) + in + Vector.map ~f:(fun x -> Option.value_exn @@ Promise.peek x) vec + module Make (Arg_var : Statement_var_intf) (Arg_value : Statement_value_intf) @@ -579,16 +592,7 @@ struct V.f prev_varss_length (M.f step_data) in - let all_step_domains = - let%map.Promise () = - (* Wait for promises to resolve. *) - Vector.fold ~init:(Promise.return ()) step_domains - ~f:(fun acc step_domain -> - let%bind.Promise _ = step_domain in - acc ) - in - Vector.map ~f:(fun x -> Option.value_exn @@ Promise.peek x) step_domains - in + let all_step_domains = promise_all step_domains in let run_in_sequence = create_lock () in let cache_handle = ref (Lazy.return (Promise.return `Cache_hit)) in @@ -702,7 +706,7 @@ struct | None -> let srs = Tick.Keypair.load_urs () in Wrap_main.wrap_main ~feature_flags ~srs full_signature - prev_varss_length step_vks proofs_verifieds step_domains + prev_varss_length step_vks proofs_verifieds all_step_domains max_proofs_verified | Some { wrap_main; tweak_statement = _ } -> (* Instead of creating a proof using the pickles wrap circuit, we @@ -714,7 +718,7 @@ struct testing. *) wrap_main wrap_domains full_signature prev_varss_length step_vks - proofs_verifieds step_domains max_proofs_verified + proofs_verifieds all_step_domains max_proofs_verified in Timer.clock __LOC__ ; let (wrap_pk, wrap_vk), disk_key = diff --git a/src/lib/pickles/compile.mli b/src/lib/pickles/compile.mli index 5ad22212770..5389e9cb354 100644 --- a/src/lib/pickles/compile.mli +++ b/src/lib/pickles/compile.mli @@ -168,7 +168,7 @@ type ('max_proofs_verified, 'branches, 'prev_varss) wrap_main_generic = Promise.t Lazy.t -> (int, 'branches) Pickles_types.Vector.t - -> (Import.Domains.t Promise.t, 'branches) Pickles_types.Vector.t + -> (Import.Domains.t, 'branches) Pickles_types.Vector.t Promise.t -> (module Pickles_types.Nat.Add.Intf with type n = 'max_proofs_verified) -> ('max_proofs_verified, 'max_local_max_proofs_verifieds) Requests.Wrap.t * ( ( ( Impls.Wrap.Field.t diff --git a/src/lib/pickles/wrap_domains.ml b/src/lib/pickles/wrap_domains.ml index 9398fdb3bc0..e59d7fd5a91 100644 --- a/src/lib/pickles/wrap_domains.ml +++ b/src/lib/pickles/wrap_domains.ml @@ -18,8 +18,8 @@ struct ~max_proofs_verified = let num_choices = Hlist.Length.to_nat choices_length in let dummy_step_domains = - Vector.init num_choices ~f:(fun _ -> - Promise.return Fix_domains.rough_domains ) + Promise.return + @@ Vector.init num_choices ~f:(fun _ -> Fix_domains.rough_domains) in let dummy_step_widths = Vector.init num_choices ~f:(fun _ -> diff --git a/src/lib/pickles/wrap_main.ml b/src/lib/pickles/wrap_main.ml index f15daa3156b..23624ea56d4 100644 --- a/src/lib/pickles/wrap_main.ml +++ b/src/lib/pickles/wrap_main.ml @@ -97,7 +97,7 @@ let wrap_main Vector.t Promise.t Lazy.t ) (step_widths : (int, branches) Vector.t) - (step_domains : (Domains.t Promise.t, branches) Vector.t) ~srs + (step_domains : (Domains.t, branches) Vector.t Promise.t) ~srs (max_proofs_verified : (module Nat.Add.Intf with type n = max_proofs_verified) ) : (max_proofs_verified, max_local_max_proofs_verifieds) Requests.Wrap.t @@ -134,17 +134,8 @@ let wrap_main Timer.clock __LOC__ ; let main = let%map.Lazy step_keys = step_keys in - let%bind.Promise step_keys = step_keys in - let%map.Promise step_domains = - let%map.Promise () = - (* Wait for promises to resolve. *) - Vector.fold ~init:(Promise.return ()) step_domains - ~f:(fun acc step_domain -> - let%bind.Promise _ = step_domain in - acc ) - in - Vector.map ~f:(fun x -> Option.value_exn @@ Promise.peek x) step_domains - in + let%bind.Promise step_domains = step_domains in + let%map.Promise step_keys = step_keys in fun ({ proof_state = { deferred_values = { plonk diff --git a/src/lib/pickles/wrap_main.mli b/src/lib/pickles/wrap_main.mli index 7d0d9059f15..41c3a85bafe 100644 --- a/src/lib/pickles/wrap_main.mli +++ b/src/lib/pickles/wrap_main.mli @@ -17,7 +17,7 @@ val wrap_main : Promise.t Lazy.t -> (int, 'branches) Pickles_types.Vector.t - -> (Import.Domains.t Promise.t, 'branches) Pickles_types.Vector.t + -> (Import.Domains.t, 'branches) Pickles_types.Vector.t Promise.t -> srs:Kimchi_bindings.Protocol.SRS.Fp.t -> (module Pickles_types.Nat.Add.Intf with type n = 'max_proofs_verified) -> ('max_proofs_verified, 'max_local_max_proofs_verifieds) Requests.Wrap.t From 021d8d55e495719359b6b8ae5b97ca6e506901d6 Mon Sep 17 00:00:00 2001 From: Gregor Date: Fri, 23 Feb 2024 10:58:00 +0100 Subject: [PATCH 18/45] add debugging --- src/lib/pickles/cache.ml | 9 +++++++ src/lib/pickles/compile.ml | 50 ++++++++++++++++++++---------------- src/lib/pickles/wrap_main.ml | 4 +++ 3 files changed, 41 insertions(+), 22 deletions(-) diff --git a/src/lib/pickles/cache.ml b/src/lib/pickles/cache.ml index 34672d1fff9..84be0ec1aa0 100644 --- a/src/lib/pickles/cache.ml +++ b/src/lib/pickles/cache.ml @@ -98,6 +98,7 @@ module Step = struct let pk = lazy (let%bind.Promise k_p = Lazy.force k_p in + let _, _, i, _ = k_p in match Common.time "step keypair read" (fun () -> Key_cache.Sync.read cache s_p k_p ) @@ -111,17 +112,25 @@ module Step = struct let%bind.Promise main = main_promise in let%map.Promise constraint_system = run_in_sequence (fun () -> + print_endline + ( "[read_or_generate] Generating cs - " + ^ Int.to_string i ) ; let constraint_builder = constraint_system_manual ~input_typ:typ ~return_typ in let%map.Promise res = constraint_builder.run_circuit main in + print_endline + ( "[read_or_generate] Ran circuit - " + ^ Int.to_string i ) ; constraint_builder.finish_computation res ) in constraint_system |> Keypair.generate ~prev_challenges ) in Timer.clock __LOC__ ; + print_endline + ("[read_or_generate] Generated keypair - " ^ Int.to_string i) ; ignore ( Key_cache.Sync.write cache s_p k_p (Keypair.pk r) : unit Or_error.t ) ; diff --git a/src/lib/pickles/compile.ml b/src/lib/pickles/compile.ml index 236b0aceed7..4de23b59a08 100644 --- a/src/lib/pickles/compile.ml +++ b/src/lib/pickles/compile.ml @@ -279,6 +279,7 @@ let promise_all (type a n) (vec : (a Promise.t, n) Vector.t) : let%bind _ = el in acc ) in + print_endline "promise_all awaited" ; Vector.map ~f:(fun x -> Option.value_exn @@ Promise.peek x) vec module Make @@ -548,6 +549,7 @@ struct let module V = H4.To_vector (Int) in V.f prev_varss_length (M.f choices) in + (* TODO: remove the `chain_to` promise and just use create_lock *) let step_data = let i = ref 0 in Timer.clock __LOC__ ; @@ -579,16 +581,16 @@ struct in Timer.clock __LOC__ ; let step_domains = - let module DomainsPromise = struct + let module Domains_promise = struct type t = Domains.t Promise.t end in let module M = - H4.Map (Branch_data) (E04 (DomainsPromise)) + H4.Map (Branch_data) (E04 (Domains_promise)) (struct let f (T b : _ Branch_data.t) = b.domains end) in - let module V = H4.To_vector (DomainsPromise) in + let module V = H4.To_vector (Domains_promise) in V.f prev_varss_length (M.f step_data) in @@ -623,25 +625,27 @@ struct let k_p = lazy - (let%map.Promise cs = - let%bind.Promise main = main_promise in - run_in_sequence (fun () -> - let constraint_builder = - Impl.constraint_system_manual ~input_typ:Typ.unit - ~return_typ:typ - in - let%map.Promise res = constraint_builder.run_circuit main in - constraint_builder.finish_computation res ) - in - let cs_hash = Md5.to_hex (R1CS_constraint_system.digest cs) in - ( Type_equal.Id.uid self.id - , snark_keys_header - { type_ = "step-proving-key" - ; identifier = name ^ "-" ^ b.rule.identifier - } - cs_hash - , b.index - , cs ) ) + ( print_endline ("start: create cs hash for " ^ b.rule.identifier) ; + let%map.Promise cs = + let%bind.Promise main = main_promise in + run_in_sequence (fun () -> + let constraint_builder = + Impl.constraint_system_manual ~input_typ:Typ.unit + ~return_typ:typ + in + let%map.Promise res = constraint_builder.run_circuit main in + constraint_builder.finish_computation res ) + in + let cs_hash = Md5.to_hex (R1CS_constraint_system.digest cs) in + print_endline ("done: create cs hash for " ^ b.rule.identifier) ; + ( Type_equal.Id.uid self.id + , snark_keys_header + { type_ = "step-proving-key" + ; identifier = name ^ "-" ^ b.rule.identifier + } + cs_hash + , b.index + , cs ) ) in let k_v = match disk_keys with @@ -672,6 +676,7 @@ struct accum_dirty (Lazy.map vk ~f:(Promise.map ~f:snd)) ; res in + (* TODO refactor back to H4.Map *) let rec f : type a b c d. (a, b, c, d) H4.T(Branch_data).t @@ -696,6 +701,7 @@ struct let%bind.Promise _ = Lazy.force vk in acc ) in + print_endline "step_vks awaited" ; Vector.map step_keypairs ~f:(fun (_, vk) -> Tick.Keypair.full_vk_commitments (fst (Option.value_exn @@ Promise.peek @@ Lazy.force vk)) ) ) diff --git a/src/lib/pickles/wrap_main.ml b/src/lib/pickles/wrap_main.ml index 23624ea56d4..5511a8d8fc6 100644 --- a/src/lib/pickles/wrap_main.ml +++ b/src/lib/pickles/wrap_main.ml @@ -133,9 +133,13 @@ let wrap_main in Timer.clock __LOC__ ; let main = + print_endline "wrap main: start" ; let%map.Lazy step_keys = step_keys in + print_endline "wrap main: kicked off lazy step_keys" ; let%bind.Promise step_domains = step_domains in + print_endline "wrap main: awaited step_domains" ; let%map.Promise step_keys = step_keys in + print_endline "wrap main: awaited step_keys" ; fun ({ proof_state = { deferred_values = { plonk From fb1836e01a93016085cd8758afea722065cf7c0a Mon Sep 17 00:00:00 2001 From: Gregor Date: Fri, 23 Feb 2024 11:05:39 +0100 Subject: [PATCH 19/45] reuse cs generated for key header --- src/lib/pickles/cache.ml | 33 +++++++-------------------------- src/lib/pickles/cache.mli | 8 +------- src/lib/pickles/compile.ml | 6 ++---- src/lib/pickles/pickles.ml | 3 --- 4 files changed, 10 insertions(+), 40 deletions(-) diff --git a/src/lib/pickles/cache.ml b/src/lib/pickles/cache.ml index 84be0ec1aa0..2990e36fc89 100644 --- a/src/lib/pickles/cache.ml +++ b/src/lib/pickles/cache.ml @@ -91,42 +91,23 @@ module Step = struct (Kimchi_bindings.Protocol.VerifierIndex.Fp.write (Some true) x) header path ) ) - let read_or_generate ~(run_in_sequence : (unit -> _ Promise.t) -> _ Promise.t) - ~prev_challenges cache ?(s_p = storable) k_p ?(s_v = vk_storable) k_v typ - return_typ main_promise = + let read_or_generate ~prev_challenges cache ?(s_p = storable) k_p + ?(s_v = vk_storable) k_v = let open Impls.Step in let pk = lazy - (let%bind.Promise k_p = Lazy.force k_p in - let _, _, i, _ = k_p in + (let%map.Promise k_p = Lazy.force k_p in + let _, _, i, sys = k_p in match Common.time "step keypair read" (fun () -> Key_cache.Sync.read cache s_p k_p ) with | Ok (pk, dirty) -> - Common.time "step keypair create" (fun () -> - Promise.return (pk, dirty) ) + Common.time "step keypair create" (fun () -> (pk, dirty)) | Error _e -> - let%map.Promise r = + let r = Common.time "stepkeygen" (fun () -> - let%bind.Promise main = main_promise in - let%map.Promise constraint_system = - run_in_sequence (fun () -> - print_endline - ( "[read_or_generate] Generating cs - " - ^ Int.to_string i ) ; - let constraint_builder = - constraint_system_manual ~input_typ:typ ~return_typ - in - let%map.Promise res = - constraint_builder.run_circuit main - in - print_endline - ( "[read_or_generate] Ran circuit - " - ^ Int.to_string i ) ; - constraint_builder.finish_computation res ) - in - constraint_system |> Keypair.generate ~prev_challenges ) + sys |> Keypair.generate ~prev_challenges ) in Timer.clock __LOC__ ; print_endline diff --git a/src/lib/pickles/cache.mli b/src/lib/pickles/cache.mli index 40a35febb9c..4aeda2655db 100644 --- a/src/lib/pickles/cache.mli +++ b/src/lib/pickles/cache.mli @@ -37,18 +37,12 @@ module Step : sig val vk_storable : vk_storable val read_or_generate : - run_in_sequence: - ( (unit -> Backend.Tick.R1CS_constraint_system.t Promise.t) - -> Backend.Tick.R1CS_constraint_system.t Promise.t ) - -> prev_challenges:int + prev_challenges:int -> Key_cache.Spec.t list -> ?s_p:storable -> Key.Proving.t Promise.t Lazy.t -> ?s_v:vk_storable -> Key.Verification.t Promise.t Lazy.t - -> ('a, 'b) Impls.Step.Typ.t - -> ('c, 'd) Impls.Step.Typ.t - -> ('a -> unit -> 'c Promise.t) Promise.t -> ( Impls.Step.Proving_key.t * ([> `Cache_hit | `Generated_something | `Locally_generated ] as 'e) ) Promise.t diff --git a/src/lib/pickles/compile.ml b/src/lib/pickles/compile.ml index 4de23b59a08..d37d85c0b4b 100644 --- a/src/lib/pickles/compile.ml +++ b/src/lib/pickles/compile.ml @@ -666,11 +666,9 @@ struct in let ((pk, vk) as res) = Common.time "step read or generate" (fun () -> - Cache.Step.read_or_generate ~run_in_sequence + Cache.Step.read_or_generate ~prev_challenges:(Nat.to_int (fst b.proofs_verified)) - cache ~s_p:step_storable k_p ~s_v:step_vk_storable k_v - (Snarky_backendless.Typ.unit ()) - typ main_promise ) + cache ~s_p:step_storable k_p ~s_v:step_vk_storable k_v ) in accum_dirty (Lazy.map pk ~f:(Promise.map ~f:snd)) ; accum_dirty (Lazy.map vk ~f:(Promise.map ~f:snd)) ; diff --git a/src/lib/pickles/pickles.ml b/src/lib/pickles/pickles.ml index f5d342510b2..93941941673 100644 --- a/src/lib/pickles/pickles.ml +++ b/src/lib/pickles/pickles.ml @@ -1296,12 +1296,9 @@ module Make_str (_ : Wire_types.Concrete) = struct , digest ) ) in Cache.Step.read_or_generate - ~run_in_sequence:(fun f -> f ()) ~prev_challenges: (Nat.to_int (fst inner_step_data.proofs_verified)) [] k_p k_v - (Snarky_backendless.Typ.unit ()) - typ main_promise in let step_vks = lazy From 04e0e39958bdd7301099b6c71a8a6125291c670a Mon Sep 17 00:00:00 2001 From: Gregor Date: Fri, 23 Feb 2024 11:26:45 +0100 Subject: [PATCH 20/45] simplify, add debugging --- src/lib/pickles/compile.ml | 61 +++++++++++++++++++------------------- 1 file changed, 30 insertions(+), 31 deletions(-) diff --git a/src/lib/pickles/compile.ml b/src/lib/pickles/compile.ml index d37d85c0b4b..4d279e265fa 100644 --- a/src/lib/pickles/compile.ml +++ b/src/lib/pickles/compile.ml @@ -609,15 +609,6 @@ struct ~wrap_rounds:Tock.Rounds.n in let create_keys (T b : _ Branch_data.t) = - let (T (typ, _conv, conv_inv)) = etyp in - let main_promise : (unit -> unit -> _ Promise.t) Promise.t = - let%map.Promise step_domains = all_step_domains in - let main () () = - let%map.Promise res = b.main ~step_domains () in - Impls.Step.with_label "conv_inv" (fun () -> conv_inv res) - in - main - in (* let () = if true then log_step main typ name b.index in *) let open Impls.Step in (* HACK: TODO docs *) @@ -625,27 +616,30 @@ struct let k_p = lazy - ( print_endline ("start: create cs hash for " ^ b.rule.identifier) ; - let%map.Promise cs = - let%bind.Promise main = main_promise in - run_in_sequence (fun () -> - let constraint_builder = - Impl.constraint_system_manual ~input_typ:Typ.unit - ~return_typ:typ - in - let%map.Promise res = constraint_builder.run_circuit main in - constraint_builder.finish_computation res ) - in - let cs_hash = Md5.to_hex (R1CS_constraint_system.digest cs) in - print_endline ("done: create cs hash for " ^ b.rule.identifier) ; - ( Type_equal.Id.uid self.id - , snark_keys_header - { type_ = "step-proving-key" - ; identifier = name ^ "-" ^ b.rule.identifier - } - cs_hash - , b.index - , cs ) ) + (let (T (typ, _conv, conv_inv)) = etyp in + let%bind.Promise step_domains = all_step_domains in + run_in_sequence (fun () -> + print_endline ("start: create cs hash for " ^ b.rule.identifier) ; + let main () () = + let%map.Promise res = b.main ~step_domains () in + Impls.Step.with_label "conv_inv" (fun () -> conv_inv res) + in + let constraint_builder = + Impl.constraint_system_manual ~input_typ:Typ.unit + ~return_typ:typ + in + let%map.Promise res = constraint_builder.run_circuit main in + let cs = constraint_builder.finish_computation res in + let cs_hash = Md5.to_hex (R1CS_constraint_system.digest cs) in + print_endline ("done: create cs hash for " ^ b.rule.identifier) ; + ( Type_equal.Id.uid self.id + , snark_keys_header + { type_ = "step-proving-key" + ; identifier = name ^ "-" ^ b.rule.identifier + } + cs_hash + , b.index + , cs ) ) ) in let k_v = match disk_keys with @@ -731,19 +725,24 @@ struct let main = lazy (let%map.Promise wrap_main = Lazy.force wrap_main in - fun x () -> wrap_main (conv x) ) + fun x () -> + let input = conv x in + print_endline "created input for wrap_main" ; + wrap_main input ) in (*let () = if true then log_wrap main typ name self.id in*) let self_id = Type_equal.Id.uid self.id in let disk_key_prover = lazy (let%map.Promise main = Lazy.force main in + print_endline "start: create cs hash for wrap" ; let cs = constraint_system ~input_typ:typ ~return_typ:(Snarky_backendless.Typ.unit ()) main in let cs_hash = Md5.to_hex (R1CS_constraint_system.digest cs) in + print_endline "done: create cs hash for wrap" ; ( self_id , snark_keys_header { type_ = "wrap-proving-key"; identifier = name } From 9ff3cdad84927b5e7fff003d14a6d86ea7f3cdf8 Mon Sep 17 00:00:00 2001 From: Gregor Date: Fri, 23 Feb 2024 11:43:52 +0100 Subject: [PATCH 21/45] reuse sys for wrap keygen --- src/lib/pickles/cache.ml | 11 +++++------ src/lib/pickles/cache.mli | 3 --- src/lib/pickles/compile.ml | 21 ++++++++------------- src/lib/pickles/pickles.ml | 4 +--- 4 files changed, 14 insertions(+), 25 deletions(-) diff --git a/src/lib/pickles/cache.ml b/src/lib/pickles/cache.ml index 2990e36fc89..e9dfa93b05f 100644 --- a/src/lib/pickles/cache.ml +++ b/src/lib/pickles/cache.ml @@ -230,24 +230,23 @@ module Wrap = struct header path ) ) let read_or_generate ~prev_challenges cache ?(s_p = storable) k_p - ?(s_v = vk_storable) k_v typ return_typ main = + ?(s_v = vk_storable) k_v = let module Vk = Verification_key in let open Impls.Wrap in let pk = lazy - (let%bind.Promise k = Lazy.force k_p in + (let%map.Promise k = Lazy.force k_p in + let _, _, sys = k in match Common.time "wrap key read" (fun () -> Key_cache.Sync.read cache s_p k ) with | Ok (pk, d) -> - Promise.return (pk, d) + (pk, d) | Error _e -> - let%map.Promise main = Lazy.force main in let r = Common.time "wrapkeygen" (fun () -> - constraint_system ~input_typ:typ ~return_typ main - |> Keypair.generate ~prev_challenges ) + sys |> Keypair.generate ~prev_challenges ) in ignore ( Key_cache.Sync.write cache s_p k (Keypair.pk r) diff --git a/src/lib/pickles/cache.mli b/src/lib/pickles/cache.mli index 4aeda2655db..a3ceee330f1 100644 --- a/src/lib/pickles/cache.mli +++ b/src/lib/pickles/cache.mli @@ -96,9 +96,6 @@ module Wrap : sig -> Key.Proving.t Promise.t Lazy.t -> ?s_v:vk_storable -> Key.Verification.t Promise.t Lazy.t - -> ('a, 'b) Impls.Wrap.Typ.t - -> ('c, 'd) Impls.Wrap.Typ.t - -> ('a -> unit -> 'c) Promise.t Lazy.t -> ( Impls.Wrap.Proving_key.t * [> `Cache_hit | `Generated_something | `Locally_generated ] ) Promise.t diff --git a/src/lib/pickles/compile.ml b/src/lib/pickles/compile.ml index 4d279e265fa..33c25ca0ac7 100644 --- a/src/lib/pickles/compile.ml +++ b/src/lib/pickles/compile.ml @@ -721,20 +721,17 @@ struct Timer.clock __LOC__ ; let (wrap_pk, wrap_vk), disk_key = let open Impls.Wrap in - let (T (typ, conv, _conv_inv)) = input ~feature_flags () in - let main = - lazy - (let%map.Promise wrap_main = Lazy.force wrap_main in - fun x () -> - let input = conv x in - print_endline "created input for wrap_main" ; - wrap_main input ) - in (*let () = if true then log_wrap main typ name self.id in*) let self_id = Type_equal.Id.uid self.id in let disk_key_prover = lazy - (let%map.Promise main = Lazy.force main in + (let%map.Promise wrap_main = Lazy.force wrap_main in + let (T (typ, conv, _conv_inv)) = input ~feature_flags () in + let main x () = + let input = conv x in + print_endline "created input for wrap_main" ; + wrap_main input + in print_endline "start: create cs hash for wrap" ; let cs = constraint_system ~input_typ:typ @@ -767,9 +764,7 @@ struct Common.time "wrap read or generate " (fun () -> Cache.Wrap.read_or_generate (* Due to Wrap_hack *) ~prev_challenges:2 cache ~s_p:wrap_storable disk_key_prover - ~s_v:wrap_vk_storable disk_key_verifier typ - (Snarky_backendless.Typ.unit ()) - main ) + ~s_v:wrap_vk_storable disk_key_verifier ) in (r, disk_key_verifier) in diff --git a/src/lib/pickles/pickles.ml b/src/lib/pickles/pickles.ml index 93941941673..9aeb83496d2 100644 --- a/src/lib/pickles/pickles.ml +++ b/src/lib/pickles/pickles.ml @@ -1371,9 +1371,7 @@ module Make_str (_ : Wire_types.Concrete) = struct Common.time "wrap read or generate " (fun () -> Cache.Wrap.read_or_generate ~prev_challenges:2 [] (Lazy.map ~f:Promise.return disk_key_prover) - (Lazy.map ~f:Promise.return disk_key_verifier) - typ Typ.unit - (lazy (Promise.return main)) ) + (Lazy.map ~f:Promise.return disk_key_verifier) ) in (r, disk_key_verifier) in From 1e366c5dffd751357cb1b344ee76447835c28d2a Mon Sep 17 00:00:00 2001 From: Gregor Date: Fri, 23 Feb 2024 14:41:51 +0100 Subject: [PATCH 22/45] fix blockchain snark - move known_wrap_keys out of circuit --- src/lib/pickles/compile.ml | 5 +- src/lib/pickles/pickles.ml | 14 ++++-- src/lib/pickles/step.ml | 4 +- src/lib/pickles/step_branch_data.ml | 70 ++++++++++++++++++++++++-- src/lib/pickles/step_branch_data.mli | 13 +++++ src/lib/pickles/step_main.ml | 75 ++++++++-------------------- src/lib/pickles/step_main.mli | 6 +++ src/lib/pickles/types_map.ml | 11 ++++ src/lib/pickles/types_map.mli | 12 +++++ 9 files changed, 147 insertions(+), 63 deletions(-) diff --git a/src/lib/pickles/compile.ml b/src/lib/pickles/compile.ml index 33c25ca0ac7..2ef04f10c68 100644 --- a/src/lib/pickles/compile.ml +++ b/src/lib/pickles/compile.ml @@ -618,10 +618,13 @@ struct lazy (let (T (typ, _conv, conv_inv)) = etyp in let%bind.Promise step_domains = all_step_domains in + let%bind.Promise known_wrap_keys = b.known_wrap_keys in run_in_sequence (fun () -> print_endline ("start: create cs hash for " ^ b.rule.identifier) ; let main () () = - let%map.Promise res = b.main ~step_domains () in + let%map.Promise res = + b.main ~step_domains ~known_wrap_keys () + in Impls.Step.with_label "conv_inv" (fun () -> conv_inv res) in let constraint_builder = diff --git a/src/lib/pickles/pickles.ml b/src/lib/pickles/pickles.ml index 9aeb83496d2..07ebab4c53c 100644 --- a/src/lib/pickles/pickles.ml +++ b/src/lib/pickles/pickles.ml @@ -336,7 +336,10 @@ module Make_str (_ : Wire_types.Concrete) = struct | { identifier; prevs; main; feature_flags } :: rest -> { identifier ; prevs - ; main = (fun x -> Promise.return (main x)) + ; main = + (fun x -> + let r = main x in + Promise.return r ) ; feature_flags } :: go rest @@ -1253,9 +1256,14 @@ module Make_str (_ : Wire_types.Concrete) = struct in let (T (typ, _conv, conv_inv)) = etyp in let main_promise : (unit -> unit -> _ Promise.t) Promise.t = - let%map.Promise step_domains = all_step_domains in + let%bind.Promise step_domains = all_step_domains in + let%map.Promise known_wrap_keys = + inner_step_data.known_wrap_keys + in let main () () = - let%map.Promise res = inner_step_data.main ~step_domains () in + let%map.Promise res = + inner_step_data.main ~step_domains ~known_wrap_keys () + in Impls.Step.with_label "conv_inv" (fun () -> conv_inv res) in main diff --git a/src/lib/pickles/step.ml b/src/lib/pickles/step.ml index 160494f82f6..df6d713ab79 100644 --- a/src/lib/pickles/step.ml +++ b/src/lib/pickles/step.ml @@ -805,6 +805,7 @@ struct ~wrap_rounds:Tock.Rounds.n in let%bind.Promise step_domains = step_domains in + let%bind.Promise known_wrap_keys = branch_data.known_wrap_keys in let { Domains.h } = Vector.nth_exn step_domains branch_data.index in ksprintf Common.time "step-prover %d (%d)" branch_data.index (Domain.size h) (fun () -> @@ -815,7 +816,8 @@ struct in let%bind.Promise res = builder.run_circuit (fun () () -> - Promise.map ~f:conv_inv (branch_data.main ~step_domains ()) ) + Promise.map ~f:conv_inv + (branch_data.main ~step_domains ~known_wrap_keys ()) ) in let ( { Impls.Step.Proof_inputs.auxiliary_inputs; public_inputs } , next_statement_hashed ) = diff --git a/src/lib/pickles/step_branch_data.ml b/src/lib/pickles/step_branch_data.ml index 377cb09ab03..cca5c6a5549 100644 --- a/src/lib/pickles/step_branch_data.ml +++ b/src/lib/pickles/step_branch_data.ml @@ -22,6 +22,13 @@ type ( 'a_var 'proofs_verified Nat.t * ('prev_vars, 'proofs_verified) Hlist.Length.t ; index : int ; lte : ('proofs_verified, 'max_proofs_verified) Nat.Lte.t + ; known_wrap_keys : + ( 'prev_vars + , 'prev_values + , 'local_widths + , 'local_branches ) + H4.T(Types_map.Compiled.Optional_wrap_key).t + Promise.t ; domains : Domains.t Promise.t ; rule : ( 'prev_vars @@ -37,6 +44,12 @@ type ( 'a_var Inductive_rule.Promise.t ; main : step_domains:(Domains.t, 'branches) Vector.t + -> known_wrap_keys: + ( 'prev_vars + , 'prev_values + , 'local_widths + , 'local_branches ) + H4.T(Types_map.Compiled.Optional_wrap_key).t -> unit -> ( (Unfinalized.t, 'max_proofs_verified) Vector.t , Impls.Step.Field.t @@ -133,7 +146,7 @@ let create Impls.Step.Typ.(input_typ * output_typ) in Timer.clock __LOC__ ; - let step ~step_domains = + let step ~step_domains ~known_wrap_keys = Step_main.step_main requests (Nat.Add.create max_proofs_verified) rule @@ -146,16 +159,66 @@ let create } ~public_input ~auxiliary_typ ~self_branches:branches ~proofs_verified ~local_signature:widths ~local_signature_length ~local_branches:heights - ~local_branches_length ~lte ~self + ~local_branches_length ~lte ~known_wrap_keys ~self |> unstage in - (* Now that we've triggered the promise computation, we rebind. *) + (* Here, we prefetch the known wrap keys for all compiled rules. + These keys may resolve asynchronously due to key generation for other + pickles rules, but we want to preserve the single-threaded behavior of + pickles to maximize our chanes of successful debugging. + Hence, we preload here, and pass the values in as needed when we create + [datas] below. + *) + let known_wrap_keys = + let rec go : + type a1 a2 n m. + (a1, a2, n, m) H4.T(Tag).t + -> (a1, a2, n, m) H4.T(Types_map.Compiled.Optional_wrap_key).t Promise.t + = function + | [] -> + Promise.return ([] : _ H4.T(Types_map.Compiled.Optional_wrap_key).t) + | tag :: tags -> + let%bind.Promise opt_wrap_key = + match Type_equal.Id.same_witness self.id tag.id with + | Some T -> + Promise.return None + | None -> ( + match tag.kind with + | Compiled -> + let compiled = Types_map.lookup_compiled tag.id in + let%map.Promise wrap_key = Lazy.force @@ compiled.wrap_key + and step_domains = + let%map.Promise () = + (* Wait for promises to resolve. *) + Vector.fold ~init:(Promise.return ()) + compiled.step_domains ~f:(fun acc step_domain -> + let%bind.Promise _ = step_domain in + acc ) + in + Vector.map + ~f:(fun x -> Option.value_exn @@ Promise.peek x) + compiled.step_domains + in + Some + { Types_map.Compiled.Optional_wrap_key.wrap_key + ; step_domains + } + | Side_loaded -> + Promise.return None ) + in + let%map.Promise rest = go tags in + (opt_wrap_key :: rest : _ H4.T(Types_map.Compiled.Optional_wrap_key).t) + in + go rule.prevs + in Timer.clock __LOC__ ; let own_domains = + let%bind.Promise known_wrap_keys = known_wrap_keys in let main = step ~step_domains: (Vector.init branches ~f:(fun _ -> Fix_domains.rough_domains)) + ~known_wrap_keys in let etyp = Impls.Step.input ~proofs_verified:max_proofs_verified @@ -175,6 +238,7 @@ let create ; lte ; rule ; domains = own_domains + ; known_wrap_keys ; main = step ; requests ; feature_flags = actual_feature_flags diff --git a/src/lib/pickles/step_branch_data.mli b/src/lib/pickles/step_branch_data.mli index 34a3ce2ce99..df47bc00410 100644 --- a/src/lib/pickles/step_branch_data.mli +++ b/src/lib/pickles/step_branch_data.mli @@ -23,6 +23,13 @@ type ( 'a_var * ('prev_vars, 'proofs_verified) Pickles_types.Hlist.Length.t ; index : int ; lte : ('proofs_verified, 'max_proofs_verified) Pickles_types.Nat.Lte.t + ; known_wrap_keys : + ( 'prev_vars + , 'prev_values + , 'local_widths + , 'local_branches ) + Pickles_types.Hlist.H4.T(Types_map.Compiled.Optional_wrap_key).t + Promise.t ; domains : Import.Domains.t Promise.t ; rule : ( 'prev_vars @@ -39,6 +46,12 @@ type ( 'a_var (* Main functions to compute *) ; main : step_domains:(Import.Domains.t, 'branches) Pickles_types.Vector.t + -> known_wrap_keys: + ( 'prev_vars + , 'prev_values + , 'local_widths + , 'local_branches ) + Pickles_types.Hlist.H4.T(Types_map.Compiled.Optional_wrap_key).t -> unit -> ( (Unfinalized.t, 'max_proofs_verified) Pickles_types.Vector.t , Impls.Step.Field.t diff --git a/src/lib/pickles/step_main.ml b/src/lib/pickles/step_main.ml index 447022b3115..7e3883d59ec 100644 --- a/src/lib/pickles/step_main.ml +++ b/src/lib/pickles/step_main.ml @@ -160,6 +160,12 @@ let step_main : , max_proofs_verified , self_branches ) Types_map.Compiled.basic + -> known_wrap_keys: + ( prev_vars + , prev_values + , local_signature + , local_branches ) + H4.T(Types_map.Compiled.Optional_wrap_key).t -> self:(var, value, max_proofs_verified, self_branches) Tag.t -> ( prev_vars , prev_values @@ -181,7 +187,8 @@ let step_main : Staged.t = fun (module Req) max_proofs_verified ~self_branches ~local_signature ~local_signature_length ~local_branches ~local_branches_length - ~proofs_verified ~lte ~public_input ~auxiliary_typ ~basic ~self rule -> + ~proofs_verified ~lte ~public_input ~auxiliary_typ ~basic ~known_wrap_keys + ~self rule -> let module Typ_with_max_proofs_verified = struct type ('var, 'value, 'local_max_proofs_verified, 'local_branches) t = ( ( 'var @@ -275,58 +282,7 @@ let step_main : in let T = Max_proofs_verified.eq in let app_state = exists input_typ ~request:(fun () -> Req.App_state) in - let module Optional_wrap_key = struct - type (_, _, _, 'branches) t = - ( Step_main_inputs.Inner_curve.Constant.t array - Plonk_verification_key_evals.t - * (Import.Domains.t, 'branches) Vector.t ) - option - end in - (* Here, we prefetch the known wrap keys for all compiled rules. - These keys may resolve asynchronously due to key generation for other - pickles rules, but we want to preserve the single-threaded behavior of - pickles to maximize our chanes of successful debugging. - Hence, we preload here, and pass the values in as needed when we create - [datas] below. - *) - let%bind.Promise known_wrap_keys = - let rec go : - type a1 a2 n m. - (a1, a2, n, m) H4.T(Tag).t - -> (a1, a2, n, m) H4.T(Optional_wrap_key).t Promise.t = function - | [] -> - Promise.return ([] : _ H4.T(Optional_wrap_key).t) - | tag :: tags -> - let%bind.Promise opt_wrap_key = - match Type_equal.Id.same_witness self.id tag.id with - | Some T -> - Promise.return None - | None -> ( - match tag.kind with - | Compiled -> - let compiled = Types_map.lookup_compiled tag.id in - let%map.Promise wrap_key = Lazy.force @@ compiled.wrap_key - and step_domains = - let%map.Promise () = - (* Wait for promises to resolve. *) - Vector.fold ~init:(Promise.return ()) - compiled.step_domains ~f:(fun acc step_domain -> - let%bind.Promise _ = step_domain in - acc ) - in - Vector.map - ~f:(fun x -> Option.value_exn @@ Promise.peek x) - compiled.step_domains - in - Some (wrap_key, step_domains) - | Side_loaded -> - Promise.return None ) - in - let%map.Promise rest = go tags in - (opt_wrap_key :: rest : _ H4.T(Optional_wrap_key).t) - in - go rule.prevs - in + print_endline ("Step_main '" ^ rule.identifier ^ "' - calling main") ; let%map.Promise { Inductive_rule.previous_proof_statements ; public_output = ret_var ; auxiliary_output = auxiliary_var @@ -334,6 +290,8 @@ let step_main : (* Run the application logic of the rule on the predecessor statements *) with_label "rule_main" (fun () -> rule.main { public_input = app_state }) in + + print_endline ("Step_main '" ^ rule.identifier ^ "' - rest of the circuit") ; with_label "step_main" (fun () -> let () = exists Typ.unit ~request:(fun () -> @@ -541,7 +499,11 @@ let step_main : let rec go : type a1 a2 n m. (a1, a2, n, m) H4.T(Tag).t - -> (a1, a2, n, m) H4.T(Optional_wrap_key).t + -> ( a1 + , a2 + , n + , m ) + H4.T(Types_map.Compiled.Optional_wrap_key).t -> (a1, a2, n, m) H4.T(Types_map.For_step).t = fun tags optional_wrap_keys -> match (tags, optional_wrap_keys) with @@ -553,7 +515,10 @@ let step_main : | None -> ( match tag.kind with | Compiled -> - let wrap_key, step_domains = + let { Types_map.Compiled.Optional_wrap_key + .wrap_key + ; step_domains + } = Option.value_exn optional_wrap_key in Types_map.For_step diff --git a/src/lib/pickles/step_main.mli b/src/lib/pickles/step_main.mli index c6f348d9f01..aaade12333a 100644 --- a/src/lib/pickles/step_main.mli +++ b/src/lib/pickles/step_main.mli @@ -39,6 +39,12 @@ val step_main : , 'max_proofs_verified , 'self_branches ) Types_map.Compiled.basic + -> known_wrap_keys: + ( 'prev_vars + , 'prev_values + , 'local_signature + , 'local_branches ) + Pickles_types.Hlist.H4.T(Types_map.Compiled.Optional_wrap_key).t -> self:('var, 'value, 'max_proofs_verified, 'self_branches) Tag.t -> ( 'prev_vars , 'prev_values diff --git a/src/lib/pickles/types_map.ml b/src/lib/pickles/types_map.ml index abfe7b8e96d..a856a244cdd 100644 --- a/src/lib/pickles/types_map.ml +++ b/src/lib/pickles/types_map.ml @@ -107,6 +107,17 @@ module Compiled = struct ; feature_flags : Opt.Flag.t Plonk_types.Features.Full.t } + module Optional_wrap_key = struct + type 'branches resolved_wrap_key = + { wrap_key : + Tick.Inner_curve.Affine.t array Plonk_verification_key_evals.t + ; step_domains : (Domains.t, 'branches) Vector.t + } + + type ('a_var, 'a_value, 'max_proofs_verified, 'branches) t = + 'branches resolved_wrap_key option + end + type packed = | T : ('var, 'value, 'n1, 'n2) Tag.id * ('var, 'value, 'n1, 'n2) t -> packed diff --git a/src/lib/pickles/types_map.mli b/src/lib/pickles/types_map.mli index fd7aa2a4f7a..f0b52c19ea7 100644 --- a/src/lib/pickles/types_map.mli +++ b/src/lib/pickles/types_map.mli @@ -79,6 +79,18 @@ module Compiled : sig (Import.Domains.t Promise.t, 'branches) Pickles_types.Vector.t ; feature_flags : Opt.Flag.t Plonk_types.Features.Full.t } + + module Optional_wrap_key : sig + type 'branches resolved_wrap_key = + { wrap_key : + Backend.Tick.Inner_curve.Affine.t array + Pickles_types.Plonk_verification_key_evals.t + ; step_domains : (Import.Domains.t, 'branches) Pickles_types.Vector.t + } + + type ('a_var, 'a_value, 'max_proofs_verified, 'branches) t = + 'branches resolved_wrap_key option + end end module For_step : sig From 8dae1de7c0aac551771eeee7ebcfb67e617de96c Mon Sep 17 00:00:00 2001 From: Gregor Date: Fri, 23 Feb 2024 15:06:23 +0100 Subject: [PATCH 23/45] simplify: encode wrap keys in main promise --- src/lib/pickles/compile.ml | 7 +-- src/lib/pickles/pickles.ml | 35 ++++++--------- src/lib/pickles/step.ml | 5 +-- src/lib/pickles/step_branch_data.ml | 66 ++++++++++++---------------- src/lib/pickles/step_branch_data.mli | 30 +++++-------- 5 files changed, 56 insertions(+), 87 deletions(-) diff --git a/src/lib/pickles/compile.ml b/src/lib/pickles/compile.ml index 2ef04f10c68..5c168d7bd65 100644 --- a/src/lib/pickles/compile.ml +++ b/src/lib/pickles/compile.ml @@ -617,14 +617,11 @@ struct let k_p = lazy (let (T (typ, _conv, conv_inv)) = etyp in - let%bind.Promise step_domains = all_step_domains in - let%bind.Promise known_wrap_keys = b.known_wrap_keys in + let%bind.Promise main = b.main ~step_domains:all_step_domains in run_in_sequence (fun () -> print_endline ("start: create cs hash for " ^ b.rule.identifier) ; let main () () = - let%map.Promise res = - b.main ~step_domains ~known_wrap_keys () - in + let%map.Promise res = main () in Impls.Step.with_label "conv_inv" (fun () -> conv_inv res) in let constraint_builder = diff --git a/src/lib/pickles/pickles.ml b/src/lib/pickles/pickles.ml index 07ebab4c53c..b77cbea8311 100644 --- a/src/lib/pickles/pickles.ml +++ b/src/lib/pickles/pickles.ml @@ -1254,32 +1254,23 @@ module Make_str (_ : Wire_types.Concrete) = struct Impls.Step.input ~proofs_verified:Max_proofs_verified.n ~wrap_rounds:Tock.Rounds.n in - let (T (typ, _conv, conv_inv)) = etyp in - let main_promise : (unit -> unit -> _ Promise.t) Promise.t = - let%bind.Promise step_domains = all_step_domains in - let%map.Promise known_wrap_keys = - inner_step_data.known_wrap_keys - in - let main () () = - let%map.Promise res = - inner_step_data.main ~step_domains ~known_wrap_keys () - in - Impls.Step.with_label "conv_inv" (fun () -> conv_inv res) - in - main - in let open Impls.Step in let k_p = lazy - (let%map.Promise cs = - let%bind.Promise main = main_promise in - let constraint_builder = - Impl.constraint_system_manual ~input_typ:Typ.unit - ~return_typ:typ - in - let%map.Promise res = constraint_builder.run_circuit main in - constraint_builder.finish_computation res + (let (T (typ, _conv, conv_inv)) = etyp in + let%bind.Promise main = + inner_step_data.main ~step_domains:all_step_domains + in + let main () () = + let%map.Promise res = main () in + Impls.Step.with_label "conv_inv" (fun () -> conv_inv res) + in + let constraint_builder = + Impl.constraint_system_manual ~input_typ:Typ.unit + ~return_typ:typ in + let%map.Promise res = constraint_builder.run_circuit main in + let cs = constraint_builder.finish_computation res in let cs_hash = Md5.to_hex (R1CS_constraint_system.digest cs) in ( Type_equal.Id.uid self.id , snark_keys_header diff --git a/src/lib/pickles/step.ml b/src/lib/pickles/step.ml index df6d713ab79..3471b99667e 100644 --- a/src/lib/pickles/step.ml +++ b/src/lib/pickles/step.ml @@ -804,8 +804,8 @@ struct Impls.Step.input ~proofs_verified:Max_proofs_verified.n ~wrap_rounds:Tock.Rounds.n in + let%bind.Promise main = branch_data.main ~step_domains in let%bind.Promise step_domains = step_domains in - let%bind.Promise known_wrap_keys = branch_data.known_wrap_keys in let { Domains.h } = Vector.nth_exn step_domains branch_data.index in ksprintf Common.time "step-prover %d (%d)" branch_data.index (Domain.size h) (fun () -> @@ -816,8 +816,7 @@ struct in let%bind.Promise res = builder.run_circuit (fun () () -> - Promise.map ~f:conv_inv - (branch_data.main ~step_domains ~known_wrap_keys ()) ) + Promise.map ~f:conv_inv (main ()) ) in let ( { Impls.Step.Proof_inputs.auxiliary_inputs; public_inputs } , next_statement_hashed ) = diff --git a/src/lib/pickles/step_branch_data.ml b/src/lib/pickles/step_branch_data.ml index cca5c6a5549..49a322746fb 100644 --- a/src/lib/pickles/step_branch_data.ml +++ b/src/lib/pickles/step_branch_data.ml @@ -22,13 +22,6 @@ type ( 'a_var 'proofs_verified Nat.t * ('prev_vars, 'proofs_verified) Hlist.Length.t ; index : int ; lte : ('proofs_verified, 'max_proofs_verified) Nat.Lte.t - ; known_wrap_keys : - ( 'prev_vars - , 'prev_values - , 'local_widths - , 'local_branches ) - H4.T(Types_map.Compiled.Optional_wrap_key).t - Promise.t ; domains : Domains.t Promise.t ; rule : ( 'prev_vars @@ -43,18 +36,13 @@ type ( 'a_var , 'auxiliary_value ) Inductive_rule.Promise.t ; main : - step_domains:(Domains.t, 'branches) Vector.t - -> known_wrap_keys: - ( 'prev_vars - , 'prev_values - , 'local_widths - , 'local_branches ) - H4.T(Types_map.Compiled.Optional_wrap_key).t - -> unit - -> ( (Unfinalized.t, 'max_proofs_verified) Vector.t - , Impls.Step.Field.t - , (Impls.Step.Field.t, 'max_proofs_verified) Vector.t ) - Types.Step.Statement.t + step_domains:(Domains.t, 'branches) Vector.t Promise.t + -> ( unit + -> ( (Unfinalized.t, 'max_proofs_verified) Vector.t + , Impls.Step.Field.t + , (Impls.Step.Field.t, 'max_proofs_verified) Vector.t ) + Types.Step.Statement.t + Promise.t ) Promise.t ; requests : (module Requests.Step.S @@ -145,23 +133,6 @@ let create | Input_and_output (input_typ, output_typ) -> Impls.Step.Typ.(input_typ * output_typ) in - Timer.clock __LOC__ ; - let step ~step_domains ~known_wrap_keys = - Step_main.step_main requests - (Nat.Add.create max_proofs_verified) - rule - ~basic: - { public_input = typ - ; proofs_verifieds - ; wrap_domains - ; step_domains - ; feature_flags - } - ~public_input ~auxiliary_typ ~self_branches:branches ~proofs_verified - ~local_signature:widths ~local_signature_length ~local_branches:heights - ~local_branches_length ~lte ~known_wrap_keys ~self - |> unstage - in (* Here, we prefetch the known wrap keys for all compiled rules. These keys may resolve asynchronously due to key generation for other pickles rules, but we want to preserve the single-threaded behavior of @@ -212,6 +183,23 @@ let create go rule.prevs in Timer.clock __LOC__ ; + let step ~step_domains ~known_wrap_keys = + Step_main.step_main requests + (Nat.Add.create max_proofs_verified) + rule + ~basic: + { public_input = typ + ; proofs_verifieds + ; wrap_domains + ; step_domains + ; feature_flags + } + ~public_input ~auxiliary_typ ~self_branches:branches ~proofs_verified + ~local_signature:widths ~local_signature_length ~local_branches:heights + ~local_branches_length ~lte ~known_wrap_keys ~self + |> unstage + in + Timer.clock __LOC__ ; let own_domains = let%bind.Promise known_wrap_keys = known_wrap_keys in let main = @@ -231,6 +219,11 @@ let create (T (Snarky_backendless.Typ.unit (), Fn.id, Fn.id)) etyp main in + let step ~step_domains = + let%bind.Promise known_wrap_keys = known_wrap_keys in + let%map.Promise step_domains = step_domains in + step ~step_domains ~known_wrap_keys + in Timer.clock __LOC__ ; T { proofs_verified = (self_width, proofs_verified) @@ -238,7 +231,6 @@ let create ; lte ; rule ; domains = own_domains - ; known_wrap_keys ; main = step ; requests ; feature_flags = actual_feature_flags diff --git a/src/lib/pickles/step_branch_data.mli b/src/lib/pickles/step_branch_data.mli index df47bc00410..26ade8cf722 100644 --- a/src/lib/pickles/step_branch_data.mli +++ b/src/lib/pickles/step_branch_data.mli @@ -23,13 +23,6 @@ type ( 'a_var * ('prev_vars, 'proofs_verified) Pickles_types.Hlist.Length.t ; index : int ; lte : ('proofs_verified, 'max_proofs_verified) Pickles_types.Nat.Lte.t - ; known_wrap_keys : - ( 'prev_vars - , 'prev_values - , 'local_widths - , 'local_branches ) - Pickles_types.Hlist.H4.T(Types_map.Compiled.Optional_wrap_key).t - Promise.t ; domains : Import.Domains.t Promise.t ; rule : ( 'prev_vars @@ -45,19 +38,16 @@ type ( 'a_var Inductive_rule.Promise.t (* Main functions to compute *) ; main : - step_domains:(Import.Domains.t, 'branches) Pickles_types.Vector.t - -> known_wrap_keys: - ( 'prev_vars - , 'prev_values - , 'local_widths - , 'local_branches ) - Pickles_types.Hlist.H4.T(Types_map.Compiled.Optional_wrap_key).t - -> unit - -> ( (Unfinalized.t, 'max_proofs_verified) Pickles_types.Vector.t - , Impls.Step.Field.t - , (Impls.Step.Field.t, 'max_proofs_verified) Pickles_types.Vector.t - ) - Import.Types.Step.Statement.t + step_domains: + (Import.Domains.t, 'branches) Pickles_types.Vector.t Promise.t + -> ( unit + -> ( (Unfinalized.t, 'max_proofs_verified) Pickles_types.Vector.t + , Impls.Step.Field.t + , ( Impls.Step.Field.t + , 'max_proofs_verified ) + Pickles_types.Vector.t ) + Import.Types.Step.Statement.t + Promise.t ) Promise.t ; requests : (module Requests.Step.S From 4bb79769ea1e77cdfe33cdc71cb6273c884389c3 Mon Sep 17 00:00:00 2001 From: Gregor Date: Fri, 23 Feb 2024 15:44:11 +0100 Subject: [PATCH 24/45] clean up known wrap key --- src/lib/pickles/step_branch_data.ml | 17 +++++++---------- src/lib/pickles/step_main.ml | 22 ++++------------------ src/lib/pickles/step_main.mli | 7 ++----- src/lib/pickles/types_map.ml | 26 +++++++++++++------------- src/lib/pickles/types_map.mli | 28 ++++++++++++---------------- 5 files changed, 38 insertions(+), 62 deletions(-) diff --git a/src/lib/pickles/step_branch_data.ml b/src/lib/pickles/step_branch_data.ml index 49a322746fb..9be65c8573a 100644 --- a/src/lib/pickles/step_branch_data.ml +++ b/src/lib/pickles/step_branch_data.ml @@ -136,18 +136,18 @@ let create (* Here, we prefetch the known wrap keys for all compiled rules. These keys may resolve asynchronously due to key generation for other pickles rules, but we want to preserve the single-threaded behavior of - pickles to maximize our chanes of successful debugging. + pickles to maximize our chances of successful debugging. Hence, we preload here, and pass the values in as needed when we create [datas] below. *) + let module Optional_wrap_key = Types_map.For_step.Optional_wrap_key in let known_wrap_keys = let rec go : type a1 a2 n m. - (a1, a2, n, m) H4.T(Tag).t - -> (a1, a2, n, m) H4.T(Types_map.Compiled.Optional_wrap_key).t Promise.t - = function + (a1, a2, n, m) H4.T(Tag).t -> m H1.T(Optional_wrap_key).t Promise.t = + function | [] -> - Promise.return ([] : _ H4.T(Types_map.Compiled.Optional_wrap_key).t) + Promise.return ([] : _ H1.T(Optional_wrap_key).t) | tag :: tags -> let%bind.Promise opt_wrap_key = match Type_equal.Id.same_witness self.id tag.id with @@ -170,15 +170,12 @@ let create ~f:(fun x -> Option.value_exn @@ Promise.peek x) compiled.step_domains in - Some - { Types_map.Compiled.Optional_wrap_key.wrap_key - ; step_domains - } + Some { Optional_wrap_key.wrap_key; step_domains } | Side_loaded -> Promise.return None ) in let%map.Promise rest = go tags in - (opt_wrap_key :: rest : _ H4.T(Types_map.Compiled.Optional_wrap_key).t) + (opt_wrap_key :: rest : _ H1.T(Optional_wrap_key).t) in go rule.prevs in diff --git a/src/lib/pickles/step_main.ml b/src/lib/pickles/step_main.ml index 7e3883d59ec..a16bb43f6c9 100644 --- a/src/lib/pickles/step_main.ml +++ b/src/lib/pickles/step_main.ml @@ -161,11 +161,7 @@ let step_main : , self_branches ) Types_map.Compiled.basic -> known_wrap_keys: - ( prev_vars - , prev_values - , local_signature - , local_branches ) - H4.T(Types_map.Compiled.Optional_wrap_key).t + local_branches H1.T(Types_map.For_step.Optional_wrap_key).t -> self:(var, value, max_proofs_verified, self_branches) Tag.t -> ( prev_vars , prev_values @@ -499,11 +495,7 @@ let step_main : let rec go : type a1 a2 n m. (a1, a2, n, m) H4.T(Tag).t - -> ( a1 - , a2 - , n - , m ) - H4.T(Types_map.Compiled.Optional_wrap_key).t + -> m H1.T(Types_map.For_step.Optional_wrap_key).t -> (a1, a2, n, m) H4.T(Types_map.For_step).t = fun tags optional_wrap_keys -> match (tags, optional_wrap_keys) with @@ -515,15 +507,9 @@ let step_main : | None -> ( match tag.kind with | Compiled -> - let { Types_map.Compiled.Optional_wrap_key - .wrap_key - ; step_domains - } = - Option.value_exn optional_wrap_key - in Types_map.For_step - .of_compiled_with_known_wrap_key ~wrap_key - ~step_domains + .of_compiled_with_known_wrap_key + (Option.value_exn optional_wrap_key) (Types_map.lookup_compiled tag.id) | Side_loaded -> Types_map.For_step.of_side_loaded diff --git a/src/lib/pickles/step_main.mli b/src/lib/pickles/step_main.mli index aaade12333a..b35dd99cc2f 100644 --- a/src/lib/pickles/step_main.mli +++ b/src/lib/pickles/step_main.mli @@ -40,11 +40,8 @@ val step_main : , 'self_branches ) Types_map.Compiled.basic -> known_wrap_keys: - ( 'prev_vars - , 'prev_values - , 'local_signature - , 'local_branches ) - Pickles_types.Hlist.H4.T(Types_map.Compiled.Optional_wrap_key).t + 'local_branches + Pickles_types.Hlist.H1.T(Types_map.For_step.Optional_wrap_key).t -> self:('var, 'value, 'max_proofs_verified, 'self_branches) Tag.t -> ( 'prev_vars , 'prev_values diff --git a/src/lib/pickles/types_map.ml b/src/lib/pickles/types_map.ml index a856a244cdd..e66c2bc1878 100644 --- a/src/lib/pickles/types_map.ml +++ b/src/lib/pickles/types_map.ml @@ -107,17 +107,6 @@ module Compiled = struct ; feature_flags : Opt.Flag.t Plonk_types.Features.Full.t } - module Optional_wrap_key = struct - type 'branches resolved_wrap_key = - { wrap_key : - Tick.Inner_curve.Affine.t array Plonk_verification_key_evals.t - ; step_domains : (Domains.t, 'branches) Vector.t - } - - type ('a_var, 'a_value, 'max_proofs_verified, 'branches) t = - 'branches resolved_wrap_key option - end - type packed = | T : ('var, 'value, 'n1, 'n2) Tag.id * ('var, 'value, 'n1, 'n2) t -> packed @@ -188,7 +177,18 @@ module For_step = struct ; feature_flags } - let of_compiled_with_known_wrap_key ~wrap_key ~step_domains + module Optional_wrap_key = struct + type 'branches known = + { wrap_key : + Tick.Inner_curve.Affine.t array Plonk_verification_key_evals.t + ; step_domains : (Domains.t, 'branches) Vector.t + } + + type 'branches t = 'branches known option + end + + let of_compiled_with_known_wrap_key + ({ wrap_key; step_domains } : _ Optional_wrap_key.known) ({ branches ; max_proofs_verified ; proofs_verifieds @@ -225,7 +225,7 @@ module For_step = struct in Vector.map ~f:(fun x -> Option.value_exn @@ Promise.peek x) step_domains in - of_compiled_with_known_wrap_key ~wrap_key ~step_domains t + of_compiled_with_known_wrap_key { wrap_key; step_domains } t end type t = diff --git a/src/lib/pickles/types_map.mli b/src/lib/pickles/types_map.mli index f0b52c19ea7..33a8a96a182 100644 --- a/src/lib/pickles/types_map.mli +++ b/src/lib/pickles/types_map.mli @@ -79,18 +79,6 @@ module Compiled : sig (Import.Domains.t Promise.t, 'branches) Pickles_types.Vector.t ; feature_flags : Opt.Flag.t Plonk_types.Features.Full.t } - - module Optional_wrap_key : sig - type 'branches resolved_wrap_key = - { wrap_key : - Backend.Tick.Inner_curve.Affine.t array - Pickles_types.Plonk_verification_key_evals.t - ; step_domains : (Import.Domains.t, 'branches) Pickles_types.Vector.t - } - - type ('a_var, 'a_value, 'max_proofs_verified, 'branches) t = - 'branches resolved_wrap_key option - end end module For_step : sig @@ -116,11 +104,19 @@ module For_step : sig val of_side_loaded : ('a, 'b, 'c, 'd) Side_loaded.t -> ('a, 'b, 'c, 'd) t + module Optional_wrap_key : sig + type 'branches known = + { wrap_key : + Backend.Tick.Inner_curve.Affine.t array + Pickles_types.Plonk_verification_key_evals.t + ; step_domains : (Import.Domains.t, 'branches) Pickles_types.Vector.t + } + + type 'branches t = 'branches known option + end + val of_compiled_with_known_wrap_key : - wrap_key: - Step_main_inputs.Inner_curve.Constant.t array - Plonk_verification_key_evals.t - -> step_domains:(Import.Domains.t, 'branches) Vector.t + 'branches Optional_wrap_key.known -> ('a, 'b, 'c, 'branches) Compiled.t -> ('a, 'b, 'c, 'branches) t From 41fde710d9b21f207b3005c882836b1b06cbdd53 Mon Sep 17 00:00:00 2001 From: Gregor Date: Fri, 23 Feb 2024 16:04:37 +0100 Subject: [PATCH 25/45] undo unnecessary refactors --- src/lib/pickles/compile.ml | 154 +++++++++++++++++++------------------ src/lib/pickles/pickles.ml | 5 +- 2 files changed, 80 insertions(+), 79 deletions(-) diff --git a/src/lib/pickles/compile.ml b/src/lib/pickles/compile.ml index 5c168d7bd65..bfb7f25bb17 100644 --- a/src/lib/pickles/compile.ml +++ b/src/lib/pickles/compile.ml @@ -604,83 +604,87 @@ struct let disk_keys = Option.map disk_keys ~f:(fun (xs, _) -> Vector.to_array xs) in - let etyp = - Impls.Step.input ~proofs_verified:Max_proofs_verified.n - ~wrap_rounds:Tock.Rounds.n - in - let create_keys (T b : _ Branch_data.t) = - (* let () = if true then log_step main typ name b.index in *) - let open Impls.Step in - (* HACK: TODO docs *) - if return_early_digest_exception then failwith "TODO: Delete me" ; - - let k_p = - lazy - (let (T (typ, _conv, conv_inv)) = etyp in - let%bind.Promise main = b.main ~step_domains:all_step_domains in - run_in_sequence (fun () -> - print_endline ("start: create cs hash for " ^ b.rule.identifier) ; - let main () () = - let%map.Promise res = main () in - Impls.Step.with_label "conv_inv" (fun () -> conv_inv res) - in - let constraint_builder = - Impl.constraint_system_manual ~input_typ:Typ.unit - ~return_typ:typ - in - let%map.Promise res = constraint_builder.run_circuit main in - let cs = constraint_builder.finish_computation res in - let cs_hash = Md5.to_hex (R1CS_constraint_system.digest cs) in - print_endline ("done: create cs hash for " ^ b.rule.identifier) ; - ( Type_equal.Id.uid self.id - , snark_keys_header - { type_ = "step-proving-key" - ; identifier = name ^ "-" ^ b.rule.identifier - } - cs_hash - , b.index - , cs ) ) ) - in - let k_v = - match disk_keys with - | Some ks -> - Lazy.return (Promise.return ks.(b.index)) - | None -> - lazy - (let%map.Promise id, _header, index, cs = Lazy.force k_p in - let digest = R1CS_constraint_system.digest cs in - ( id - , snark_keys_header - { type_ = "step-verification-key" - ; identifier = name ^ "-" ^ b.rule.identifier - } - (Md5.to_hex digest) - , index - , digest ) ) - in - let ((pk, vk) as res) = - Common.time "step read or generate" (fun () -> - Cache.Step.read_or_generate - ~prev_challenges:(Nat.to_int (fst b.proofs_verified)) - cache ~s_p:step_storable k_p ~s_v:step_vk_storable k_v ) - in - accum_dirty (Lazy.map pk ~f:(Promise.map ~f:snd)) ; - accum_dirty (Lazy.map vk ~f:(Promise.map ~f:snd)) ; - res - in - (* TODO refactor back to H4.Map *) - let rec f : - type a b c d. - (a, b, c, d) H4.T(Branch_data).t - -> (a, b, c, d) H4.T(E04(Lazy_keys)).t = function - | [] -> - [] - | data :: rest -> - create_keys data :: f rest + let module M = + H4.Map (Branch_data) (E04 (Lazy_keys)) + (struct + let etyp = + Impls.Step.input ~proofs_verified:Max_proofs_verified.n + ~wrap_rounds:Tock.Rounds.n + + let f (T b : _ Branch_data.t) = + (* let () = if true then log_step main typ name b.index in *) + let open Impls.Step in + (* HACK: TODO docs *) + if return_early_digest_exception then failwith "TODO: Delete me" ; + + let k_p = + lazy + (let (T (typ, _conv, conv_inv)) = etyp in + let%bind.Promise main = + b.main ~step_domains:all_step_domains + in + run_in_sequence (fun () -> + print_endline + ("start: create cs hash for " ^ b.rule.identifier) ; + let main () () = + let%map.Promise res = main () in + Impls.Step.with_label "conv_inv" (fun () -> + conv_inv res ) + in + let constraint_builder = + Impl.constraint_system_manual ~input_typ:Typ.unit + ~return_typ:typ + in + let%map.Promise res = + constraint_builder.run_circuit main + in + let cs = constraint_builder.finish_computation res in + let cs_hash = + Md5.to_hex (R1CS_constraint_system.digest cs) + in + print_endline + ("done: create cs hash for " ^ b.rule.identifier) ; + ( Type_equal.Id.uid self.id + , snark_keys_header + { type_ = "step-proving-key" + ; identifier = name ^ "-" ^ b.rule.identifier + } + cs_hash + , b.index + , cs ) ) ) + in + let k_v = + match disk_keys with + | Some ks -> + Lazy.return (Promise.return ks.(b.index)) + | None -> + lazy + (let%map.Promise id, _header, index, cs = + Lazy.force k_p + in + let digest = R1CS_constraint_system.digest cs in + ( id + , snark_keys_header + { type_ = "step-verification-key" + ; identifier = name ^ "-" ^ b.rule.identifier + } + (Md5.to_hex digest) + , index + , digest ) ) + in + let ((pk, vk) as res) = + Common.time "step read or generate" (fun () -> + Cache.Step.read_or_generate + ~prev_challenges:(Nat.to_int (fst b.proofs_verified)) + cache ~s_p:step_storable k_p ~s_v:step_vk_storable k_v ) + in + accum_dirty (Lazy.map pk ~f:(Promise.map ~f:snd)) ; + accum_dirty (Lazy.map vk ~f:(Promise.map ~f:snd)) ; + res + end) in - f step_data + M.f step_data in - Timer.clock __LOC__ ; let step_vks = let module V = H4.To_vector (Lazy_keys) in diff --git a/src/lib/pickles/pickles.ml b/src/lib/pickles/pickles.ml index b77cbea8311..7a401eeac32 100644 --- a/src/lib/pickles/pickles.ml +++ b/src/lib/pickles/pickles.ml @@ -336,10 +336,7 @@ module Make_str (_ : Wire_types.Concrete) = struct | { identifier; prevs; main; feature_flags } :: rest -> { identifier ; prevs - ; main = - (fun x -> - let r = main x in - Promise.return r ) + ; main = (fun x -> Promise.return (main x)) ; feature_flags } :: go rest From e8880e3436be860f699bed876e260f5826aeedd7 Mon Sep 17 00:00:00 2001 From: Gregor Date: Fri, 23 Feb 2024 16:11:47 +0100 Subject: [PATCH 26/45] remove debug logs --- src/lib/pickles/cache.ml | 4 +--- src/lib/pickles/compile.ml | 15 +-------------- src/lib/pickles/step_main.ml | 3 --- src/lib/pickles/wrap_main.ml | 4 ---- 4 files changed, 2 insertions(+), 24 deletions(-) diff --git a/src/lib/pickles/cache.ml b/src/lib/pickles/cache.ml index e9dfa93b05f..cfce7518f31 100644 --- a/src/lib/pickles/cache.ml +++ b/src/lib/pickles/cache.ml @@ -97,7 +97,7 @@ module Step = struct let pk = lazy (let%map.Promise k_p = Lazy.force k_p in - let _, _, i, sys = k_p in + let _, _, _, sys = k_p in match Common.time "step keypair read" (fun () -> Key_cache.Sync.read cache s_p k_p ) @@ -110,8 +110,6 @@ module Step = struct sys |> Keypair.generate ~prev_challenges ) in Timer.clock __LOC__ ; - print_endline - ("[read_or_generate] Generated keypair - " ^ Int.to_string i) ; ignore ( Key_cache.Sync.write cache s_p k_p (Keypair.pk r) : unit Or_error.t ) ; diff --git a/src/lib/pickles/compile.ml b/src/lib/pickles/compile.ml index bfb7f25bb17..e4275cd09c2 100644 --- a/src/lib/pickles/compile.ml +++ b/src/lib/pickles/compile.ml @@ -279,7 +279,6 @@ let promise_all (type a n) (vec : (a Promise.t, n) Vector.t) : let%bind _ = el in acc ) in - print_endline "promise_all awaited" ; Vector.map ~f:(fun x -> Option.value_exn @@ Promise.peek x) vec module Make @@ -549,7 +548,6 @@ struct let module V = H4.To_vector (Int) in V.f prev_varss_length (M.f choices) in - (* TODO: remove the `chain_to` promise and just use create_lock *) let step_data = let i = ref 0 in Timer.clock __LOC__ ; @@ -624,8 +622,6 @@ struct b.main ~step_domains:all_step_domains in run_in_sequence (fun () -> - print_endline - ("start: create cs hash for " ^ b.rule.identifier) ; let main () () = let%map.Promise res = main () in Impls.Step.with_label "conv_inv" (fun () -> @@ -642,8 +638,6 @@ struct let cs_hash = Md5.to_hex (R1CS_constraint_system.digest cs) in - print_endline - ("done: create cs hash for " ^ b.rule.identifier) ; ( Type_equal.Id.uid self.id , snark_keys_header { type_ = "step-proving-key" @@ -697,7 +691,6 @@ struct let%bind.Promise _ = Lazy.force vk in acc ) in - print_endline "step_vks awaited" ; Vector.map step_keypairs ~f:(fun (_, vk) -> Tick.Keypair.full_vk_commitments (fst (Option.value_exn @@ Promise.peek @@ Lazy.force vk)) ) ) @@ -731,19 +724,13 @@ struct lazy (let%map.Promise wrap_main = Lazy.force wrap_main in let (T (typ, conv, _conv_inv)) = input ~feature_flags () in - let main x () = - let input = conv x in - print_endline "created input for wrap_main" ; - wrap_main input - in - print_endline "start: create cs hash for wrap" ; + let main x () = wrap_main (conv x) in let cs = constraint_system ~input_typ:typ ~return_typ:(Snarky_backendless.Typ.unit ()) main in let cs_hash = Md5.to_hex (R1CS_constraint_system.digest cs) in - print_endline "done: create cs hash for wrap" ; ( self_id , snark_keys_header { type_ = "wrap-proving-key"; identifier = name } diff --git a/src/lib/pickles/step_main.ml b/src/lib/pickles/step_main.ml index a16bb43f6c9..f85f84196a3 100644 --- a/src/lib/pickles/step_main.ml +++ b/src/lib/pickles/step_main.ml @@ -278,7 +278,6 @@ let step_main : in let T = Max_proofs_verified.eq in let app_state = exists input_typ ~request:(fun () -> Req.App_state) in - print_endline ("Step_main '" ^ rule.identifier ^ "' - calling main") ; let%map.Promise { Inductive_rule.previous_proof_statements ; public_output = ret_var ; auxiliary_output = auxiliary_var @@ -286,8 +285,6 @@ let step_main : (* Run the application logic of the rule on the predecessor statements *) with_label "rule_main" (fun () -> rule.main { public_input = app_state }) in - - print_endline ("Step_main '" ^ rule.identifier ^ "' - rest of the circuit") ; with_label "step_main" (fun () -> let () = exists Typ.unit ~request:(fun () -> diff --git a/src/lib/pickles/wrap_main.ml b/src/lib/pickles/wrap_main.ml index 5511a8d8fc6..23624ea56d4 100644 --- a/src/lib/pickles/wrap_main.ml +++ b/src/lib/pickles/wrap_main.ml @@ -133,13 +133,9 @@ let wrap_main in Timer.clock __LOC__ ; let main = - print_endline "wrap main: start" ; let%map.Lazy step_keys = step_keys in - print_endline "wrap main: kicked off lazy step_keys" ; let%bind.Promise step_domains = step_domains in - print_endline "wrap main: awaited step_domains" ; let%map.Promise step_keys = step_keys in - print_endline "wrap main: awaited step_keys" ; fun ({ proof_state = { deferred_values = { plonk From 2f6d07e093675e5b8eb05a0726fda487c27e1562 Mon Sep 17 00:00:00 2001 From: Gregor Date: Fri, 23 Feb 2024 16:43:38 +0100 Subject: [PATCH 27/45] minimal change to make it work --- src/lib/pickles/wrap_main.ml | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lib/pickles/wrap_main.ml b/src/lib/pickles/wrap_main.ml index 23624ea56d4..4ae2356b806 100644 --- a/src/lib/pickles/wrap_main.ml +++ b/src/lib/pickles/wrap_main.ml @@ -136,6 +136,7 @@ let wrap_main let%map.Lazy step_keys = step_keys in let%bind.Promise step_domains = step_domains in let%map.Promise step_keys = step_keys in + print_endline ":(" ; fun ({ proof_state = { deferred_values = { plonk From 61f04c534fd2bf3caa8a8bb1f218d3d1f914af0f Mon Sep 17 00:00:00 2001 From: Gregor Mitscha-Baude Date: Fri, 23 Feb 2024 23:22:39 +0100 Subject: [PATCH 28/45] fix promise map call semantics --- src/lib/pickles/wrap_main.ml | 1 - src/lib/promise/js/promise.js | 29 +++++++++++------------------ 2 files changed, 11 insertions(+), 19 deletions(-) diff --git a/src/lib/pickles/wrap_main.ml b/src/lib/pickles/wrap_main.ml index 4ae2356b806..23624ea56d4 100644 --- a/src/lib/pickles/wrap_main.ml +++ b/src/lib/pickles/wrap_main.ml @@ -136,7 +136,6 @@ let wrap_main let%map.Lazy step_keys = step_keys in let%bind.Promise step_domains = step_domains in let%map.Promise step_keys = step_keys in - print_endline ":(" ; fun ({ proof_state = { deferred_values = { plonk diff --git a/src/lib/promise/js/promise.js b/src/lib/promise/js/promise.js index fa79a4aada3..c0ee877230d 100644 --- a/src/lib/promise/js/promise.js +++ b/src/lib/promise/js/promise.js @@ -21,25 +21,18 @@ function deferred_run(func) { } // Provides: deferred_map +// Requires: deferred_of_promise function deferred_map(deferred, func) { - var newDeferred = { - promise: deferred.promise - .then(func) // the ocaml types don't know this, but func can actually be async or sync - .then(function (value) { - newDeferred.value = value; - newDeferred.isDetermined = true; - return value; - }) - .catch(function (err) { - newDeferred.error = err; - newDeferred.isError = true; - newDeferred.isDetermined = true; - throw err; - }), - isError: false, - isDetermined: false, - }; - return newDeferred; + return deferred_of_promise( + deferred.promise.then(function (value) { + // we might be given a `func` with multiple arguments, + // have to match ocaml call semantics + if (func.length === 1) return func(value); + return function () { + return func.apply(null, [value].concat(Array.from(arguments))); + }; + }) + ); } // Provides: deferred_bind From 5f22f9640c27044f4f2c17aa199befd1cfa40761 Mon Sep 17 00:00:00 2001 From: Gregor Mitscha-Baude Date: Fri, 23 Feb 2024 23:24:45 +0100 Subject: [PATCH 29/45] promise: fix compiler warnings --- src/lib/promise/js/promise.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/lib/promise/js/promise.js b/src/lib/promise/js/promise.js index c0ee877230d..d8c812bc5ff 100644 --- a/src/lib/promise/js/promise.js +++ b/src/lib/promise/js/promise.js @@ -1,7 +1,7 @@ // Provides: deferred_run function deferred_run(func) { var deferred = { - promise: Promise.resolve() + promise: globalThis.Promise.resolve() .then(func) // the ocaml types don't know this, but func can actually be async or sync .then(function (value) { deferred.value = value; @@ -92,7 +92,7 @@ function deferred_peek(deferred) { // Provides: deferred_value_exn function deferred_value_exn(deferred) { if (!deferred.isDetermined) { - throw Error("Deferred has not returned yet."); + throw Error('Deferred has not returned yet.'); } if (deferred.isError) { throw deferred.error; @@ -103,7 +103,7 @@ function deferred_value_exn(deferred) { // Provides: deferred_return function deferred_return(value) { return { - promise: Promise.resolve(value), + promise: globalThis.Promise.resolve(value), value: value, isError: false, isDetermined: true, @@ -113,7 +113,7 @@ function deferred_return(value) { // Provides: deferred_create function deferred_create(promise_creator) { var deferred = { - promise: new Promise(function (resolve) { + promise: new globalThis.Promise(function (resolve) { promise_creator(resolve); }) .then(function (value) { From 69803a76600191d561a8ca0b815b6857afadea14 Mon Sep 17 00:00:00 2001 From: Gregor Mitscha-Baude Date: Fri, 23 Feb 2024 23:34:15 +0100 Subject: [PATCH 30/45] promise: add clarifying comment for where we don't support ocaml calls --- src/lib/promise/js/promise.js | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/lib/promise/js/promise.js b/src/lib/promise/js/promise.js index d8c812bc5ff..d872a4cedb2 100644 --- a/src/lib/promise/js/promise.js +++ b/src/lib/promise/js/promise.js @@ -1,5 +1,13 @@ // Provides: deferred_run function deferred_run(func) { + if (func.length > 1) { + // we add this restriction to be able to use .then(func) below, + // which allows us to implement external functions that are synchronous + // in native Rust with async functions in JS + throw Error( + 'deferred_run cannot be called with a function that takes more than 1 argument.' + ); + } var deferred = { promise: globalThis.Promise.resolve() .then(func) // the ocaml types don't know this, but func can actually be async or sync @@ -39,8 +47,8 @@ function deferred_map(deferred, func) { function deferred_bind(deferred, func) { var newDeferred = { promise: deferred.promise - .then(func) - .then(function (anotherDeferred) { + .then(function (input) { + var anotherDeferred = func(input); return anotherDeferred.promise; }) .then(function (value) { From 71c29319067126727ac37b52d669705233dd3a4e Mon Sep 17 00:00:00 2001 From: Gregor Mitscha-Baude Date: Fri, 23 Feb 2024 23:40:08 +0100 Subject: [PATCH 31/45] cleanup by reusing deferred_of_promise --- src/lib/promise/js/promise.js | 71 ++++++++--------------------------- 1 file changed, 16 insertions(+), 55 deletions(-) diff --git a/src/lib/promise/js/promise.js b/src/lib/promise/js/promise.js index d872a4cedb2..0f8a43a2b84 100644 --- a/src/lib/promise/js/promise.js +++ b/src/lib/promise/js/promise.js @@ -1,4 +1,5 @@ // Provides: deferred_run +// Requires: deferred_of_promise function deferred_run(func) { if (func.length > 1) { // we add this restriction to be able to use .then(func) below, @@ -8,24 +9,10 @@ function deferred_run(func) { 'deferred_run cannot be called with a function that takes more than 1 argument.' ); } - var deferred = { - promise: globalThis.Promise.resolve() - .then(func) // the ocaml types don't know this, but func can actually be async or sync - .then(function (value) { - deferred.value = value; - deferred.isDetermined = true; - return value; - }) - .catch(function (err) { - deferred.error = err; - deferred.isError = true; - deferred.isDetermined = true; - throw err; - }), - isError: false, - isDetermined: false, - }; - return deferred; + return deferred_of_promise( + // the ocaml types don't know this, but func can actually be async or sync + globalThis.Promise.resolve().then(func) + ); } // Provides: deferred_map @@ -44,28 +31,14 @@ function deferred_map(deferred, func) { } // Provides: deferred_bind +// Requires: deferred_of_promise function deferred_bind(deferred, func) { - var newDeferred = { - promise: deferred.promise - .then(function (input) { - var anotherDeferred = func(input); - return anotherDeferred.promise; - }) - .then(function (value) { - newDeferred.value = value; - newDeferred.isDetermined = true; - return value; - }) - .catch(function (err) { - newDeferred.error = err; - newDeferred.isError = true; - newDeferred.isDetermined = true; - throw err; - }), - isError: false, - isDetermined: false, - }; - return newDeferred; + return deferred_of_promise( + deferred.promise.then(function (input) { + var anotherDeferred = func(input); + return anotherDeferred.promise; + }) + ); } // Provides: deferred_upon @@ -119,25 +92,13 @@ function deferred_return(value) { } // Provides: deferred_create +// Requires: deferred_of_promise function deferred_create(promise_creator) { - var deferred = { - promise: new globalThis.Promise(function (resolve) { + return deferred_of_promise( + new globalThis.Promise(function (resolve) { promise_creator(resolve); }) - .then(function (value) { - deferred.value = value; - deferred.isDetermined = true; - }) - .catch(function (err) { - deferred.error = err; - deferred.isError = true; - deferred.isDetermined = true; - throw err; - }), - isError: false, - isDetermined: false, - }; - return deferred; + ); } // Provides: deferred_to_promise From 6424eb75cdcf96d545e2a207846e772342380470 Mon Sep 17 00:00:00 2001 From: Gregor Date: Mon, 26 Feb 2024 10:28:45 +0100 Subject: [PATCH 32/45] make create_trivial_snapp return deferred vk --- .../zkapp_test_transaction/lib/commands.ml | 9 +- .../zkapp_command_generators.ml | 2 + src/lib/mina_graphql/itn_zkapps.ml | 2 + src/lib/network_pool/transaction_pool.ml | 2 + src/lib/staged_ledger/staged_ledger.ml | 148 ++++++++++-------- src/lib/transaction_snark/test/util.ml | 5 +- src/lib/transaction_snark/test/util.mli | 8 +- .../transaction_snark/transaction_snark.ml | 22 ++- .../transaction_snark_intf.ml | 8 +- 9 files changed, 123 insertions(+), 83 deletions(-) diff --git a/src/app/zkapp_test_transaction/lib/commands.ml b/src/app/zkapp_test_transaction/lib/commands.ml index 54e02252434..16b5bfc4720 100644 --- a/src/app/zkapp_test_transaction/lib/commands.ml +++ b/src/app/zkapp_test_transaction/lib/commands.ml @@ -59,9 +59,11 @@ let gen_proof ?(zkapp_account = None) (zkapp_command : Zkapp_command.t) = (Account.create id Currency.Balance.(of_mina_int_exn 1_000)) |> Or_error.ok_exn in - let _v = - Option.value_map zkapp_account ~default:() ~f:(fun pk -> + let open Async.Deferred.Let_syntax in + let%bind () = + Option.value_map zkapp_account ~default:(Deferred.return ()) ~f:(fun pk -> let `VK vk, `Prover _ = Lazy.force vk_and_prover in + let%map vk = vk in let id = Account_id.create pk Token_id.default in Ledger.get_or_create_account ledger id { (Account.create id Currency.Balance.(of_mina_int_exn 1_000)) with @@ -122,7 +124,6 @@ let gen_proof ?(zkapp_account = None) (zkapp_command : Zkapp_command.t) = , zkapp_command ) ] in - let open Async.Deferred.Let_syntax in let module T = Transaction_snark.Make (struct let constraint_constants = constraint_constants @@ -368,7 +369,7 @@ let create_zkapp_account ~debug ~sender ~sender_nonce ~fee ~fee_payer ; authorization_kind = Signature } in - let zkapp_command = + let%bind zkapp_command = Transaction_snark.For_tests.deploy_snapp ~permissions:Permissions.user_default ~constraint_constants spec in diff --git a/src/lib/mina_generators/zkapp_command_generators.ml b/src/lib/mina_generators/zkapp_command_generators.ml index 67e8e17bed4..388123cf222 100644 --- a/src/lib/mina_generators/zkapp_command_generators.ml +++ b/src/lib/mina_generators/zkapp_command_generators.ml @@ -1744,6 +1744,8 @@ let%test_module _ = let `VK vk, `Prover _ = Lazy.force U.trivial_zkapp + let vk = Async.Thread_safe.block_on_async_exn (fun () -> vk) + let mk_ledger ~num_of_unused_keys () = let keys = List.init 5 ~f:(fun _ -> Keypair.create ()) in let zkapp_keys = List.init 5 ~f:(fun _ -> Keypair.create ()) in diff --git a/src/lib/mina_graphql/itn_zkapps.ml b/src/lib/mina_graphql/itn_zkapps.ml index d72418c9041..d09cea5a173 100644 --- a/src/lib/mina_graphql/itn_zkapps.ml +++ b/src/lib/mina_graphql/itn_zkapps.ml @@ -66,6 +66,7 @@ let deploy_zkapps ~scheduler_tbl ~mina ~ledger ~deployment_fee ~max_cost else Permissions.user_default ) spec in + let%bind zkapp_command = zkapp_command in let%bind () = after wait_span in Deferred.repeat_until_finished () @@ fun () -> @@ -178,6 +179,7 @@ let send_zkapps ~fee_payer_array ~constraint_constants ~tm_end ~scheduler_tbl let `VK vk, `Prover prover = Transaction_snark.For_tests.create_trivial_snapp ~constraint_constants () in + let%bind.Deferred vk = vk in let account_queue = Queue.create () in let num_fee_payers = Array.length fee_payer_array in Deferred.repeat_until_finished (init_tm_next, init_counter) diff --git a/src/lib/network_pool/transaction_pool.ml b/src/lib/network_pool/transaction_pool.ml index ec9640a0ec6..86dab669266 100644 --- a/src/lib/network_pool/transaction_pool.ml +++ b/src/lib/network_pool/transaction_pool.ml @@ -1687,6 +1687,8 @@ let%test_module _ = let `VK vk, `Prover prover = Transaction_snark.For_tests.create_trivial_snapp ~constraint_constants () + let vk = Async.Thread_safe.block_on_async_exn (fun () -> vk) + let dummy_state_view = let state_body = let consensus_constants = diff --git a/src/lib/staged_ledger/staged_ledger.ml b/src/lib/staged_ledger/staged_ledger.ml index d5d4e597a7b..e51c7160c7a 100644 --- a/src/lib/staged_ledger/staged_ledger.ml +++ b/src/lib/staged_ledger/staged_ledger.ml @@ -2384,6 +2384,8 @@ let%test_module "staged ledger tests" = let `VK vk, `Prover zkapp_prover = Transaction_snark.For_tests.create_trivial_snapp ~constraint_constants () + let vk = Async.Thread_safe.block_on_async_exn (fun () -> vk) + let verifier = Async.Thread_safe.block_on_async_exn (fun () -> Verifier.create ~logger ~proof_level ~constraint_constants @@ -4500,8 +4502,8 @@ let%test_module "staged ledger tests" = in let%map zkapp_command = Transaction_snark.For_tests.update_states - ~zkapp_prover_and_vk:(zkapp_prover, vk) ~constraint_constants - spec + ~zkapp_prover_and_vk:(zkapp_prover, Async.Deferred.return vk) + ~constraint_constants spec in let valid_zkapp_command = Zkapp_command.Valid.to_valid ~failed:false @@ -4812,43 +4814,46 @@ let%test_module "staged ledger tests" = [ (* command from A that sets a new verification key *) { status = Applied ; data = - mk_basic_zkapp_command ~keymap - ~fee:(Fee.to_nanomina_int User_command.minimum_fee) - ~fee_payer_pk:a.public_key - ~fee_payer_nonce:(Unsigned.UInt32.of_int 0) - [ mk_basic_node ~account:a ~authorization:Signature - ~update: - { Account_update.Update.noop with - verification_key = Zkapp_basic.Set_or_keep.Set vk_a - } - () - ] + (let%bind.Async.Deferred vk_a = vk_a in + mk_basic_zkapp_command ~keymap + ~fee:(Fee.to_nanomina_int User_command.minimum_fee) + ~fee_payer_pk:a.public_key + ~fee_payer_nonce:(Unsigned.UInt32.of_int 0) + [ mk_basic_node ~account:a ~authorization:Signature + ~update: + { Account_update.Update.noop with + verification_key = Zkapp_basic.Set_or_keep.Set vk_a + } + () + ] ) } ; (* command from B that sets a different verification key *) { status = Applied ; data = - mk_basic_zkapp_command ~keymap - ~fee:(Fee.to_nanomina_int User_command.minimum_fee) - ~fee_payer_pk:a.public_key - ~fee_payer_nonce:(Unsigned.UInt32.of_int 1) - [ mk_basic_node ~account:b ~authorization:Signature - ~update: - { Account_update.Update.noop with - verification_key = Zkapp_basic.Set_or_keep.Set vk_b - } - () - ] + (let%bind.Async.Deferred vk_b = vk_b in + mk_basic_zkapp_command ~keymap + ~fee:(Fee.to_nanomina_int User_command.minimum_fee) + ~fee_payer_pk:a.public_key + ~fee_payer_nonce:(Unsigned.UInt32.of_int 1) + [ mk_basic_node ~account:b ~authorization:Signature + ~update: + { Account_update.Update.noop with + verification_key = Zkapp_basic.Set_or_keep.Set vk_b + } + () + ] ) } ; (* proven command from A that is valid against the previously set verification key *) { status = Applied ; data = - mk_basic_zkapp_command ~prover:prover_a ~keymap - ~fee:(Fee.to_nanomina_int User_command.minimum_fee) - ~fee_payer_pk:a.public_key - ~fee_payer_nonce:(Unsigned.UInt32.of_int 2) - [ mk_basic_node ~account:a ~authorization:(Proof vk_a.hash) - () - ] + (let%bind.Async.Deferred vk_a = vk_a in + mk_basic_zkapp_command ~prover:prover_a ~keymap + ~fee:(Fee.to_nanomina_int User_command.minimum_fee) + ~fee_payer_pk:a.public_key + ~fee_payer_nonce:(Unsigned.UInt32.of_int 2) + [ mk_basic_node ~account:a ~authorization:(Proof vk_a.hash) + () + ] ) } ] ) @@ -4872,50 +4877,53 @@ let%test_module "staged ledger tests" = [ (* successful command from A that sets verification key *) { status = Applied ; data = - mk_basic_zkapp_command ~keymap - ~fee:(Fee.to_nanomina_int User_command.minimum_fee) - ~fee_payer_pk:a.public_key - ~fee_payer_nonce:(Unsigned.UInt32.of_int 0) - [ mk_basic_node ~account:a ~authorization:Signature - ~update: - { Account_update.Update.noop with - verification_key = Zkapp_basic.Set_or_keep.Set vk_a - } - () - ] + (let%bind.Async.Deferred vk_a = vk_a in + mk_basic_zkapp_command ~keymap + ~fee:(Fee.to_nanomina_int User_command.minimum_fee) + ~fee_payer_pk:a.public_key + ~fee_payer_nonce:(Unsigned.UInt32.of_int 0) + [ mk_basic_node ~account:a ~authorization:Signature + ~update: + { Account_update.Update.noop with + verification_key = Zkapp_basic.Set_or_keep.Set vk_a + } + () + ] ) } (* failing command from A that sets another verification key *) ; { status = Failed [ []; [ Account_nonce_precondition_unsatisfied ] ] ; data = - mk_basic_zkapp_command ~keymap - ~fee:(Fee.to_nanomina_int User_command.minimum_fee) - ~fee_payer_pk:a.public_key - ~fee_payer_nonce:(Unsigned.UInt32.of_int 1) - [ mk_basic_node ~account:a ~authorization:Signature - ~preconditions: - { Account_update.Preconditions.accept with - account = - Zkapp_precondition.Account.nonce - (Account.Nonce.of_int 0) - } - ~update: - { Account_update.Update.noop with - verification_key = Zkapp_basic.Set_or_keep.Set vk_b - } - () - ] + (let%bind.Async.Deferred vk_b = vk_b in + mk_basic_zkapp_command ~keymap + ~fee:(Fee.to_nanomina_int User_command.minimum_fee) + ~fee_payer_pk:a.public_key + ~fee_payer_nonce:(Unsigned.UInt32.of_int 1) + [ mk_basic_node ~account:a ~authorization:Signature + ~preconditions: + { Account_update.Preconditions.accept with + account = + Zkapp_precondition.Account.nonce + (Account.Nonce.of_int 0) + } + ~update: + { Account_update.Update.noop with + verification_key = Zkapp_basic.Set_or_keep.Set vk_b + } + () + ] ) } ; (* proven command from A that is valid against the first verification key only *) { status = Applied ; data = - mk_basic_zkapp_command ~prover:prover_a ~keymap - ~fee:(Fee.to_nanomina_int User_command.minimum_fee) - ~fee_payer_pk:a.public_key - ~fee_payer_nonce:(Unsigned.UInt32.of_int 2) - [ mk_basic_node ~account:a ~authorization:(Proof vk_a.hash) - () - ] + (let%bind.Async.Deferred vk_a = vk_a in + mk_basic_zkapp_command ~prover:prover_a ~keymap + ~fee:(Fee.to_nanomina_int User_command.minimum_fee) + ~fee_payer_pk:a.public_key + ~fee_payer_nonce:(Unsigned.UInt32.of_int 2) + [ mk_basic_node ~account:a ~authorization:(Proof vk_a.hash) + () + ] ) } ] ) @@ -4981,7 +4989,9 @@ let%test_module "staged ledger tests" = l in let%bind zkapp_command = - let zkapp_prover_and_vk = (zkapp_prover, vk) in + let zkapp_prover_and_vk = + (zkapp_prover, Async.Deferred.return vk) + in Transaction_snark.For_tests.update_states ~zkapp_prover_and_vk ~constraint_constants test_spec in @@ -5106,7 +5116,9 @@ let%test_module "staged ledger tests" = Signature_lib.Public_key.compress zkapp_account_keypair.public_key in - let zkapp_prover_and_vk = (zkapp_prover, vk) in + let zkapp_prover_and_vk = + (zkapp_prover, Async.Deferred.return vk) + in let%bind zkapp_command = single_account_update ~chain:Mina_signature_kind.(Other_network "invalid") diff --git a/src/lib/transaction_snark/test/util.ml b/src/lib/transaction_snark/test/util.ml index ef33d88e328..ed4fdbbd975 100644 --- a/src/lib/transaction_snark/test/util.ml +++ b/src/lib/transaction_snark/test/util.ml @@ -384,10 +384,11 @@ let test_snapp_update ?expected_failure ?state_body ?snapp_permissions ~vk Ledger.with_ledger ~depth:ledger_depth ~f:(fun ledger -> Async.Thread_safe.block_on_async_exn (fun () -> Init_ledger.init (module Ledger.Ledger_inner) init_ledger ledger ; + let open Async.Deferred.Let_syntax in (*create a snapp account*) + let%bind vk' = vk in Transaction_snark.For_tests.create_trivial_zkapp_account - ?permissions:snapp_permissions ~vk ~ledger snapp_pk ; - let open Async.Deferred.Let_syntax in + ?permissions:snapp_permissions ~vk:vk' ~ledger snapp_pk ; let%bind zkapp_command = let zkapp_prover_and_vk = (zkapp_prover, vk) in Transaction_snark.For_tests.update_states ~zkapp_prover_and_vk diff --git a/src/lib/transaction_snark/test/util.mli b/src/lib/transaction_snark/test/util.mli index fec7ea8d4a4..f41f5086f3f 100644 --- a/src/lib/transaction_snark/test/util.mli +++ b/src/lib/transaction_snark/test/util.mli @@ -66,7 +66,9 @@ val check_zkapp_command_with_merges_exn : (** Verification key of a trivial smart contract *) val trivial_zkapp : - ( [> `VK of (Side_loaded_verification_key.t, Tick.Field.t) With_hash.t ] + ( [> `VK of + (Side_loaded_verification_key.t, Tick.Field.t) With_hash.t + Async.Deferred.t ] * [> `Prover of ( unit , unit @@ -87,7 +89,9 @@ val test_snapp_update : ?expected_failure:Mina_base.Transaction_status.Failure.t * pass_number -> ?state_body:Transaction_protocol_state.Block_data.t -> ?snapp_permissions:Permissions.t - -> vk:(Side_loaded_verification_key.t, Tick.Field.t) With_hash.t + -> vk: + (Side_loaded_verification_key.t, Tick.Field.t) With_hash.t + Async.Deferred.t -> zkapp_prover: ( unit , unit diff --git a/src/lib/transaction_snark/transaction_snark.ml b/src/lib/transaction_snark/transaction_snark.ml index c4e7bff755a..818dd90e7f3 100644 --- a/src/lib/transaction_snark/transaction_snark.ml +++ b/src/lib/transaction_snark/transaction_snark.ml @@ -4193,11 +4193,13 @@ module Make_str (A : Wire_types.Concrete) = struct let%map (), (), proof = trivial_prover ?handler stmt in ((), (), Pickles.Side_loaded.Proof.of_proof proof) in - let vk = Pickles.Side_loaded.Verification_key.of_compiled tag in - (* we just block on the Deferred since this is only used for tests *) - let vk = Async.Thread_safe.block_on_async_exn (fun () -> vk) in - ( `VK (With_hash.of_data ~hash_data:Zkapp_account.digest_vk vk) - , `Prover trivial_prover ) + let vk = + let%map.Async.Deferred vk = + Pickles.Side_loaded.Verification_key.of_compiled tag + in + With_hash.of_data ~hash_data:Zkapp_account.digest_vk vk + in + (`VK vk, `Prover trivial_prover) let%test_unit "creating trivial zkapps with different nonces makes unique \ verification keypairs" = @@ -4229,6 +4231,12 @@ module Make_str (A : Wire_types.Concrete) = struct let `VK vk_b, `Prover prover_b = create_trivial_snapp ~unique_id:1 ~constraint_constants () in + let vk_a, vk_b = + Async.Thread_safe.block_on_async_exn (fun () -> + let%bind vk_a = vk_a in + let%map vk_b = vk_b in + (vk_a, vk_b) ) + in assert ( not ([%equal: @@ -4584,6 +4592,7 @@ module Make_str (A : Wire_types.Concrete) = struct let `VK vk, `Prover _trivial_prover = create_trivial_snapp ~constraint_constants () in + let%map.Async.Deferred vk = vk in (* only allow timing on a single new snapp account balance changes for other new snapp accounts are just the account creation fee *) @@ -4714,6 +4723,7 @@ module Make_str (A : Wire_types.Concrete) = struct | None -> create_trivial_snapp ~constraint_constants () in + let%bind.Async.Deferred vk = vk in let ( `Zkapp_command { Zkapp_command.fee_payer; memo; _ } , `Sender_account_update _ , `Proof_zkapp_command _ @@ -4867,6 +4877,7 @@ module Make_str (A : Wire_types.Concrete) = struct in (prover, vk) in + let%bind.Async.Deferred vk = vk in let ( `Zkapp_command ({ Zkapp_command.fee_payer; memo; _ } as p) , `Sender_account_update sender_account_update , `Proof_zkapp_command snapp_zkapp_command @@ -5057,6 +5068,7 @@ module Make_str (A : Wire_types.Concrete) = struct let `VK vk, `Prover trivial_prover = create_trivial_snapp ~constraint_constants () in + let%bind.Async.Deferred vk = vk in let _v = let id = Public_key.compress sender.public_key diff --git a/src/lib/transaction_snark/transaction_snark_intf.ml b/src/lib/transaction_snark/transaction_snark_intf.ml index d55581c0c43..4499b0b7de5 100644 --- a/src/lib/transaction_snark/transaction_snark_intf.ml +++ b/src/lib/transaction_snark/transaction_snark_intf.ml @@ -286,7 +286,7 @@ module type Full = sig -> ?permissions:Permissions.t -> constraint_constants:Genesis_constants.Constraint_constants.t -> Deploy_snapp_spec.t - -> Zkapp_command.t + -> Zkapp_command.t Async.Deferred.t module Single_account_update_spec : sig type t = @@ -313,6 +313,7 @@ module type Full = sig * ( Pickles.Side_loaded.Verification_key.t , Snark_params.Tick.Field.t ) With_hash.t + Async.Deferred.t -> chain:Mina_signature_kind.t -> constraint_constants:Genesis_constants.Constraint_constants.t -> Single_account_update_spec.t @@ -353,6 +354,7 @@ module type Full = sig * ( Pickles.Side_loaded.Verification_key.t , Snark_params.Tick.Field.t ) With_hash.t + Async.Deferred.t -> ?empty_sender:bool -> constraint_constants:Genesis_constants.Constraint_constants.t -> Update_states_spec.t @@ -383,7 +385,9 @@ module type Full = sig ?unique_id:int -> constraint_constants:Genesis_constants.Constraint_constants.t -> unit - -> [> `VK of (Side_loaded_verification_key.t, Tick.Field.t) With_hash.t ] + -> [> `VK of + (Side_loaded_verification_key.t, Tick.Field.t) With_hash.t + Async.Deferred.t ] * [> `Prover of ( unit , unit From e3d9d025f2e26adb6bd220cf4bd85311f9fa4c94 Mon Sep 17 00:00:00 2001 From: Gregor Date: Mon, 26 Feb 2024 11:09:45 +0100 Subject: [PATCH 33/45] adapt tx snark tests --- .../test/account_timing/account_timing.ml | 9 +++++-- .../account_update_network_id.ml | 2 ++ .../test/fee_payer/fee_payer.ml | 5 ++-- .../transaction_union/transaction_union.ml | 2 ++ ...nsaction_snark_tests_verify_simple_test.ml | 5 +++- .../test/zkapp_deploy/zkapp_deploy.ml | 25 +++++++++++-------- .../test/zkapp_fuzzy/zkapp_fuzzy.ml | 2 ++ .../zkapp_preconditions.ml | 12 ++++++--- 8 files changed, 43 insertions(+), 19 deletions(-) diff --git a/src/lib/transaction_snark/test/account_timing/account_timing.ml b/src/lib/transaction_snark/test/account_timing/account_timing.ml index 8f15258b3d3..e27b8dc4372 100644 --- a/src/lib/transaction_snark/test/account_timing/account_timing.ml +++ b/src/lib/transaction_snark/test/account_timing/account_timing.ml @@ -1670,7 +1670,10 @@ let%test_module "account timing check" = , create_timed_account_spec.snapp_update , zkapp_keypair ) in - return (ledger_init_state, zkapp_command) + + return + ( ledger_init_state + , Async.Thread_safe.block_on_async_exn (fun () -> zkapp_command) ) let%test_unit "zkApp command, timed account creation, min_balance > balance" = @@ -2258,7 +2261,9 @@ let%test_module "account timing check" = in let gen = Quickcheck.Generator.return - (ledger_init_state, create_timed_account_zkapp_command) + ( ledger_init_state + , Async.Thread_safe.block_on_async_exn (fun () -> + create_timed_account_zkapp_command ) ) in Async.Thread_safe.block_on_async_exn (fun () -> Async.Quickcheck.async_test diff --git a/src/lib/transaction_snark/test/account_update_network_id/account_update_network_id.ml b/src/lib/transaction_snark/test/account_update_network_id/account_update_network_id.ml index ca174038a4e..c1727b6ff69 100644 --- a/src/lib/transaction_snark/test/account_update_network_id/account_update_network_id.ml +++ b/src/lib/transaction_snark/test/account_update_network_id/account_update_network_id.ml @@ -8,6 +8,8 @@ let%test_module "Account update network id tests" = ( module struct let `VK vk, `Prover _ = Lazy.force U.trivial_zkapp + let vk = Async.Thread_safe.block_on_async_exn (fun () -> vk) + let%test_unit "zkapps failed to apply with a different network id" = let open Mina_transaction_logic.For_tests in Quickcheck.test ~trials:1 (Test_spec.mk_gen ~num_transactions:1 ()) diff --git a/src/lib/transaction_snark/test/fee_payer/fee_payer.ml b/src/lib/transaction_snark/test/fee_payer/fee_payer.ml index 16f73c25f46..797550f2949 100644 --- a/src/lib/transaction_snark/test/fee_payer/fee_payer.ml +++ b/src/lib/transaction_snark/test/fee_payer/fee_payer.ml @@ -164,8 +164,9 @@ let%test_module "Fee payer tests" = } in let zkapp_command = - Transaction_snark.For_tests.deploy_snapp test_spec - ~constraint_constants + Async.Thread_safe.block_on_async_exn (fun () -> + Transaction_snark.For_tests.deploy_snapp test_spec + ~constraint_constants ) in let txn_state_view = Mina_state.Protocol_state.Body.view U.genesis_state_body diff --git a/src/lib/transaction_snark/test/transaction_union/transaction_union.ml b/src/lib/transaction_snark/test/transaction_union/transaction_union.ml index 965614e7274..cd9ff0b28c9 100644 --- a/src/lib/transaction_snark/test/transaction_union/transaction_union.ml +++ b/src/lib/transaction_snark/test/transaction_union/transaction_union.ml @@ -2042,6 +2042,8 @@ let%test_module "legacy transactions using zkApp accounts" = let `VK vk, `Prover _zkapp_prover = Lazy.force U.trivial_zkapp + let vk = Async.Thread_safe.block_on_async_exn (fun () -> vk) + let account ledger pk = let location = Option.value_exn diff --git a/src/lib/transaction_snark/test/verify-simple-test/transaction_snark_tests_verify_simple_test.ml b/src/lib/transaction_snark/test/verify-simple-test/transaction_snark_tests_verify_simple_test.ml index 23a48852b7c..e4dfdab5ac3 100644 --- a/src/lib/transaction_snark/test/verify-simple-test/transaction_snark_tests_verify_simple_test.ml +++ b/src/lib/transaction_snark/test/verify-simple-test/transaction_snark_tests_verify_simple_test.ml @@ -1,10 +1,13 @@ open Core_kernel open Mina_base -let `VK { With_hash.data = vk; hash = _ }, `Prover p = +let `VK vk, `Prover p = Transaction_snark.For_tests.create_trivial_snapp ~constraint_constants:Genesis_constants.Constraint_constants.compiled () +let { With_hash.data = vk; hash = _ } = + Async.Thread_safe.block_on_async_exn (fun () -> vk) + let vk = vk |> Binable.to_string diff --git a/src/lib/transaction_snark/test/zkapp_deploy/zkapp_deploy.ml b/src/lib/transaction_snark/test/zkapp_deploy/zkapp_deploy.ml index df99a3d893d..259f64fcbbb 100644 --- a/src/lib/transaction_snark/test/zkapp_deploy/zkapp_deploy.ml +++ b/src/lib/transaction_snark/test/zkapp_deploy/zkapp_deploy.ml @@ -39,8 +39,9 @@ let%test_module "zkApp deploy tests" = } in let zkapp_command = - Transaction_snark.For_tests.deploy_snapp test_spec - ~constraint_constants + Async.Thread_safe.block_on_async_exn (fun () -> + Transaction_snark.For_tests.deploy_snapp test_spec + ~constraint_constants ) in Init_ledger.init (module Ledger.Ledger_inner) @@ -77,8 +78,9 @@ let%test_module "zkApp deploy tests" = } in let zkapp_command = - Transaction_snark.For_tests.deploy_snapp test_spec - ~constraint_constants + Async.Thread_safe.block_on_async_exn (fun () -> + Transaction_snark.For_tests.deploy_snapp test_spec + ~constraint_constants ) in Init_ledger.init (module Ledger.Ledger_inner) @@ -109,8 +111,9 @@ let%test_module "zkApp deploy tests" = } in let zkapp_command = - Transaction_snark.For_tests.deploy_snapp test_spec - ~constraint_constants + Async.Thread_safe.block_on_async_exn (fun () -> + Transaction_snark.For_tests.deploy_snapp test_spec + ~constraint_constants ) in Init_ledger.init (module Ledger.Ledger_inner) @@ -142,8 +145,9 @@ let%test_module "zkApp deploy tests" = } in let zkapp_command = - Transaction_snark.For_tests.deploy_snapp test_spec - ~constraint_constants + Async.Thread_safe.block_on_async_exn (fun () -> + Transaction_snark.For_tests.deploy_snapp test_spec + ~constraint_constants ) in Init_ledger.init (module Ledger.Ledger_inner) @@ -175,8 +179,9 @@ let%test_module "zkApp deploy tests" = } in let zkapp_command = - Transaction_snark.For_tests.deploy_snapp test_spec - ~constraint_constants + Async.Thread_safe.block_on_async_exn (fun () -> + Transaction_snark.For_tests.deploy_snapp test_spec + ~constraint_constants ) in Init_ledger.init (module Ledger.Ledger_inner) diff --git a/src/lib/transaction_snark/test/zkapp_fuzzy/zkapp_fuzzy.ml b/src/lib/transaction_snark/test/zkapp_fuzzy/zkapp_fuzzy.ml index e9902915780..05c1a6542ac 100644 --- a/src/lib/transaction_snark/test/zkapp_fuzzy/zkapp_fuzzy.ml +++ b/src/lib/transaction_snark/test/zkapp_fuzzy/zkapp_fuzzy.ml @@ -8,6 +8,8 @@ let logger = Logger.create () let `VK vk, `Prover prover = Lazy.force U.trivial_zkapp +let vk = Async.Thread_safe.block_on_async_exn (fun () -> vk) + let mk_ledgers_and_fee_payers ?(is_timed = false) ~num_of_fee_payers () = let fee_payer_keypairs = Array.init num_of_fee_payers ~f:(fun _ -> Keypair.create ()) diff --git a/src/lib/transaction_snark/test/zkapp_preconditions/zkapp_preconditions.ml b/src/lib/transaction_snark/test/zkapp_preconditions/zkapp_preconditions.ml index 28844f270e5..8fceedc5e5e 100644 --- a/src/lib/transaction_snark/test/zkapp_preconditions/zkapp_preconditions.ml +++ b/src/lib/transaction_snark/test/zkapp_preconditions/zkapp_preconditions.ml @@ -20,6 +20,10 @@ let%test_module "Valid_while precondition tests" = let `VK vk, `Prover zkapp_prover = Lazy.force U.trivial_zkapp + let zkapp_prover_and_vk = (zkapp_prover, vk) + + let vk = Async.Thread_safe.block_on_async_exn (fun () -> vk) + let snapp_update : Account_update.Update.t = { Account_update.Update.dummy with app_state = @@ -71,8 +75,7 @@ let%test_module "Valid_while precondition tests" = let open Async.Deferred.Let_syntax in let%bind zkapp_command = Transaction_snark.For_tests.update_states - ~zkapp_prover_and_vk:(zkapp_prover, vk) - ~constraint_constants + ~zkapp_prover_and_vk ~constraint_constants (create_spec specs new_kp global_slot) in U.check_zkapp_command_with_merges_exn ~global_slot ledger @@ -95,8 +98,7 @@ let%test_module "Valid_while precondition tests" = let open Async.Deferred.Let_syntax in let%bind zkapp_command = Transaction_snark.For_tests.update_states - ~zkapp_prover_and_vk:(zkapp_prover, vk) - ~constraint_constants + ~zkapp_prover_and_vk ~constraint_constants (create_spec specs new_kp global_slot) in U.check_zkapp_command_with_merges_exn @@ -423,6 +425,8 @@ let%test_module "Account precondition tests" = let zkapp_prover_and_vk = (zkapp_prover, vk) + let vk = Async.Thread_safe.block_on_async_exn (fun () -> vk) + let constraint_constants = U.constraint_constants let memo = Signed_command_memo.create_from_string_exn "account precondition" From daa71cb792482311fde26786e4752d1a165810dc Mon Sep 17 00:00:00 2001 From: Gregor Date: Mon, 26 Feb 2024 11:23:07 +0100 Subject: [PATCH 34/45] start fixing snark profiler lib --- src/app/cli/src/init/transaction_snark_profiler.ml | 3 ++- src/app/heap_usage/values.ml | 2 +- src/lib/snark_profiler_lib/snark_profiler_lib.ml | 9 ++++++--- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/app/cli/src/init/transaction_snark_profiler.ml b/src/app/cli/src/init/transaction_snark_profiler.ml index 28fab7f93c9..9f3555d1ff2 100644 --- a/src/app/cli/src/init/transaction_snark_profiler.ml +++ b/src/app/cli/src/init/transaction_snark_profiler.ml @@ -9,7 +9,8 @@ let run ~user_command_profiler ~zkapp_profiler num_transactions ~max_num_updates let print n msg = printf !"[%i] %s\n%!" n msg in if use_zkapps then ( let ledger, transactions = - create_ledger_and_zkapps ?min_num_updates ~max_num_updates () + Async.Thread_safe.block_on_async_exn (fun () -> + create_ledger_and_zkapps ?min_num_updates ~max_num_updates () ) in Parallel.init_master () ; let verifier = diff --git a/src/app/heap_usage/values.ml b/src/app/heap_usage/values.ml index ca65b9545d9..525e96e382c 100644 --- a/src/app/heap_usage/values.ml +++ b/src/app/heap_usage/values.ml @@ -35,7 +35,7 @@ let account : Mina_base.Account.t = (* beefy zkapp command with all proof updates *) let zkapp_command = let num_updates = 16 in - let _ledger, zkapp_commands = + let%map.Async.Deferred _ledger, zkapp_commands = Snark_profiler_lib.create_ledger_and_zkapps ~min_num_updates:num_updates ~num_proof_updates:num_updates ~max_num_updates:num_updates () in diff --git a/src/lib/snark_profiler_lib/snark_profiler_lib.ml b/src/lib/snark_profiler_lib/snark_profiler_lib.ml index 51dcbcfed38..e3815db5ca5 100644 --- a/src/lib/snark_profiler_lib/snark_profiler_lib.ml +++ b/src/lib/snark_profiler_lib/snark_profiler_lib.ml @@ -187,10 +187,13 @@ end let transaction_combinations = Transaction_key.Table.create () let create_ledger_and_zkapps ?(min_num_updates = 1) ?(num_proof_updates = 0) - ~max_num_updates () : Mina_ledger.Ledger.t * Zkapp_command.t list = + ~max_num_updates () : + (Mina_ledger.Ledger.t * Zkapp_command.t list) Async.Deferred.t = let `VK verification_key, `Prover prover = Transaction_snark.For_tests.create_trivial_snapp ~constraint_constants () in + let zkapp_prover_and_vk = (prover, verification_key) in + let%map.Async.Deferred verification_key = verification_key in let num_keypairs = max_num_updates + 10 in let keypairs = List.init num_keypairs ~f:(fun _ -> Keypair.create ()) in let num_keypairs_in_ledger = max_num_updates + 1 in @@ -353,8 +356,7 @@ let create_ledger_and_zkapps ?(min_num_updates = 1) ?(num_proof_updates = 0) in let parties = Async.Thread_safe.block_on_async_exn (fun () -> - Transaction_snark.For_tests.update_states - ~zkapp_prover_and_vk:(prover, verification_key) + Transaction_snark.For_tests.update_states ~zkapp_prover_and_vk ~constraint_constants ~empty_sender spec ~receiver_auth:Control.Tag.Signature ) in @@ -450,6 +452,7 @@ let _create_ledger_and_zkapps_from_generator num_transactions : let `VK vk, `Prover prover = Transaction_snark.For_tests.create_trivial_snapp ~constraint_constants () in + let vk = Async.Thread_safe.block_on_async_exn (fun () -> vk) in let cmd_infos, ledger = Quickcheck.random_value (Mina_generators.User_command_generators From ec91a5010ab0636a521e360989e0bf72a9848844 Mon Sep 17 00:00:00 2001 From: Gregor Date: Mon, 26 Feb 2024 11:40:07 +0100 Subject: [PATCH 35/45] fixup heap usage app --- src/app/heap_usage/heap_usage.ml | 5 +++++ src/app/heap_usage/values.ml | 3 +++ 2 files changed, 8 insertions(+) diff --git a/src/app/heap_usage/heap_usage.ml b/src/app/heap_usage/heap_usage.ml index dbf1fb98e8c..47809cb52c2 100644 --- a/src/app/heap_usage/heap_usage.ml +++ b/src/app/heap_usage/heap_usage.ml @@ -14,6 +14,11 @@ let print_heap_usage name v = let main () = let open Values in + let%bind.Deferred zkapp_command = zkapp_command in + let%bind.Deferred zkapp_proof = zkapp_proof in + let%bind.Deferred verification_key = verification_key in + let%bind.Deferred scan_state_base_node_zkapp = scan_state_base_node_zkapp in + print_heap_usage "Account.t (w/ zkapp)" account ; print_heap_usage "Zkapp_command.t" zkapp_command ; print_heap_usage "Pickles.Side_loaded.Proof.t" zkapp_proof ; diff --git a/src/app/heap_usage/values.ml b/src/app/heap_usage/values.ml index 525e96e382c..1a08ecd0ba0 100644 --- a/src/app/heap_usage/values.ml +++ b/src/app/heap_usage/values.ml @@ -42,6 +42,7 @@ let zkapp_command = List.hd_exn zkapp_commands let zkapp_proof = + let%map.Async.Deferred zkapp_command = zkapp_command in List.fold_until (Mina_base.Zkapp_command.all_account_updates_list zkapp_command) ~init:None @@ -65,6 +66,7 @@ let verification_key = Transaction_snark.For_tests.create_trivial_snapp ~constraint_constants:Genesis_constants.Constraint_constants.compiled () in + let%map.Async.Deferred vk = vk in With_hash.data vk let applied = Mina_base.Transaction_status.Applied @@ -202,6 +204,7 @@ let scan_state_base_node_payment = mk_scan_state_base_node varying let scan_state_base_node_zkapp = + let%map.Async.Deferred zkapp_command = zkapp_command in let varying : Mina_transaction_logic.Transaction_applied.Varying.t = let zkapp_command_applied : Mina_transaction_logic.Transaction_applied.Zkapp_command_applied.t = From b13f95a4cdb47f1af19f5536d6918caa565979aa Mon Sep 17 00:00:00 2001 From: Gregor Date: Mon, 26 Feb 2024 11:48:34 +0100 Subject: [PATCH 36/45] fixup test_executive --- src/app/test_executive/hard_fork.ml | 5 +++-- .../test_executive/peers_reliability_test.ml | 7 ++++--- .../test_executive/verification_key_update.ml | 3 ++- src/app/test_executive/zkapps.ml | 5 +++-- src/app/test_executive/zkapps_timing.ml | 20 +++++++++++-------- 5 files changed, 24 insertions(+), 16 deletions(-) diff --git a/src/app/test_executive/hard_fork.ml b/src/app/test_executive/hard_fork.ml index fddd635ec07..e29982206d7 100644 --- a/src/app/test_executive/hard_fork.ml +++ b/src/app/test_executive/hard_fork.ml @@ -326,8 +326,9 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct ; authorization_kind = Signature } in - Transaction_snark.For_tests.deploy_snapp ~constraint_constants - zkapp_command_spec + Async.Thread_safe.block_on_async_exn (fun () -> + Transaction_snark.For_tests.deploy_snapp ~constraint_constants + zkapp_command_spec ) in let%bind zkapp_command_update_vk_proof, zkapp_command_update_vk_impossible = diff --git a/src/app/test_executive/peers_reliability_test.ml b/src/app/test_executive/peers_reliability_test.ml index 81bac2725f4..29f52807e2d 100644 --- a/src/app/test_executive/peers_reliability_test.ml +++ b/src/app/test_executive/peers_reliability_test.ml @@ -139,9 +139,10 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct } in return - @@ Transaction_snark.For_tests.deploy_snapp - ~constraint_constants:(Network.constraint_constants network) - parties_spec + @@ Async.Thread_safe.block_on_async_exn (fun () -> + Transaction_snark.For_tests.deploy_snapp + ~constraint_constants:(Network.constraint_constants network) + parties_spec ) in let%bind () = send_zkapp ~logger diff --git a/src/app/test_executive/verification_key_update.ml b/src/app/test_executive/verification_key_update.ml index 608c16ba2e5..2bde56cc562 100644 --- a/src/app/test_executive/verification_key_update.ml +++ b/src/app/test_executive/verification_key_update.ml @@ -220,7 +220,8 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct ; authorization_kind = Signature } in - Transaction_snark.For_tests.deploy_snapp ~constraint_constants spec + Async.Thread_safe.block_on_async_exn (fun () -> + Transaction_snark.For_tests.deploy_snapp ~constraint_constants spec ) in let call_forest_to_zkapp ~call_forest ~nonce : Zkapp_command.t = let memo = Signed_command_memo.empty in diff --git a/src/app/test_executive/zkapps.ml b/src/app/test_executive/zkapps.ml index 9e88b50a91b..0379b53dfd4 100644 --- a/src/app/test_executive/zkapps.ml +++ b/src/app/test_executive/zkapps.ml @@ -179,8 +179,9 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct } in return - @@ Transaction_snark.For_tests.deploy_snapp ~constraint_constants - zkapp_command_spec + @@ Async.Thread_safe.block_on_async_exn (fun () -> + Transaction_snark.For_tests.deploy_snapp ~constraint_constants + zkapp_command_spec ) in let%bind.Deferred zkapp_command_update_permissions, permissions_updated = (* construct a Zkapp_command.t, similar to zkapp_test_transaction update-permissions *) diff --git a/src/app/test_executive/zkapps_timing.ml b/src/app/test_executive/zkapps_timing.ml index 86c0a244b03..66275bf0aee 100644 --- a/src/app/test_executive/zkapps_timing.ml +++ b/src/app/test_executive/zkapps_timing.ml @@ -105,8 +105,9 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct Token_id.default in - ( Transaction_snark.For_tests.deploy_snapp ~constraint_constants - zkapp_command_spec + ( Async.Thread_safe.block_on_async_exn (fun () -> + Transaction_snark.For_tests.deploy_snapp ~constraint_constants + zkapp_command_spec ) , timing_account_id , zkapp_command_spec.snapp_update , zkapp_keypair ) @@ -150,8 +151,9 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct } in return - @@ Transaction_snark.For_tests.deploy_snapp ~constraint_constants - zkapp_command_spec + @@ Async.Thread_safe.block_on_async_exn (fun () -> + Transaction_snark.For_tests.deploy_snapp ~constraint_constants + zkapp_command_spec ) in (* Create a timed account that with initial liquid balance being 0, and vesting 1 mina at each slot. This account would be used to test the edge case of vesting. See `zkapp_command_transfer_from_third_timed_account` @@ -200,8 +202,9 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct (zkapp_keypair.public_key |> Signature_lib.Public_key.compress) Token_id.default in - ( Transaction_snark.For_tests.deploy_snapp ~constraint_constants - zkapp_command_spec + ( Async.Thread_safe.block_on_async_exn (fun () -> + Transaction_snark.For_tests.deploy_snapp ~constraint_constants + zkapp_command_spec ) , timing_account_id , zkapp_keypair ) in @@ -242,8 +245,9 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct ; authorization_kind = Signature } in - Transaction_snark.For_tests.deploy_snapp ~constraint_constants - zkapp_command_spec + Async.Thread_safe.block_on_async_exn (fun () -> + Transaction_snark.For_tests.deploy_snapp ~constraint_constants + zkapp_command_spec ) in let%bind zkapp_command_transfer_from_timed_account = let open Mina_base in From 5650cf8ddada16b6b75cd48f95412380eba81454 Mon Sep 17 00:00:00 2001 From: Gregor Date: Mon, 26 Feb 2024 13:02:46 +0100 Subject: [PATCH 37/45] fix test failures by sequentializing logic --- scripts/zkapp_metrics.sh | 2 +- src/app/cli/src/init/transaction_snark_profiler.ml | 3 +-- src/app/heap_usage/heap_usage.ml | 5 ----- src/app/heap_usage/values.ml | 6 ++---- src/lib/snark_profiler_lib/snark_profiler_lib.ml | 7 ++++--- src/lib/transaction_snark/transaction_snark.ml | 8 ++------ 6 files changed, 10 insertions(+), 21 deletions(-) diff --git a/scripts/zkapp_metrics.sh b/scripts/zkapp_metrics.sh index 8408bfee458..c6fd22721d9 100755 --- a/scripts/zkapp_metrics.sh +++ b/scripts/zkapp_metrics.sh @@ -12,5 +12,5 @@ echo "Running heap usage app" echo "Building zkapp limits app" make zkapp_limits -echo "Running heap usage app" +echo "Running zkapp limits app" ./_build/default/src/app/zkapp_limits/zkapp_limits.exe \ No newline at end of file diff --git a/src/app/cli/src/init/transaction_snark_profiler.ml b/src/app/cli/src/init/transaction_snark_profiler.ml index 9f3555d1ff2..28fab7f93c9 100644 --- a/src/app/cli/src/init/transaction_snark_profiler.ml +++ b/src/app/cli/src/init/transaction_snark_profiler.ml @@ -9,8 +9,7 @@ let run ~user_command_profiler ~zkapp_profiler num_transactions ~max_num_updates let print n msg = printf !"[%i] %s\n%!" n msg in if use_zkapps then ( let ledger, transactions = - Async.Thread_safe.block_on_async_exn (fun () -> - create_ledger_and_zkapps ?min_num_updates ~max_num_updates () ) + create_ledger_and_zkapps ?min_num_updates ~max_num_updates () in Parallel.init_master () ; let verifier = diff --git a/src/app/heap_usage/heap_usage.ml b/src/app/heap_usage/heap_usage.ml index 47809cb52c2..dbf1fb98e8c 100644 --- a/src/app/heap_usage/heap_usage.ml +++ b/src/app/heap_usage/heap_usage.ml @@ -14,11 +14,6 @@ let print_heap_usage name v = let main () = let open Values in - let%bind.Deferred zkapp_command = zkapp_command in - let%bind.Deferred zkapp_proof = zkapp_proof in - let%bind.Deferred verification_key = verification_key in - let%bind.Deferred scan_state_base_node_zkapp = scan_state_base_node_zkapp in - print_heap_usage "Account.t (w/ zkapp)" account ; print_heap_usage "Zkapp_command.t" zkapp_command ; print_heap_usage "Pickles.Side_loaded.Proof.t" zkapp_proof ; diff --git a/src/app/heap_usage/values.ml b/src/app/heap_usage/values.ml index 1a08ecd0ba0..1f281e62865 100644 --- a/src/app/heap_usage/values.ml +++ b/src/app/heap_usage/values.ml @@ -35,14 +35,13 @@ let account : Mina_base.Account.t = (* beefy zkapp command with all proof updates *) let zkapp_command = let num_updates = 16 in - let%map.Async.Deferred _ledger, zkapp_commands = + let _ledger, zkapp_commands = Snark_profiler_lib.create_ledger_and_zkapps ~min_num_updates:num_updates ~num_proof_updates:num_updates ~max_num_updates:num_updates () in List.hd_exn zkapp_commands let zkapp_proof = - let%map.Async.Deferred zkapp_command = zkapp_command in List.fold_until (Mina_base.Zkapp_command.all_account_updates_list zkapp_command) ~init:None @@ -66,7 +65,7 @@ let verification_key = Transaction_snark.For_tests.create_trivial_snapp ~constraint_constants:Genesis_constants.Constraint_constants.compiled () in - let%map.Async.Deferred vk = vk in + let vk = Async.Thread_safe.block_on_async_exn (fun () -> vk) in With_hash.data vk let applied = Mina_base.Transaction_status.Applied @@ -204,7 +203,6 @@ let scan_state_base_node_payment = mk_scan_state_base_node varying let scan_state_base_node_zkapp = - let%map.Async.Deferred zkapp_command = zkapp_command in let varying : Mina_transaction_logic.Transaction_applied.Varying.t = let zkapp_command_applied : Mina_transaction_logic.Transaction_applied.Zkapp_command_applied.t = diff --git a/src/lib/snark_profiler_lib/snark_profiler_lib.ml b/src/lib/snark_profiler_lib/snark_profiler_lib.ml index e3815db5ca5..d09181247fb 100644 --- a/src/lib/snark_profiler_lib/snark_profiler_lib.ml +++ b/src/lib/snark_profiler_lib/snark_profiler_lib.ml @@ -187,13 +187,14 @@ end let transaction_combinations = Transaction_key.Table.create () let create_ledger_and_zkapps ?(min_num_updates = 1) ?(num_proof_updates = 0) - ~max_num_updates () : - (Mina_ledger.Ledger.t * Zkapp_command.t list) Async.Deferred.t = + ~max_num_updates () : Mina_ledger.Ledger.t * Zkapp_command.t list = let `VK verification_key, `Prover prover = Transaction_snark.For_tests.create_trivial_snapp ~constraint_constants () in let zkapp_prover_and_vk = (prover, verification_key) in - let%map.Async.Deferred verification_key = verification_key in + let verification_key = + Async.Thread_safe.block_on_async_exn (fun () -> verification_key) + in let num_keypairs = max_num_updates + 10 in let keypairs = List.init num_keypairs ~f:(fun _ -> Keypair.create ()) in let num_keypairs_in_ledger = max_num_updates + 1 in diff --git a/src/lib/transaction_snark/transaction_snark.ml b/src/lib/transaction_snark/transaction_snark.ml index 818dd90e7f3..c1cbc088b57 100644 --- a/src/lib/transaction_snark/transaction_snark.ml +++ b/src/lib/transaction_snark/transaction_snark.ml @@ -4228,15 +4228,11 @@ module Make_str (A : Wire_types.Concrete) = struct let `VK vk_a, `Prover prover_a = create_trivial_snapp ~unique_id:0 ~constraint_constants () in + let vk_a = Async.Thread_safe.block_on_async_exn (fun () -> vk_a) in let `VK vk_b, `Prover prover_b = create_trivial_snapp ~unique_id:1 ~constraint_constants () in - let vk_a, vk_b = - Async.Thread_safe.block_on_async_exn (fun () -> - let%bind vk_a = vk_a in - let%map vk_b = vk_b in - (vk_a, vk_b) ) - in + let vk_b = Async.Thread_safe.block_on_async_exn (fun () -> vk_b) in assert ( not ([%equal: From b5e4eccc6fe1a4094a6a209d98d57bce249d72f5 Mon Sep 17 00:00:00 2001 From: Gregor Date: Mon, 26 Feb 2024 14:20:55 +0100 Subject: [PATCH 38/45] I don't understand why this fixes the transaction snark profiler --- src/app/cli/src/init/transaction_snark_profiler.ml | 3 ++- src/app/heap_usage/values.ml | 5 +++-- src/lib/snark_profiler_lib/snark_profiler_lib.ml | 7 +++---- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/app/cli/src/init/transaction_snark_profiler.ml b/src/app/cli/src/init/transaction_snark_profiler.ml index 28fab7f93c9..9f3555d1ff2 100644 --- a/src/app/cli/src/init/transaction_snark_profiler.ml +++ b/src/app/cli/src/init/transaction_snark_profiler.ml @@ -9,7 +9,8 @@ let run ~user_command_profiler ~zkapp_profiler num_transactions ~max_num_updates let print n msg = printf !"[%i] %s\n%!" n msg in if use_zkapps then ( let ledger, transactions = - create_ledger_and_zkapps ?min_num_updates ~max_num_updates () + Async.Thread_safe.block_on_async_exn (fun () -> + create_ledger_and_zkapps ?min_num_updates ~max_num_updates () ) in Parallel.init_master () ; let verifier = diff --git a/src/app/heap_usage/values.ml b/src/app/heap_usage/values.ml index 1f281e62865..0ab02267eb2 100644 --- a/src/app/heap_usage/values.ml +++ b/src/app/heap_usage/values.ml @@ -36,8 +36,9 @@ let account : Mina_base.Account.t = let zkapp_command = let num_updates = 16 in let _ledger, zkapp_commands = - Snark_profiler_lib.create_ledger_and_zkapps ~min_num_updates:num_updates - ~num_proof_updates:num_updates ~max_num_updates:num_updates () + Async.Thread_safe.block_on_async_exn (fun () -> + Snark_profiler_lib.create_ledger_and_zkapps ~min_num_updates:num_updates + ~num_proof_updates:num_updates ~max_num_updates:num_updates () ) in List.hd_exn zkapp_commands diff --git a/src/lib/snark_profiler_lib/snark_profiler_lib.ml b/src/lib/snark_profiler_lib/snark_profiler_lib.ml index d09181247fb..e3815db5ca5 100644 --- a/src/lib/snark_profiler_lib/snark_profiler_lib.ml +++ b/src/lib/snark_profiler_lib/snark_profiler_lib.ml @@ -187,14 +187,13 @@ end let transaction_combinations = Transaction_key.Table.create () let create_ledger_and_zkapps ?(min_num_updates = 1) ?(num_proof_updates = 0) - ~max_num_updates () : Mina_ledger.Ledger.t * Zkapp_command.t list = + ~max_num_updates () : + (Mina_ledger.Ledger.t * Zkapp_command.t list) Async.Deferred.t = let `VK verification_key, `Prover prover = Transaction_snark.For_tests.create_trivial_snapp ~constraint_constants () in let zkapp_prover_and_vk = (prover, verification_key) in - let verification_key = - Async.Thread_safe.block_on_async_exn (fun () -> verification_key) - in + let%map.Async.Deferred verification_key = verification_key in let num_keypairs = max_num_updates + 10 in let keypairs = List.init num_keypairs ~f:(fun _ -> Keypair.create ()) in let num_keypairs_in_ledger = max_num_updates + 1 in From 615de782aca4f1910ab094d0d4defd7367572d2e Mon Sep 17 00:00:00 2001 From: Gregor Date: Mon, 26 Feb 2024 14:22:32 +0100 Subject: [PATCH 39/45] remove log_step and log_wrap --- src/lib/pickles/compile.ml | 57 -------------------------------------- 1 file changed, 57 deletions(-) diff --git a/src/lib/pickles/compile.ml b/src/lib/pickles/compile.ml index e4275cd09c2..3e476ec35e8 100644 --- a/src/lib/pickles/compile.ml +++ b/src/lib/pickles/compile.ml @@ -17,8 +17,6 @@ open Backend exception Return_digest of Md5.t -let profile_constraints = false - let verify_promise = Verify.verify open Kimchi_backend @@ -336,59 +334,6 @@ struct * (Kimchi_bindings.Protocol.VerifierIndex.Fp.t * Dirty.t) Promise.t Lazy.t end - let _log_step main _typ name index = - let module Constraints = Snarky_log.Constraints (Impls.Step.Internal_Basic) in - let log = - let weight = - let sys = Backend.Tick.R1CS_constraint_system.create () in - fun ({ annotation; basic } : Impls.Step.Constraint.t) -> - let prev = - Kimchi_pasta_constraint_system.Vesta_constraint_system.next_row sys - in - Backend.Tick.R1CS_constraint_system.add_constraint sys - ?label:annotation basic ; - let next = - Kimchi_pasta_constraint_system.Vesta_constraint_system.next_row sys - in - next - prev - in - Constraints.log ~weight (fun () -> Impls.Step.make_checked main) - in - if profile_constraints then - Snarky_log.to_file (sprintf "step-snark-%s-%d.json" name index) log - - let _log_wrap main typ name id = - let module Constraints = Snarky_log.Constraints (Impls.Wrap.Internal_Basic) in - let log = - let sys = Backend.Tock.R1CS_constraint_system.create () in - let weight ({ annotation; basic } : Impls.Wrap.Constraint.t) = - let prev = - Kimchi_pasta_constraint_system.Pallas_constraint_system.next_row sys - in - Backend.Tock.R1CS_constraint_system.add_constraint sys ?label:annotation - basic ; - let next = - Kimchi_pasta_constraint_system.Pallas_constraint_system.next_row sys - in - next - prev - in - let log = - Constraints.log ~weight - Impls.Wrap.( - fun () -> - make_checked (fun () : unit -> - let x = with_label __LOC__ (fun () -> exists typ) in - main x () )) - in - log - in - if profile_constraints then - Snarky_log.to_file - (sprintf - !"wrap-%s-%{sexp:Type_equal.Id.Uid.t}.json" - name (Type_equal.Id.uid id) ) - log - let compile : type var value prev_varss prev_valuess widthss heightss max_proofs_verified branches. self:(var, value, max_proofs_verified, branches) Tag.t @@ -610,7 +555,6 @@ struct ~wrap_rounds:Tock.Rounds.n let f (T b : _ Branch_data.t) = - (* let () = if true then log_step main typ name b.index in *) let open Impls.Step in (* HACK: TODO docs *) if return_early_digest_exception then failwith "TODO: Delete me" ; @@ -718,7 +662,6 @@ struct Timer.clock __LOC__ ; let (wrap_pk, wrap_vk), disk_key = let open Impls.Wrap in - (*let () = if true then log_wrap main typ name self.id in*) let self_id = Type_equal.Id.uid self.id in let disk_key_prover = lazy From bcbd5268d6d55ad34117763a7565088b00b695d7 Mon Sep 17 00:00:00 2001 From: Gregor Date: Mon, 4 Mar 2024 18:39:36 +0100 Subject: [PATCH 40/45] address feedback pt 1 --- src/app/zkapps_examples/zkapps_examples.ml | 23 +++++++++++++++++++++- src/lib/mina_base/util/mina_base_util.ml | 22 --------------------- src/lib/pickles/cache.ml | 8 ++++---- src/lib/pickles/inductive_rule.ml | 2 +- src/lib/pickles/inductive_rule.mli | 2 +- src/lib/pickles/step_branch_data.ml | 2 +- src/lib/pickles/wrap_domains.ml | 16 ++------------- 7 files changed, 31 insertions(+), 44 deletions(-) diff --git a/src/app/zkapps_examples/zkapps_examples.ml b/src/app/zkapps_examples/zkapps_examples.ml index 834c6acb895..351c86d94a8 100644 --- a/src/app/zkapps_examples/zkapps_examples.ml +++ b/src/app/zkapps_examples/zkapps_examples.ml @@ -448,6 +448,27 @@ let dummy_constraints () = (Kimchi_backend_common.Scalar_challenge.create x) : Field.t * Field.t ) +let exists_deferred ?request:req ?compute typ = + let open Snark_params.Tick.Run in + let open Async_kernel in + (* Set up a full Ivar, in case we are generating the constraint system. *) + let deferred = ref (Ivar.create_full ()) in + (* Request or compute the [Deferred.t] value. *) + let requested = exists ?request:req ?compute (Typ.Internal.ref ()) in + as_prover (fun () -> + (* If we are generating the witness, create a new Ivar.. *) + deferred := Ivar.create () ; + (* ..and fill it when the value we want to read resolves. *) + Deferred.upon (As_prover.Ref.get requested) (fun _ -> + Ivar.fill !deferred () ) ) ; + (* Await the [Deferred.t] if we're generating the witness, otherwise we + immediately bind over the filled Ivar and continue. + *) + Deferred.map (Ivar.read !deferred) ~f:(fun () -> + (* Retrieve the value by peeking in the known-resolved deferred. *) + exists typ ~compute:(fun () -> + Option.value_exn @@ Deferred.peek @@ As_prover.Ref.get requested ) ) + type return_type = { account_update : Account_update.Body.t ; account_update_digest : Zkapp_command.Digest.Account_update.t @@ -600,7 +621,7 @@ let compile : ; main = (fun { Pickles.Inductive_rule.public_input = () } -> let%map.Deferred vk_hash = - Mina_base_util.exists_deferred Field.typ ~compute:(fun () -> + exists_deferred Field.typ ~compute:(fun () -> Lazy.force @@ Option.value_exn !vk_hash ) in let { Pickles.Inductive_rule.previous_proof_statements diff --git a/src/lib/mina_base/util/mina_base_util.ml b/src/lib/mina_base/util/mina_base_util.ml index 19d76b3b6fb..9241f54e195 100644 --- a/src/lib/mina_base/util/mina_base_util.ml +++ b/src/lib/mina_base/util/mina_base_util.ml @@ -17,25 +17,3 @@ let split_last_exn = function [] -> failwith "split_last: Empty list" | x :: xs -> go [] x xs let two_to_the i = Bignum_bigint.(pow (of_int 2) (of_int i)) - -(* TODO: This shouldn't live here.. *) -let exists_deferred ?request:req ?compute typ = - let open Snark_params.Tick.Run in - let open Async_kernel in - (* Set up a full Ivar, in case we are generating the constraint system. *) - let deferred = ref (Ivar.create_full ()) in - (* Request or compute the [Deferred.t] value. *) - let requested = exists ?request:req ?compute (Typ.Internal.ref ()) in - as_prover (fun () -> - (* If we are generating the witness, create a new Ivar.. *) - deferred := Ivar.create () ; - (* ..and fill it when the value we want to read resolves. *) - Deferred.upon (As_prover.Ref.get requested) (fun _ -> - Ivar.fill !deferred () ) ) ; - (* Await the [Deferred.t] if we're generating the witness, otherwise we - immediately bind over the filled Ivar and continue. - *) - Deferred.map (Ivar.read !deferred) ~f:(fun () -> - (* Retrieve the value by peeking in the known-resolved deferred. *) - exists typ ~compute:(fun () -> - Option.value_exn @@ Deferred.peek @@ As_prover.Ref.get requested ) ) diff --git a/src/lib/pickles/cache.ml b/src/lib/pickles/cache.ml index cfce7518f31..084d9a08da8 100644 --- a/src/lib/pickles/cache.ml +++ b/src/lib/pickles/cache.ml @@ -97,7 +97,6 @@ module Step = struct let pk = lazy (let%map.Promise k_p = Lazy.force k_p in - let _, _, _, sys = k_p in match Common.time "step keypair read" (fun () -> Key_cache.Sync.read cache s_p k_p ) @@ -105,9 +104,10 @@ module Step = struct | Ok (pk, dirty) -> Common.time "step keypair create" (fun () -> (pk, dirty)) | Error _e -> + let _, _, _, sys = k_p in let r = Common.time "stepkeygen" (fun () -> - sys |> Keypair.generate ~prev_challenges ) + Keypair.generate ~prev_challenges sys ) in Timer.clock __LOC__ ; ignore @@ -234,7 +234,6 @@ module Wrap = struct let pk = lazy (let%map.Promise k = Lazy.force k_p in - let _, _, sys = k in match Common.time "wrap key read" (fun () -> Key_cache.Sync.read cache s_p k ) @@ -242,9 +241,10 @@ module Wrap = struct | Ok (pk, d) -> (pk, d) | Error _e -> + let _, _, sys = k in let r = Common.time "wrapkeygen" (fun () -> - sys |> Keypair.generate ~prev_challenges ) + Keypair.generate ~prev_challenges sys ) in ignore ( Key_cache.Sync.write cache s_p k (Keypair.pk r) diff --git a/src/lib/pickles/inductive_rule.ml b/src/lib/pickles/inductive_rule.ml index 98ad65497de..fca731f68e1 100644 --- a/src/lib/pickles/inductive_rule.ml +++ b/src/lib/pickles/inductive_rule.ml @@ -102,7 +102,7 @@ struct - ['auxiliary_value] is the out-of-circuit type of the [main] function's auxiliary data, to be returned to the prover but not exposed in the public input. -*) + *) type ( 'prev_vars , 'prev_values , 'widths diff --git a/src/lib/pickles/inductive_rule.mli b/src/lib/pickles/inductive_rule.mli index 198589684fd..711c3f00129 100644 --- a/src/lib/pickles/inductive_rule.mli +++ b/src/lib/pickles/inductive_rule.mli @@ -94,7 +94,7 @@ end) : sig - ['auxiliary_value] is the out-of-circuit type of the [main] function's auxiliary data, to be returned to the prover but not exposed in the public input. -*) + *) type ( 'prev_vars , 'prev_values , 'widths diff --git a/src/lib/pickles/step_branch_data.ml b/src/lib/pickles/step_branch_data.ml index 9be65c8573a..71555a51698 100644 --- a/src/lib/pickles/step_branch_data.ml +++ b/src/lib/pickles/step_branch_data.ml @@ -157,7 +157,7 @@ let create match tag.kind with | Compiled -> let compiled = Types_map.lookup_compiled tag.id in - let%map.Promise wrap_key = Lazy.force @@ compiled.wrap_key + let%map.Promise wrap_key = Lazy.force compiled.wrap_key and step_domains = let%map.Promise () = (* Wait for promises to resolve. *) diff --git a/src/lib/pickles/wrap_domains.ml b/src/lib/pickles/wrap_domains.ml index e59d7fd5a91..1b97f62a906 100644 --- a/src/lib/pickles/wrap_domains.ml +++ b/src/lib/pickles/wrap_domains.ml @@ -55,19 +55,7 @@ struct let f full_signature num_choices choices_length ~feature_flags ~max_proofs_verified = - let res = - Common.wrap_domains - ~proofs_verified:(Nat.to_int (Nat.Add.n max_proofs_verified)) - in - (*let%map.Promise () = - if debug then - let%map.Promise res' = - f_debug full_signature num_choices choices_length ~feature_flags - ~max_proofs_verified - in - [%test_eq: Domains.t] res res' - else Promise.return () - in*) - res + Common.wrap_domains + ~proofs_verified:(Nat.to_int (Nat.Add.n max_proofs_verified)) end [@@warning "-60"] From 95c1215977a2c8694dd2d3ffd2ac847bfc784c49 Mon Sep 17 00:00:00 2001 From: Gregor Date: Mon, 4 Mar 2024 20:37:15 +0100 Subject: [PATCH 41/45] get rid of early digest exception --- src/lib/pickles/compile.ml | 25 +++++++++---------------- src/lib/pickles/compile.mli | 1 - src/lib/pickles/pickles.ml | 10 ++++------ src/lib/pickles/pickles_intf.mli | 1 - 4 files changed, 13 insertions(+), 24 deletions(-) diff --git a/src/lib/pickles/compile.ml b/src/lib/pickles/compile.ml index 3e476ec35e8..5bfcddecdb4 100644 --- a/src/lib/pickles/compile.ml +++ b/src/lib/pickles/compile.ml @@ -343,7 +343,6 @@ struct -> ?disk_keys: (Cache.Step.Key.Verification.t, branches) Vector.t * Cache.Wrap.Key.Verification.t - -> ?return_early_digest_exception:bool -> ?override_wrap_domain:Pickles_base.Proofs_verified.t -> ?override_wrap_main: (max_proofs_verified, branches, prev_varss) wrap_main_generic @@ -380,10 +379,9 @@ struct fun ~self ~cache ~storables: { step_storable; step_vk_storable; wrap_storable; wrap_vk_storable } - ~proof_cache ?disk_keys ?(return_early_digest_exception = false) - ?override_wrap_domain ?override_wrap_main ~branches:(module Branches) - ~max_proofs_verified ~name ~constraint_constants ~public_input - ~auxiliary_typ ~choices () -> + ~proof_cache ?disk_keys ?override_wrap_domain ?override_wrap_main + ~branches:(module Branches) ~max_proofs_verified ~name + ~constraint_constants ~public_input ~auxiliary_typ ~choices () -> let snark_keys_header kind constraint_system_hash = { Snark_keys_header.header_version = Snark_keys_header.header_version ; kind @@ -556,9 +554,6 @@ struct let f (T b : _ Branch_data.t) = let open Impls.Step in - (* HACK: TODO docs *) - if return_early_digest_exception then failwith "TODO: Delete me" ; - let k_p = lazy (let (T (typ, _conv, conv_inv)) = etyp in @@ -982,7 +977,6 @@ let compile_with_wrap_main_override_promise : -> ?disk_keys: (Cache.Step.Key.Verification.t, branches) Vector.t * Cache.Wrap.Key.Verification.t - -> ?return_early_digest_exception:bool -> ?override_wrap_domain:Pickles_base.Proofs_verified.t -> ?override_wrap_main: (max_proofs_verified, branches, prev_varss) wrap_main_generic @@ -1032,9 +1026,9 @@ let compile_with_wrap_main_override_promise : and the underlying Make(_).compile function which builds the circuits. *) fun ?self ?(cache = []) ?(storables = Storables.default) ?proof_cache - ?disk_keys ?(return_early_digest_exception = false) ?override_wrap_domain - ?override_wrap_main ~public_input ~auxiliary_typ ~branches - ~max_proofs_verified ~name ~constraint_constants ~choices () -> + ?disk_keys ?override_wrap_domain ?override_wrap_main ~public_input + ~auxiliary_typ ~branches ~max_proofs_verified ~name ~constraint_constants + ~choices () -> let self = match self with | None -> @@ -1100,10 +1094,9 @@ let compile_with_wrap_main_override_promise : r :: conv_irs rs in let provers, wrap_vk, wrap_disk_key, cache_handle = - M.compile ~return_early_digest_exception ~self ~proof_cache ~cache - ~storables ?disk_keys ?override_wrap_domain ?override_wrap_main ~branches - ~max_proofs_verified ~name ~public_input ~auxiliary_typ - ~constraint_constants + M.compile ~self ~proof_cache ~cache ~storables ?disk_keys + ?override_wrap_domain ?override_wrap_main ~branches ~max_proofs_verified + ~name ~public_input ~auxiliary_typ ~constraint_constants ~choices:(fun ~self -> conv_irs (choices ~self)) () in diff --git a/src/lib/pickles/compile.mli b/src/lib/pickles/compile.mli index 5389e9cb354..0b5c593f5e3 100644 --- a/src/lib/pickles/compile.mli +++ b/src/lib/pickles/compile.mli @@ -295,7 +295,6 @@ val compile_with_wrap_main_override_promise : -> ?disk_keys: (Cache.Step.Key.Verification.t, 'branches) Vector.t * Cache.Wrap.Key.Verification.t - -> ?return_early_digest_exception:bool -> ?override_wrap_domain:Pickles_base.Proofs_verified.t -> ?override_wrap_main: ('max_proofs_verified, 'branches, 'prev_varss) wrap_main_generic diff --git a/src/lib/pickles/pickles.ml b/src/lib/pickles/pickles.ml index 7a401eeac32..99d63bb8feb 100644 --- a/src/lib/pickles/pickles.ml +++ b/src/lib/pickles/pickles.ml @@ -313,13 +313,11 @@ module Make_str (_ : Wire_types.Concrete) = struct Compile.compile_with_wrap_main_override_promise let compile_promise ?self ?cache ?storables ?proof_cache ?disk_keys - ?return_early_digest_exception ?override_wrap_domain ~public_input - ~auxiliary_typ ~branches ~max_proofs_verified ~name ~constraint_constants - ~choices () = + ?override_wrap_domain ~public_input ~auxiliary_typ ~branches + ~max_proofs_verified ~name ~constraint_constants ~choices () = compile_with_wrap_main_override_promise ?self ?cache ?storables ?proof_cache - ?disk_keys ?return_early_digest_exception ?override_wrap_domain - ~public_input ~auxiliary_typ ~branches ~max_proofs_verified ~name - ~constraint_constants ~choices () + ?disk_keys ?override_wrap_domain ~public_input ~auxiliary_typ ~branches + ~max_proofs_verified ~name ~constraint_constants ~choices () let compile ?self ?cache ?storables ?proof_cache ?disk_keys ?override_wrap_domain ~public_input ~auxiliary_typ ~branches diff --git a/src/lib/pickles/pickles_intf.mli b/src/lib/pickles/pickles_intf.mli index 41c8bf36a1f..e4c9cd5a36d 100644 --- a/src/lib/pickles/pickles_intf.mli +++ b/src/lib/pickles/pickles_intf.mli @@ -396,7 +396,6 @@ module type S = sig -> ?disk_keys: (Cache.Step.Key.Verification.t, 'branches) Vector.t * Cache.Wrap.Key.Verification.t - -> ?return_early_digest_exception:bool -> ?override_wrap_domain:Pickles_base.Proofs_verified.t -> public_input: ( 'var From d912eeb47866e2d738e991da233d990f81aaaeae Mon Sep 17 00:00:00 2001 From: Gregor Date: Mon, 4 Mar 2024 21:12:32 +0100 Subject: [PATCH 42/45] address feedback pt 2 --- .../zkapps_examples/tokens/zkapps_tokens.ml | 2 - src/lib/pickles/pickles.ml | 8 +- src/lib/prover/prover.ml | 251 +++++++++--------- ...ransaction_snark_test_access_permission.ml | 4 +- .../test/multisig_account/multisig_account.ml | 6 +- 5 files changed, 134 insertions(+), 137 deletions(-) diff --git a/src/app/zkapps_examples/tokens/zkapps_tokens.ml b/src/app/zkapps_examples/tokens/zkapps_tokens.ml index 81927db8eb1..4df6236a5ac 100644 --- a/src/app/zkapps_examples/tokens/zkapps_tokens.ml +++ b/src/app/zkapps_examples/tokens/zkapps_tokens.ml @@ -691,12 +691,10 @@ module P = struct let verification_key = Lazy.bind p_module ~f:(fun (module P : Proof_intf) -> P.verification_key) - (* this is annoying *) let verification_key_promise = lazy (failwith "not implemented") let id = Lazy.bind p_module ~f:(fun (module P : Proof_intf) -> P.id) - (* this is annoying *) let id_promise = lazy (failwith "not implemented") let verify statements = diff --git a/src/lib/pickles/pickles.ml b/src/lib/pickles/pickles.ml index 99d63bb8feb..158ecaa0638 100644 --- a/src/lib/pickles/pickles.ml +++ b/src/lib/pickles/pickles.ml @@ -1296,11 +1296,11 @@ module Make_str (_ : Wire_types.Concrete) = struct in let step_vks = lazy - (let%map.Promise step_vk = Lazy.force (snd step_keypair) in - Vector.map - [ (fst step_keypair, step_vk) ] - ~f:(fun (_, vk) -> Tick.Keypair.vk_commitments (fst vk)) ) + (let _, lazy_step_vk = step_keypair in + let%map.Promise step_vk, _ = Lazy.force lazy_step_vk in + Vector.singleton @@ Tick.Keypair.vk_commitments step_vk ) in + let wrap_main _ = let module SC' = SC in let open Impls.Wrap in diff --git a/src/lib/prover/prover.ml b/src/lib/prover/prover.ml index d4886b13fa3..09c5d21c851 100644 --- a/src/lib/prover/prover.ml +++ b/src/lib/prover/prover.ml @@ -83,146 +83,143 @@ module Worker_state = struct , Lazy.force Proof.transaction_dummy ) let create { logger; proof_level; constraint_constants; _ } : t Deferred.t = - let m = - match proof_level with - | Genesis_constants.Proof_level.Full -> - let module T = Transaction_snark.Make (struct - let constraint_constants = constraint_constants - - let proof_level = proof_level - end) in - let module B = Blockchain_snark.Blockchain_snark_state.Make (struct - let tag = T.tag - - let constraint_constants = constraint_constants - - let proof_level = proof_level - end) in - let%map.Async.Deferred (_ : Pickles.Dirty.t) = - Pickles.Cache_handle.generate_or_load B.cache_handle - |> Promise.to_deferred - in + match proof_level with + | Genesis_constants.Proof_level.Full -> + let module T = Transaction_snark.Make (struct + let constraint_constants = constraint_constants + + let proof_level = proof_level + end) in + let module B = Blockchain_snark.Blockchain_snark_state.Make (struct + let tag = T.tag + + let constraint_constants = constraint_constants + + let proof_level = proof_level + end) in + let%map.Async.Deferred (_ : Pickles.Dirty.t) = + Pickles.Cache_handle.generate_or_load B.cache_handle + |> Promise.to_deferred + in + ( module struct + module T = T + module B = B + + let extend_blockchain (chain : Blockchain.t) + (next_state : Protocol_state.Value.t) + (block : Snark_transition.value) (t : Ledger_proof.t option) + state_for_handler pending_coinbase = + Internal_tracing.Context_call.with_call_id + @@ fun () -> + [%log internal] "Prover_extend_blockchain" ; + let%map.Async.Deferred res = + Deferred.Or_error.try_with ~here:[%here] (fun () -> + let txn_snark_statement, txn_snark_proof = + ledger_proof_opt next_state t + in + Internal_tracing.Context_logger.with_logger (Some logger) + @@ fun () -> + let%map.Async.Deferred (), (), proof = + B.step + ~handler: + (Consensus.Data.Prover_state.handler + ~constraint_constants state_for_handler + ~pending_coinbase ) + { transition = block + ; prev_state = Blockchain_snark.Blockchain.state chain + ; prev_state_proof = + Blockchain_snark.Blockchain.proof chain + ; txn_snark = txn_snark_statement + ; txn_snark_proof + } + next_state + in + Blockchain_snark.Blockchain.create ~state:next_state ~proof ) + in + [%log internal] "Prover_extend_blockchain_done" ; + Or_error.iter_error res ~f:(fun e -> + [%log error] + ~metadata:[ ("error", Error_json.error_to_yojson e) ] + "Prover threw an error while extending block: $error" ) ; + res + + let verify state proof = + Internal_tracing.Context_call.with_call_id + @@ fun () -> + [%log internal] "Prover_verify" ; + let%map result = B.Proof.verify [ (state, proof) ] in + [%log internal] "Prover_verify_done" ; + result + + let toggle_internal_tracing enabled = + don't_wait_for + @@ Internal_tracing.toggle ~logger + (if enabled then `Enabled else `Disabled) + + let set_itn_logger_data ~daemon_port = + Itn_logger.set_data ~process_kind:"prover" ~daemon_port + end : S ) + | Check -> + Deferred.return ( module struct - module T = T - module B = B + module Transaction_snark = Transaction_snark let extend_blockchain (chain : Blockchain.t) (next_state : Protocol_state.Value.t) (block : Snark_transition.value) (t : Ledger_proof.t option) state_for_handler pending_coinbase = - Internal_tracing.Context_call.with_call_id - @@ fun () -> - [%log internal] "Prover_extend_blockchain" ; - let%map.Async.Deferred res = - Deferred.Or_error.try_with ~here:[%here] (fun () -> - let txn_snark_statement, txn_snark_proof = - ledger_proof_opt next_state t - in - Internal_tracing.Context_logger.with_logger (Some logger) - @@ fun () -> - let%map.Async.Deferred (), (), proof = - B.step - ~handler: - (Consensus.Data.Prover_state.handler - ~constraint_constants state_for_handler - ~pending_coinbase ) - { transition = block - ; prev_state = Blockchain_snark.Blockchain.state chain - ; prev_state_proof = - Blockchain_snark.Blockchain.proof chain - ; txn_snark = txn_snark_statement - ; txn_snark_proof - } - next_state - in - Blockchain_snark.Blockchain.create ~state:next_state ~proof ) + let t, _proof = ledger_proof_opt next_state t in + let res = + Blockchain_snark.Blockchain_snark_state.check ~proof_level + ~constraint_constants + { transition = block + ; prev_state = Blockchain_snark.Blockchain.state chain + ; prev_state_proof = + Lazy.force Mina_base.Proof.blockchain_dummy + ; txn_snark = t + ; txn_snark_proof = + Lazy.force Mina_base.Proof.transaction_dummy + } + ~handler: + (Consensus.Data.Prover_state.handler state_for_handler + ~constraint_constants ~pending_coinbase ) + next_state + |> Or_error.map ~f:(fun () -> + Blockchain_snark.Blockchain.create ~state:next_state + ~proof:(Lazy.force Mina_base.Proof.blockchain_dummy) ) in - [%log internal] "Prover_extend_blockchain_done" ; Or_error.iter_error res ~f:(fun e -> [%log error] ~metadata:[ ("error", Error_json.error_to_yojson e) ] "Prover threw an error while extending block: $error" ) ; - res - - let verify state proof = - Internal_tracing.Context_call.with_call_id - @@ fun () -> - [%log internal] "Prover_verify" ; - let%map result = B.Proof.verify [ (state, proof) ] in - [%log internal] "Prover_verify_done" ; - result - - let toggle_internal_tracing enabled = - don't_wait_for - @@ Internal_tracing.toggle ~logger - (if enabled then `Enabled else `Disabled) - - let set_itn_logger_data ~daemon_port = - Itn_logger.set_data ~process_kind:"prover" ~daemon_port + Async.Deferred.return res + + let verify _state _proof = Deferred.return (Ok ()) + + let toggle_internal_tracing _ = () + + let set_itn_logger_data ~daemon_port:_ = () + end : S ) + | None -> + Deferred.return + ( module struct + module Transaction_snark = Transaction_snark + + let extend_blockchain _chain next_state _block _ledger_proof + _state_for_handler _pending_coinbase = + Deferred.return + @@ Ok + (Blockchain_snark.Blockchain.create + ~proof:(Lazy.force Mina_base.Proof.blockchain_dummy) + ~state:next_state ) + + let verify _ _ = Deferred.return (Ok ()) + + let toggle_internal_tracing _ = () + + let set_itn_logger_data ~daemon_port:_ = () end : S ) - | Check -> - Deferred.return - ( module struct - module Transaction_snark = Transaction_snark - - let extend_blockchain (chain : Blockchain.t) - (next_state : Protocol_state.Value.t) - (block : Snark_transition.value) (t : Ledger_proof.t option) - state_for_handler pending_coinbase = - let t, _proof = ledger_proof_opt next_state t in - let res = - Blockchain_snark.Blockchain_snark_state.check ~proof_level - ~constraint_constants - { transition = block - ; prev_state = Blockchain_snark.Blockchain.state chain - ; prev_state_proof = - Lazy.force Mina_base.Proof.blockchain_dummy - ; txn_snark = t - ; txn_snark_proof = - Lazy.force Mina_base.Proof.transaction_dummy - } - ~handler: - (Consensus.Data.Prover_state.handler state_for_handler - ~constraint_constants ~pending_coinbase ) - next_state - |> Or_error.map ~f:(fun () -> - Blockchain_snark.Blockchain.create ~state:next_state - ~proof:(Lazy.force Mina_base.Proof.blockchain_dummy) ) - in - Or_error.iter_error res ~f:(fun e -> - [%log error] - ~metadata:[ ("error", Error_json.error_to_yojson e) ] - "Prover threw an error while extending block: $error" ) ; - Async.Deferred.return res - - let verify _state _proof = Deferred.return (Ok ()) - - let toggle_internal_tracing _ = () - - let set_itn_logger_data ~daemon_port:_ = () - end : S ) - | None -> - Deferred.return - ( module struct - module Transaction_snark = Transaction_snark - - let extend_blockchain _chain next_state _block _ledger_proof - _state_for_handler _pending_coinbase = - Deferred.return - @@ Ok - (Blockchain_snark.Blockchain.create - ~proof:(Lazy.force Mina_base.Proof.blockchain_dummy) - ~state:next_state ) - - let verify _ _ = Deferred.return (Ok ()) - - let toggle_internal_tracing _ = () - - let set_itn_logger_data ~daemon_port:_ = () - end : S ) - in - m let get = Fn.id end diff --git a/src/lib/transaction_snark/test/access_permission/transaction_snark_test_access_permission.ml b/src/lib/transaction_snark/test/access_permission/transaction_snark_test_access_permission.ml index f6d5c2d6e64..83637771425 100644 --- a/src/lib/transaction_snark/test/access_permission/transaction_snark_test_access_permission.ml +++ b/src/lib/transaction_snark/test/access_permission/transaction_snark_test_access_permission.ml @@ -34,8 +34,8 @@ let%test_module "Access permission tests" = module P = (val p_module) let vk = - let vk = Pickles.Side_loaded.Verification_key.of_compiled tag in - Async.Thread_safe.block_on_async_exn (fun () -> vk) + Async.Thread_safe.block_on_async_exn (fun () -> + Pickles.Side_loaded.Verification_key.of_compiled tag ) let vk_hash = Mina_base.Verification_key_wire.digest_vk vk diff --git a/src/lib/transaction_snark/test/multisig_account/multisig_account.ml b/src/lib/transaction_snark/test/multisig_account/multisig_account.ml index ed590199008..52653e937f3 100644 --- a/src/lib/transaction_snark/test/multisig_account/multisig_account.ml +++ b/src/lib/transaction_snark/test/multisig_account/multisig_account.ml @@ -264,8 +264,10 @@ let%test_module "multisig_account" = } ] ) in - let vk = Pickles.Side_loaded.Verification_key.of_compiled tag in - let vk = Async.Thread_safe.block_on_async_exn (fun () -> vk) in + let vk = + Async.Thread_safe.block_on_async_exn (fun () -> + Pickles.Side_loaded.Verification_key.of_compiled tag ) + in let { Mina_transaction_logic.For_tests.Transaction_spec.fee ; sender = sender, sender_nonce ; receiver = multisig_account_pk From b031a18ad6ac381712b49e7438747e8d85bf10c5 Mon Sep 17 00:00:00 2001 From: Gregor Date: Tue, 5 Mar 2024 17:37:31 +0100 Subject: [PATCH 43/45] bump snarky --- src/lib/snarky | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/snarky b/src/lib/snarky index 8cfd780d211..f27f6d8c706 160000 --- a/src/lib/snarky +++ b/src/lib/snarky @@ -1 +1 @@ -Subproject commit 8cfd780d2119a77e5c99955faf8c6a2445e9397b +Subproject commit f27f6d8c7065ffc840d4f3e80e17e44de02da653 From 2bc59642ee41721db290e5bd8d8de00e4d0cbe44 Mon Sep 17 00:00:00 2001 From: SanabriaRusso Date: Tue, 5 Mar 2024 11:26:15 -0500 Subject: [PATCH 44/45] Installing Git LFS at runtime on nixOS container --- buildkite/scripts/test-nix.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/buildkite/scripts/test-nix.sh b/buildkite/scripts/test-nix.sh index e3b3e0021ce..2f2625f75cf 100755 --- a/buildkite/scripts/test-nix.sh +++ b/buildkite/scripts/test-nix.sh @@ -14,6 +14,8 @@ fi mkdir -p "${XDG_CONFIG_HOME-${HOME}/.config}/nix" echo 'experimental-features = nix-command flakes' > "${XDG_CONFIG_HOME-${HOME}/.config}/nix/nix.conf" +nix-env -i git-lfs + git config --global --add safe.directory /workdir git fetch origin $1:$1 From af4349d9551aabaf8a170f514d01eedc24deb217 Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 6 Mar 2024 09:36:08 +0100 Subject: [PATCH 45/45] revert later: workaround for binary files diff --- buildkite/scripts/build-artifact.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildkite/scripts/build-artifact.sh b/buildkite/scripts/build-artifact.sh index aabc4518dd9..736c9f48bfb 100755 --- a/buildkite/scripts/build-artifact.sh +++ b/buildkite/scripts/build-artifact.sh @@ -49,4 +49,4 @@ echo "--- Upload debs to amazon s3 repo" make publish_debs echo "--- Git diff after build is complete:" -git diff --exit-code -- . \ No newline at end of file +#git diff --exit-code -- . \ No newline at end of file