From 7ee87fa35bedc68234370a1fce103e427cc5b439 Mon Sep 17 00:00:00 2001 From: Yang Weitao Date: Tue, 12 Dec 2023 19:46:15 +0800 Subject: [PATCH] update plonky2 source analysis, circuit building --- db.json | 2 +- .../zkp/plonky2-code-analysis/index.html | 39 ++-- public/page/2/index.html | 33 ++-- .../cryptography/zkp/plonky2-code-analysis.md | 174 ++++++++++++++++-- 4 files changed, 191 insertions(+), 57 deletions(-) diff --git a/db.json b/db.json index aa239f21..ca5caff3 100644 --- a/db.json +++ b/db.json @@ -1 +1 @@ -{"meta":{"version":1,"warehouse":"4.0.2"},"models":{"Asset":[{"_id":"node_modules/hexo-theme-landscape/source/css/style.styl","path":"css/style.styl","modified":0,"renderable":1},{"_id":"node_modules/hexo-theme-landscape/source/js/jquery-3.4.1.min.js","path":"js/jquery-3.4.1.min.js","modified":0,"renderable":1},{"_id":"node_modules/hexo-theme-landscape/source/js/script.js","path":"js/script.js","modified":0,"renderable":1},{"_id":"node_modules/hexo-theme-landscape/source/fancybox/jquery.fancybox.min.css","path":"fancybox/jquery.fancybox.min.css","modified":0,"renderable":1},{"_id":"node_modules/hexo-theme-landscape/source/fancybox/jquery.fancybox.min.js","path":"fancybox/jquery.fancybox.min.js","modified":0,"renderable":1},{"_id":"node_modules/hexo-theme-landscape/source/css/fonts/FontAwesome.otf","path":"css/fonts/FontAwesome.otf","modified":0,"renderable":1},{"_id":"node_modules/hexo-theme-landscape/source/css/fonts/fontawesome-webfont.eot","path":"css/fonts/fontawesome-webfont.eot","modified":0,"renderable":1},{"_id":"node_modules/hexo-theme-landscape/source/css/fonts/fontawesome-webfont.ttf","path":"css/fonts/fontawesome-webfont.ttf","modified":0,"renderable":1},{"_id":"node_modules/hexo-theme-landscape/source/css/fonts/fontawesome-webfont.svg","path":"css/fonts/fontawesome-webfont.svg","modified":0,"renderable":1},{"_id":"node_modules/hexo-theme-landscape/source/css/fonts/fontawesome-webfont.woff","path":"css/fonts/fontawesome-webfont.woff","modified":0,"renderable":1},{"_id":"node_modules/hexo-theme-landscape/source/css/images/banner.jpg","path":"css/images/banner.jpg","modified":0,"renderable":1},{"_id":"node_modules/hexo-theme-landscape/source/css/fonts/fontawesome-webfont.woff2","path":"css/fonts/fontawesome-webfont.woff2","modified":0,"renderable":1},{"_id":"source/2017-552.pdf","path":"2017-552.pdf","modified":0,"renderable":0},{"_id":"source/images/evm.drawio.google.png","path":"images/evm.drawio.google.png","modified":0,"renderable":0},{"_id":"source/images/evm.layout.png","path":"images/evm.layout.png","modified":0,"renderable":0},{"_id":"source/images/geth_starts.drawio.png","path":"images/geth_starts.drawio.png","modified":0,"renderable":0},{"_id":"source/images/kademlia.locating.png","path":"images/kademlia.locating.png","modified":0,"renderable":0},{"_id":"source/images/kademlia.onlineprob.png","path":"images/kademlia.onlineprob.png","modified":0,"renderable":0},{"_id":"source/images/kademlia.subtree.png","path":"images/kademlia.subtree.png","modified":0,"renderable":0},{"_id":"source/images/mpt.state.ref.png","path":"images/mpt.state.ref.png","modified":0,"renderable":0},{"_id":"source/images/mpt.png","path":"images/mpt.png","modified":0,"renderable":0},{"_id":"source/images/trie.prefix.png","path":"images/trie.prefix.png","modified":0,"renderable":0},{"_id":"source/images/geth/sync.mode.jpg","path":"images/geth/sync.mode.jpg","modified":0,"renderable":0},{"_id":"source/images/two_party_ecdsa/paillier_enc.png","path":"images/two_party_ecdsa/paillier_enc.png","modified":0,"renderable":0},{"_id":"source/images/two_party_ecdsa/schnorr_ecdsa_comparison.png","path":"images/two_party_ecdsa/schnorr_ecdsa_comparison.png","modified":0,"renderable":0},{"_id":"source/images/paillier/carmichael_thorem.png","path":"images/paillier/carmichael_thorem.png","modified":0,"renderable":0},{"_id":"source/images/paillier/carmichael_thorem_2.png","path":"images/paillier/carmichael_thorem_2.png","modified":0,"renderable":0},{"_id":"source/images/paillier/homomorphic_addition.png","path":"images/paillier/homomorphic_addition.png","modified":0,"renderable":0},{"_id":"source/images/paillier/homomorphic_mul.png","path":"images/paillier/homomorphic_mul.png","modified":0,"renderable":0},{"_id":"source/images/cryptography/elliptic_curve/paring_ec.png","path":"images/cryptography/elliptic_curve/paring_ec.png","modified":0,"renderable":0},{"_id":"source/images/cryptography/elliptic_curve/point_addition.webp","path":"images/cryptography/elliptic_curve/point_addition.webp","modified":0,"renderable":0},{"_id":"source/images/cryptography/zkp/checking_qap.png","path":"images/cryptography/zkp/checking_qap.png","modified":0,"renderable":0},{"_id":"source/images/cryptography/zkp/lagrange_interpolating.png","path":"images/cryptography/zkp/lagrange_interpolating.png","modified":0,"renderable":0},{"_id":"source/images/cryptography/zkp/r1cs.png","path":"images/cryptography/zkp/r1cs.png","modified":0,"renderable":0},{"_id":"source/images/rust/macros/16.compile_process.png","path":"images/rust/macros/16.compile_process.png","modified":0,"renderable":0},{"_id":"source/images/rust/memory/trait_object_memory.png","path":"images/rust/memory/trait_object_memory.png","modified":0,"renderable":0},{"_id":"source/images/rust/ownership/move-string-2.png","path":"images/rust/ownership/move-string-2.png","modified":0,"renderable":0},{"_id":"source/images/rust/ownership/move-string.png","path":"images/rust/ownership/move-string.png","modified":0,"renderable":0},{"_id":"source/images/rust/pointers/cycle.ref.png","path":"images/rust/pointers/cycle.ref.png","modified":0,"renderable":0},{"_id":"source/images/math/fourier_series/f_x.png","path":"images/math/fourier_series/f_x.png","modified":0,"renderable":0},{"_id":"source/images/rust/pointers/rc.png","path":"images/rust/pointers/rc.png","modified":0,"renderable":0},{"_id":"source/images/cryptography/rsa/rsa_signature.png","path":"images/cryptography/rsa/rsa_signature.png","modified":0,"renderable":0},{"_id":"source/images/zkp/stark/commitment.png","path":"images/zkp/stark/commitment.png","modified":0,"renderable":0},{"_id":"source/images/zkp/stark/decommitment.png","path":"images/zkp/stark/decommitment.png","modified":0,"renderable":0},{"_id":"source/images/zkp/stark/fri.png","path":"images/zkp/stark/fri.png","modified":0,"renderable":0},{"_id":"source/images/zkp/stark/fri_example.png","path":"images/zkp/stark/fri_example.png","modified":0,"renderable":0},{"_id":"source/images/zkp/stark/trace_to_CP.png","path":"images/zkp/stark/trace_to_CP.png","modified":0,"renderable":0},{"_id":"source/images/zkp/plonk/plonk_circuit_to_trace.png","path":"images/zkp/plonk/plonk_circuit_to_trace.png","modified":0,"renderable":0},{"_id":"source/images/zkp/plonk/zero_test.png","path":"images/zkp/plonk/zero_test.png","modified":0,"renderable":0},{"_id":"source/images/cuda/cuda_stream_example.png","path":"images/cuda/cuda_stream_example.png","modified":0,"renderable":0},{"_id":"source/images/arithmatic/multiple_precision_addition.png","path":"images/arithmatic/multiple_precision_addition.png","modified":0,"renderable":0},{"_id":"source/images/arithmatic/radix_b_repesentation.png","path":"images/arithmatic/radix_b_repesentation.png","modified":0,"renderable":0},{"_id":"source/images/zkp/plonk/gate_evaluation_zero_test.png","path":"images/zkp/plonk/gate_evaluation_zero_test.png","modified":0,"renderable":0},{"_id":"source/images/zkp/plonk/permutation_check.png","path":"images/zkp/plonk/permutation_check.png","modified":0,"renderable":0},{"_id":"source/images/zkp/plonk/prod_check_lemma.png","path":"images/zkp/plonk/prod_check_lemma.png","modified":0,"renderable":0},{"_id":"source/images/zkp/plonk/prod_check_prove_verify.png","path":"images/zkp/plonk/prod_check_prove_verify.png","modified":0,"renderable":0},{"_id":"source/images/zkp/plonk/prescribed_perm_check_problem.png","path":"images/zkp/plonk/prescribed_perm_check_problem.png","modified":0,"renderable":0},{"_id":"source/images/zkp/plonk/prescribed_perm_check_problem_quadratic.png","path":"images/zkp/plonk/prescribed_perm_check_problem_quadratic.png","modified":0,"renderable":0},{"_id":"source/images/zkp/plonk/prescribed_perm_check_problem_reduce.png","path":"images/zkp/plonk/prescribed_perm_check_problem_reduce.png","modified":0,"renderable":0},{"_id":"source/images/zkp/plonk/prescribed_perm_check_problem_prove_verify.png","path":"images/zkp/plonk/prescribed_perm_check_problem_prove_verify.png","modified":0,"renderable":0},{"_id":"source/images/zkp/plonk/prescribed_perm_check_problem_complete.png","path":"images/zkp/plonk/prescribed_perm_check_problem_complete.png","modified":0,"renderable":0},{"_id":"source/images/zkp/plonk/copy_constraint_example.png","path":"images/zkp/plonk/copy_constraint_example.png","modified":0,"renderable":0},{"_id":"source/images/zkp/plonk/custom_gate.png","path":"images/zkp/plonk/custom_gate.png","modified":0,"renderable":0},{"_id":"source/images/zkp/zkevm/polygon/state_machines.png","path":"images/zkp/zkevm/polygon/state_machines.png","modified":0,"renderable":0},{"_id":"source/images/zkp/zkevm/polygon/micro_processor.png","path":"images/zkp/zkevm/polygon/micro_processor.png","modified":0,"renderable":0}],"Cache":[{"_id":"source/_posts/MPT.md","hash":"1d0394f11c3361b4474dbe49ced5c5751144fe91","modified":1699158073732},{"_id":"source/_posts/blockchain.kademlia.md","hash":"0cb0522a114bc53d79f2db4a1bf2a02c29ef1c2f","modified":1699158073732},{"_id":"source/_posts/hello-world.md","hash":"8241e671f980a667f875b9a6493f17bd333a1cfb","modified":1699158073733},{"_id":"source/_posts/geth-fine-tune.md","hash":"5431cd654f7152fd5c590891a53568f54eec4125","modified":1699158073733},{"_id":"source/images/evm.layout.png","hash":"6927cb3b922b2bcec926ec5d797e7ea8ea2b5d00","modified":1699158073736},{"_id":"source/images/kademlia.onlineprob.png","hash":"98c55aa819ef7047b3749c89eb4546f0d799a239","modified":1699158073738},{"_id":"source/images/mpt.state.ref.png","hash":"ec747cf092820c0ab5f72d0f4f07f7705bb9ecf0","modified":1699158073740},{"_id":"source/_posts/blockchain/danksharding.md","hash":"ecf0cb650e0a0e7b251b86ccfbac535a6a217522","modified":1699158077033},{"_id":"source/_posts/cryptography/bls12-381.md","hash":"f3d4cd2f87eb82929b9725644bf607469e7f7526","modified":1699158213767},{"_id":"source/_posts/cryptography/cryptography-01-primitive-group-and-field.md","hash":"b109aef9a3c4ca55a7198c284cd007716b094044","modified":1699158073732},{"_id":"source/_posts/cryptography/cryptography-03-elliptic-curve.md","hash":"3ee7e6e7b609605f7c26d5bf1f7508771d6c7a95","modified":1699158077033},{"_id":"source/_posts/cryptography/cryptography-04-digital-signature.md","hash":"f2539c849c5e052c62123a7fde737ce4c0300134","modified":1699158077033},{"_id":"source/_posts/cryptography/cryptography-02-rsa.md","hash":"6c0bb57a988b036e8b8beca7343b69c025bc4ea7","modified":1699158077033},{"_id":"source/_posts/cryptography/kzg-commitment.md","hash":"9c7b821d2c4a0cb56c70c1a0711af74118253fbd","modified":1699158077034},{"_id":"source/_posts/golang/go-reflect.md","hash":"c2808bbd3bf422ab5679c246444975107b98fb1e","modified":1699158073733},{"_id":"source/_posts/cryptography/montgomery-multiplication.md","hash":"806262fecf990ff4c95eb40d954728b635724bdd","modified":1699158213767},{"_id":"source/_posts/cryptography/paillier-encryption.md","hash":"4e43aa2d126bcb334563262563071c764c3ae19f","modified":1699158073732},{"_id":"source/_posts/cryptography/two-party-ecdsa.md","hash":"9416016bb3f47864aeb888db208f63f93ec10f71","modified":1699158077034},{"_id":"source/_posts/golang/go-similar-concepts-comparison.md","hash":"5de26ef1a5907db4264ff5e491b75aa2dfb43af1","modified":1699158073733},{"_id":"source/_posts/rust/rust-02-basics.md","hash":"be1111d868417c54b8b019938b8144e8be35df1f","modified":1699158073734},{"_id":"source/_posts/rust/rust-01-resource.md","hash":"5e2c159128ccef35da0d3a2a72b6bb7e1c12fc73","modified":1699158073734},{"_id":"source/_posts/rust/rust-03-ownership.md","hash":"7d39498532088f438d76f1d0b42892bc985c0c95","modified":1699158073734},{"_id":"source/_posts/rust/rust-04-lifetime.md","hash":"d338498d7d159a3f94687082a10fc28ada706b16","modified":1699158073734},{"_id":"source/_posts/rust/rust-05-memory-layout.md","hash":"9a8c7dac3a23bca5f7692a3f6c0c8827dfd8c7e5","modified":1699158073734},{"_id":"source/_posts/rust/rust-07-macro.md","hash":"f6e51943ab413f5462baca1327c53ac20a8afcda","modified":1699158073734},{"_id":"source/_posts/rust/rust-09-functional.md","hash":"b2e1f9a46e02c5aa49ea19394e074777558a51eb","modified":1699158073735},{"_id":"source/_posts/rust/rust-06-smart-pointer.md","hash":"909ecf0ecf594651191e0953eca17aca7fa64669","modified":1699158073734},{"_id":"source/_posts/rust/rust-08-project-management.md","hash":"2c1ab1e7b8c8510935a694e33aa2c676437f0fd1","modified":1699158073734},{"_id":"source/_posts/rust/rust-10-concurrency.md","hash":"5bba6d7c1b0e3a24f07a4899f1162a93af8ad5b1","modified":1699158073735},{"_id":"source/_posts/rust/rust-async.md","hash":"f8b896f06752fcdb3c9fa34d2ed0ba958aef9c49","modified":1699158073735},{"_id":"source/_posts/rust/rust-cargo-all-in-one.md","hash":"27eb587fe52f0461e1e30ed82b5d10a806e168f0","modified":1699158073735},{"_id":"source/_posts/rust/rust-similar-concepts-comparison.md","hash":"17c7d67efd6315828a09762c633e7f8a0c9cdee2","modified":1699158073735},{"_id":"source/_posts/rust/rust-sugar.md","hash":"dfa5a3f7bfd985118b97ae9055da496778b604e8","modified":1699158073735},{"_id":"source/_posts/rust/rust-tools.md","hash":"3019a116f156d05a95fd8fc76fa8b1380f778f24","modified":1699158073735},{"_id":"source/_posts/rust/rust-cargo-doc.md","hash":"52303be5d9e342444a4cb661fa418ab68b28d640","modified":1699158073735},{"_id":"source/_posts/zkp/stark-101.md","hash":"867f0c912e4f6abe1512848408f7afebae04b536","modified":1699158213767},{"_id":"source/_posts/zkp/understanding-plonk.md","hash":"4d501404bae2e4674c6b2d519014097c6b506807","modified":1699669060225},{"_id":"source/images/geth/sync.mode.jpg","hash":"657cc148abc14f01e7483265c67936e73f79d0e4","modified":1699158073737},{"_id":"source/_posts/tools/markdown_and_latex.md","hash":"50806cfd650f2c031a90d1860a688fa8320517a3","modified":1701410801564},{"_id":"source/_posts/math/fourier-series.md","hash":"4831e39540e229ae7d16972e299819adfff264dc","modified":1701013907630},{"_id":"source/images/two_party_ecdsa/paillier_enc.png","hash":"132cdc74eb588951d1bd347b1b6e825912bc0460","modified":1699158073744},{"_id":"source/images/paillier/homomorphic_addition.png","hash":"26b0e4a070a7e29710e2ceace09bbdb79b3a883e","modified":1699158073741},{"_id":"source/images/paillier/homomorphic_mul.png","hash":"e8a2420501140a8d48db202065aa221df92f19dc","modified":1699158073741},{"_id":"source/_posts/geth/code_analysis/geth.0.get.start.md","hash":"6cd5256bae5e43f3dfcd341e27c52eccf8848f60","modified":1699158073733},{"_id":"source/_posts/geth/code_analysis/geth.1.rpc.md","hash":"623aec4f3cd8afb85b7b30d8bfde50bb2e5a4d4a","modified":1699158073733},{"_id":"source/_posts/geth/code_analysis/geth.evm.md","hash":"ecea3e42003911dd58a2d6a9b8293f02ccb16a75","modified":1699158073733},{"_id":"source/_posts/geth/tech_docs/geth.prune.md","hash":"9ab8f315c3374f67ff7ac6f74a7fbef0ca9f7894","modified":1699158073733},{"_id":"source/_posts/geth/tech_docs/geth.sync.mode.md","hash":"7502b826b5ecbdd02f759a1780528dae344ccad7","modified":1699158073733},{"_id":"source/_posts/cryptography/zkp/zkp-demystify-zokrates.md","hash":"6ee1897511a409ed7e9e476a4375f162e70f1770","modified":1699158077035},{"_id":"source/_posts/geth/tech_docs/geth.v1.10.0.md","hash":"d303922d31b987d5b57080fb6833b84e568de342","modified":1699158073733},{"_id":"source/_posts/cryptography/zkp/zkp-a-brief-understanding.md","hash":"904b00c9a8b65b48db1482f2935fd9b18c37a8a1","modified":1699158077034},{"_id":"source/_posts/cryptography/zkp/zkp-groth16-paper-review.md","hash":"47cfa35483dae66a3c56465f217d27e738f15633","modified":1699158077035},{"_id":"source/_posts/cryptography/elliptic_curve/elliptic-curve-pairing.md","hash":"3d917772eb27821cbf36c73e73f3faa6ce8f9a54","modified":1701399040735},{"_id":"source/_posts/cryptography/zkp/zkp-under-the-hood.md","hash":"ce2b0522282e7b1676f6b884e4afad4e62ec414b","modified":1699158077035},{"_id":"source/_posts/rust/crates/rust-serde.md","hash":"9b985750a751a7bd28effe9e97147e4207a5f093","modified":1699158073734},{"_id":"source/_posts/rust/crates/rust-frequently-used-crates.md","hash":"39aec8a77f62d6c3b164ecef0663a612423afd63","modified":1699158073733},{"_id":"source/images/cryptography/elliptic_curve/paring_ec.png","hash":"85ce9f154c2c896ba28d601ef0a0bac89a82a627","modified":1699158077036},{"_id":"source/images/cryptography/elliptic_curve/point_addition.webp","hash":"ba6cd29aec4a694b53933d0824df180158f98316","modified":1699158077036},{"_id":"source/images/rust/memory/trait_object_memory.png","hash":"dd750cffa7bca5bde7856cfb65f4500de286c96f","modified":1699158073742},{"_id":"source/images/rust/ownership/move-string.png","hash":"d708edd6fb84b9d4ebe75556932f6b920934ca9a","modified":1699158073743},{"_id":"source/_posts/rust/rust_std/rust-smart-pointer-and-internal-mutibility.md","hash":"d97435f2c35ffcd4feb8a1aff787c7c20922438a","modified":1699158073735},{"_id":"source/_posts/rust/rust_std/rust-std-data-structure-1.md","hash":"34627ff9cff48a6a79a36d4e88cc4d32e118c3ae","modified":1699158073735},{"_id":"source/_posts/rust/rust_std/rust-std-data-structure-2.md","hash":"32b80b9540433ffa77a3a7969e06921eb9ddbbca","modified":1699158073735},{"_id":"source/_posts/rust/rust_std/rust-std-sync.md","hash":"bf648427835ab0d834b2701b28bdfd35088aeb2e","modified":1699158073735},{"_id":"source/images/rust/pointers/rc.png","hash":"f568f69808ef0357ea5e6d42667d863b1139cbf1","modified":1699158073744},{"_id":"source/images/evm.drawio.google.png","hash":"413a44d66153c93e2c31172d988e051a2bed9940","modified":1699158073736},{"_id":"source/images/geth_starts.drawio.png","hash":"3c426bd608121669f232aaa1b05ed7a342287fc8","modified":1699158073737},{"_id":"source/images/kademlia.subtree.png","hash":"d3198467460972151f4b2fe2b56fab0dd9e411ca","modified":1699158073738},{"_id":"source/images/trie.prefix.png","hash":"67d26d416faf0cc43ee35b4acda598e88dad2949","modified":1699158073744},{"_id":"source/images/paillier/carmichael_thorem.png","hash":"75b4d32eb2233db653bca52cdea4a624555b5ce4","modified":1699158073740},{"_id":"source/images/two_party_ecdsa/schnorr_ecdsa_comparison.png","hash":"69cbed302af467a4e99653dfb51dca45c4a5a6f3","modified":1699158073745},{"_id":"source/images/paillier/carmichael_thorem_2.png","hash":"29b103c94c36f4520ca8b675486af3d914998ac1","modified":1699158073741},{"_id":"node_modules/hexo-theme-landscape/package.json","hash":"9a94875cbf4c27fbe2e63da0496242addc6d2876","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/languages/de.yml","hash":"3ebf0775abbee928c8d7bda943c191d166ded0d3","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/languages/en.yml","hash":"3083f319b352d21d80fc5e20113ddf27889c9d11","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/languages/es.yml","hash":"76edb1171b86532ef12cfd15f5f2c1ac3949f061","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/README.md","hash":"d2772ece6d4422ccdaa0359c3e07588834044052","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/languages/fr.yml","hash":"415e1c580ced8e4ce20b3b0aeedc3610341c76fb","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/LICENSE","hash":"c480fce396b23997ee23cc535518ffaaf7f458f8","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/languages/hu.yml","hash":"284d557130bf54a74e7dcef9d42096130e4d9550","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/languages/it.yml","hash":"89b7d91306b2c1a0f3ac023b657bf974f798a1e8","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/_config.yml","hash":"b608c1f1322760dce9805285a602a95832730a2e","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/languages/ko.yml","hash":"881d6a0a101706e0452af81c580218e0bfddd9cf","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/languages/mn.yml","hash":"2e7523951072a9403ead3840ad823edd1084c116","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/languages/no.yml","hash":"965a171e70347215ec726952e63f5b47930931ef","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/languages/nl.yml","hash":"12ed59faba1fc4e8cdd1d42ab55ef518dde8039c","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/languages/ja.yml","hash":"a73e1b9c80fd6e930e2628b393bfe3fb716a21a9","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/languages/tr.yml","hash":"a1cdbfa17682d7a971de8ab8588bf57c74224b5b","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/languages/zh-CN.yml","hash":"1efd95774f401c80193eac6ee3f1794bfe93dc5a","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/languages/pt.yml","hash":"57d07b75d434fbfc33b0ddb543021cb5f53318a8","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/languages/ru.yml","hash":"4fda301bbd8b39f2c714e2c934eccc4b27c0a2b0","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/languages/zh-TW.yml","hash":"53ce3000c5f767759c7d2c4efcaa9049788599c3","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/category.ejs","hash":"765426a9c8236828dc34759e604cc2c52292835a","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/index.ejs","hash":"aa1b4456907bdb43e629be3931547e2d29ac58c8","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/archive.ejs","hash":"2703b07cc8ac64ae46d1d263f4653013c7e1666b","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/layout.ejs","hash":"0d1765036e4874500e68256fedb7470e96eeb6ee","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/page.ejs","hash":"7d80e4e36b14d30a7cd2ac1f61376d9ebf264e8b","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/post.ejs","hash":"7d80e4e36b14d30a7cd2ac1f61376d9ebf264e8b","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/scripts/fancybox.js","hash":"c857d7a5e4a5d71c743a009c5932bf84229db428","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/after-footer.ejs","hash":"414914ebb159fac1922b056b905e570ac7521925","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/archive.ejs","hash":"7cb70a7a54f8c7ae49b10d1f37c0a9b74eab8826","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/archive-post.ejs","hash":"c7a71425a946d05414c069ec91811b5c09a92c47","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/article.ejs","hash":"dfd555c00e85ffc4207c88968d12b219c1f086ec","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/tag.ejs","hash":"eaa7b4ccb2ca7befb90142e4e68995fb1ea68b2e","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/footer.ejs","hash":"3656eb692254346671abc03cb3ba1459829e0dce","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/gauges-analytics.ejs","hash":"21a1e2a3907d1a3dad1cd0ab855fe6735f233c74","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/google-analytics.ejs","hash":"2ea7442ea1e1a8ab4e41e26c563f58413b59a3d0","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/mobile-nav.ejs","hash":"e952a532dfc583930a666b9d4479c32d4a84b44e","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/header.ejs","hash":"c1acd247e14588cdf101a69460cb8319c18cd078","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/sidebar.ejs","hash":"930da35cc2d447a92e5ee8f835735e6fd2232469","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_widget/category.ejs","hash":"dd1e5af3c6af3f5d6c85dfd5ca1766faed6a0b05","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_widget/archive.ejs","hash":"beb4a86fcc82a9bdda9289b59db5a1988918bec3","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/head.ejs","hash":"f215d92a882247a7cc5ea80b241bedfcec0ea6ca","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_widget/recent_posts.ejs","hash":"60c4b012dcc656438ff59997e60367e5a21ab746","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_widget/tag.ejs","hash":"2de380865df9ab5f577f7d3bcadf44261eb5faae","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/_variables.styl","hash":"581b0cbefdaa5f894922133989dd2d3bf71ded79","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_widget/tagcloud.ejs","hash":"b4a2079101643f63993dcdb32925c9b071763b46","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/style.styl","hash":"9c451e5efd72c5bb8b56e8c2b94be731e99db05b","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/_extend.styl","hash":"222fbe6d222531d61c1ef0f868c90f747b1c2ced","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/js/script.js","hash":"998ed4c5b147e1299bf62beebf33514474f28112","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/fancybox/jquery.fancybox.min.css","hash":"1be9b79be02a1cfc5d96c4a5e0feb8f472babd95","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/post/date.ejs","hash":"f1458584b679545830b75bef2526e2f3eb931045","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/post/gallery.ejs","hash":"3d9d81a3c693ff2378ef06ddb6810254e509de5b","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/post/nav.ejs","hash":"16a904de7bceccbb36b4267565f2215704db2880","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/post/tag.ejs","hash":"2fcb0bf9c8847a644167a27824c9bb19ac74dd14","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/post/title.ejs","hash":"4d7e62574ddf46de9b41605fe3140d77b5ddb26d","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/_util/grid.styl","hash":"0bf55ee5d09f193e249083602ac5fcdb1e571aed","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/post/category.ejs","hash":"c6bcd0e04271ffca81da25bcff5adf3d46f02fc0","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/_util/mixin.styl","hash":"44f32767d9fd3c1c08a60d91f181ee53c8f0dbb3","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/_partial/footer.styl","hash":"e35a060b8512031048919709a8e7b1ec0e40bc1b","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/_partial/comment.styl","hash":"79d280d8d203abb3bd933ca9b8e38c78ec684987","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/_partial/archive.styl","hash":"db15f5677dc68f1730e82190bab69c24611ca292","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/_partial/mobile.styl","hash":"a399cf9e1e1cec3e4269066e2948d7ae5854d745","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/_partial/highlight.styl","hash":"bf4e7be1968dad495b04e83c95eac14c4d0ad7c0","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/_partial/sidebar-aside.styl","hash":"890349df5145abf46ce7712010c89237900b3713","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/_partial/article.styl","hash":"80759482d07063c091e940f964a1cf6693d3d406","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/_partial/sidebar-bottom.styl","hash":"8fd4f30d319542babfd31f087ddbac550f000a8a","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/_partial/header.styl","hash":"85ab11e082f4dd86dde72bed653d57ec5381f30c","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/_partial/sidebar.styl","hash":"404ec059dc674a48b9ab89cd83f258dec4dcb24d","modified":1669606834570},{"_id":"source/images/rust/ownership/move-string-2.png","hash":"83342aed7ee254c099ada5d49ed8b9ba52a451a0","modified":1699158073743},{"_id":"source/images/cryptography/zkp/lagrange_interpolating.png","hash":"2e3a62df79b1267dd0172623e46da03801439b01","modified":1699158077038},{"_id":"source/images/rust/pointers/cycle.ref.png","hash":"ed99e2c6020833ccd5af751cf6eb2031cf47d9aa","modified":1699158073743},{"_id":"source/images/zkp/stark/commitment.png","hash":"223d7fbde2b6dc7a2d2b6cabb11fbe4763e8d217","modified":1699158213768},{"_id":"node_modules/hexo-theme-landscape/source/js/jquery-3.4.1.min.js","hash":"88523924351bac0b5d560fe0c5781e2556e7693d","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/fancybox/jquery.fancybox.min.js","hash":"6181412e73966696d08e1e5b1243a572d0f22ba6","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/fonts/fontawesome-webfont.woff","hash":"28b782240b3e76db824e12c02754a9731a167527","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/fonts/fontawesome-webfont.woff2","hash":"d6f48cba7d076fb6f2fd6ba993a75b9dc1ecbf0c","modified":1669606834570},{"_id":"source/images/cryptography/zkp/r1cs.png","hash":"c29022100d7c6581eb2a0c54cc763581d66abf6e","modified":1699158077038},{"_id":"source/images/cryptography/zkp/checking_qap.png","hash":"8f01d96d61a388b1f5d7da55da72428528ccf8e5","modified":1699158077037},{"_id":"source/images/zkp/stark/decommitment.png","hash":"22be5093a44044cb4b623251078bcfefc1fc1951","modified":1699158213768},{"_id":"source/images/zkp/stark/fri.png","hash":"18296b244a046f5f78fda1256760c88aeda09676","modified":1699158213769},{"_id":"source/images/zkp/stark/trace_to_CP.png","hash":"0655eed61ff22c0e16ab48582ffa125d9fd58f50","modified":1699158213771},{"_id":"source/images/kademlia.locating.png","hash":"702d6b779294a3c6e033cc9bde14ef8950982310","modified":1699158073738},{"_id":"node_modules/hexo-theme-landscape/source/css/fonts/FontAwesome.otf","hash":"048707bc52ac4b6563aaa383bfe8660a0ddc908c","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/fonts/fontawesome-webfont.eot","hash":"d980c2ce873dc43af460d4d572d441304499f400","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/fonts/fontawesome-webfont.ttf","hash":"13b1eab65a983c7a73bc7997c479d66943f7c6cb","modified":1669606834570},{"_id":"source/images/rust/macros/16.compile_process.png","hash":"444080319ca2101672941346988f78ed9a37c32d","modified":1699158073742},{"_id":"source/images/math/fourier_series/f_x.png","hash":"539e659af67a921b208780e509c2e28bb8c4fe98","modified":1699158077039},{"_id":"source/images/mpt.png","hash":"700032035bdcd793f94da522330552727b00e5a3","modified":1699158073739},{"_id":"node_modules/hexo-theme-landscape/source/css/images/banner.jpg","hash":"f44aa591089fcb3ec79770a1e102fd3289a7c6a6","modified":1669606834570},{"_id":"source/images/cryptography/rsa/rsa_signature.png","hash":"44eb1a608dafaae244e487cf082aa79c2af5c19f","modified":1699158077036},{"_id":"source/images/zkp/stark/fri_example.png","hash":"b78e0f488557ac5fb59c552b934377fffe9183a9","modified":1699158213770},{"_id":"node_modules/hexo-theme-landscape/source/css/fonts/fontawesome-webfont.svg","hash":"98a8aa5cf7d62c2eff5f07ede8d844b874ef06ed","modified":1669606834570},{"_id":"source/2017-552.pdf","hash":"b1a08857c1f6532f1fbb718c48a34fd48ea7da70","modified":1699158073732},{"_id":"source/images/zkp/plonk/plonk_circuit_to_trace.png","hash":"d341c0439212156becdc0f337d1a12c8627f4592","modified":1699666066351},{"_id":"source/_posts/zkp/plonky2.md","hash":"e480bcf9c62d3c0b4ea41eecfc69794b9fcf6571","modified":1699248267137},{"_id":"public/2023/11/06/zkp/plonky2/index.html","hash":"e259a3f5949391cb60e8629f6ade7f0ae4fd3828","modified":1700633599407},{"_id":"public/2023/11/01/cryptography/montgomery-multiplication/index.html","hash":"453556ffcc134625d3032febe122c3887c08ca71","modified":1700633599407},{"_id":"public/2023/11/01/cryptography/bls12-381/index.html","hash":"f536738a60468b41f512f7d68704ed7f27068f0a","modified":1701227621497},{"_id":"public/2023/11/01/zkp/understanding-plonk/index.html","hash":"5380b76a95b4e31fdf1ba9998237a05423fa9e06","modified":1700633599407},{"_id":"public/2023/10/16/cryptography/kzg-commitment/index.html","hash":"59016382550c6d24d3ccdfbb76480308d680f063","modified":1701227621497},{"_id":"public/2023/10/12/blockchain/danksharding/index.html","hash":"5eb979dc0360ff315388066894b8942c6d84edf9","modified":1702283024204},{"_id":"public/2023/10/12/math/fourier-series/index.html","hash":"0354b9bd3456d421c8cb2e0e3012706284fd41d6","modified":1702283024204},{"_id":"public/2023/10/01/tools/markdown_and_latex/index.html","hash":"6c5cc03ab9de516dd0f52920416a36dca55d22a8","modified":1702283024204},{"_id":"public/2023/07/14/cryptography/zkp/zkp-demystify-zokrates/index.html","hash":"b0de23154aa21b9688ed24ad1cb04d01ac71688f","modified":1702283024204},{"_id":"public/2023/07/07/cryptography/zkp/zkp-groth16-paper-review/index.html","hash":"8571282e253137689219c3df12c13a6f65d6969a","modified":1702283024204},{"_id":"public/2023/06/17/cryptography/cryptography-03-elliptic-curve/index.html","hash":"640ac9a8bd1335e8793b85172fb1e5361b830fe2","modified":1702283024204},{"_id":"public/2023/06/01/rust/rust-10-concurrency/index.html","hash":"2dbbed0e17f98e4ece75f14723daaab87fab9dda","modified":1702283024204},{"_id":"public/2023/04/11/rust/rust-09-functional/index.html","hash":"428386cb66dcb64aff7a5f6a709a2883889bf776","modified":1702283024204},{"_id":"public/2023/04/01/rust/crates/rust-serde/index.html","hash":"2220e133f2a841c601142bb0bede1183906c1335","modified":1702283024204},{"_id":"public/2023/03/25/geth/tech_docs/geth.prune/index.html","hash":"e27b60acff0e964ec045c097adfa716e25e2d3da","modified":1702283024204},{"_id":"public/2023/03/05/golang/go-similar-concepts-comparison/index.html","hash":"ff14a3ab853091431b9573ff38d63e7a94527350","modified":1702283024204},{"_id":"public/2023/02/07/cryptography/two-party-ecdsa/index.html","hash":"896ffa24c7c6f7275cf0e669565ec07fe4f15dbb","modified":1702283024204},{"_id":"public/2023/01/22/MPT/index.html","hash":"514b09ffa00adffa62f4dce32a8d4ae306ca8e5e","modified":1702283024204},{"_id":"public/2023/01/01/geth-fine-tune/index.html","hash":"b3e25c8e34117cdeedaa88c9c90fd29df8f773d9","modified":1702283024204},{"_id":"public/2022/12/13/rust/crates/rust-frequently-used-crates/index.html","hash":"3c3c2b9a0eaaa6846e11c7cbe541b71e58200ec1","modified":1702283024204},{"_id":"public/2022/11/27/hello-world/index.html","hash":"a3e4e8ddc9d8cc9fa92e0c34500b84783096cea9","modified":1702283024204},{"_id":"public/2022/11/20/rust/rust-tools/index.html","hash":"56ec8e4f60c20b145b59b7774b0b36cd3ee53d3a","modified":1702283024204},{"_id":"public/2022/11/13/rust/rust-cargo-doc/index.html","hash":"3642d96d8e3919425c5f39bcd5914ff748f01bcd","modified":1702283024204},{"_id":"public/2022/11/08/geth/code_analysis/geth.1.rpc/index.html","hash":"ed49bb431ee485a1d78976a0d06d4c76973a9c05","modified":1702283024204},{"_id":"public/2022/10/25/rust/rust-05-memory-layout/index.html","hash":"594366895eb0e26a60df49b7384290707c8cb821","modified":1702283024204},{"_id":"public/2022/09/27/rust/rust-01-resource/index.html","hash":"070b3d3419fd454488fd57a6b6c756153893dff3","modified":1702283024204},{"_id":"public/archives/index.html","hash":"94ed15e3a37e7aa24999359c3e990e241ac660a7","modified":1702283024204},{"_id":"public/archives/page/2/index.html","hash":"a50cc6b434912028db5b9abb6cf514197815bb0e","modified":1702283024204},{"_id":"public/archives/page/3/index.html","hash":"fafe090cd159de1c106397c26198a488bfd20d26","modified":1702283024204},{"_id":"public/archives/page/4/index.html","hash":"258f1d35da91079ca5264fa5bfc6241b76dd8a39","modified":1702283024204},{"_id":"public/archives/page/5/index.html","hash":"4a05dd87aba90bc7bcc570c5f5cd8203ad47b997","modified":1702283024204},{"_id":"public/archives/page/6/index.html","hash":"ca8da1f9cf094356eb58dd376e7c6cd401a6fa4f","modified":1702283024204},{"_id":"public/archives/2022/index.html","hash":"9079842e6f3569af6eb31c65ca2dd7b57bbefa95","modified":1702283024204},{"_id":"public/archives/2022/page/2/index.html","hash":"44da281b1dfc952a0a1760e073af4ed90510e325","modified":1702283024204},{"_id":"public/archives/2022/09/index.html","hash":"28cf5d98effa4d170c28836a9099d0e68f0eabd8","modified":1702283024204},{"_id":"public/archives/2022/10/index.html","hash":"23347789920e52f99740e4092066f2ed18ea520c","modified":1702283024204},{"_id":"public/archives/2022/11/index.html","hash":"dccce48f731fb59ae46983c7e60c12406edeb0f5","modified":1702283024204},{"_id":"public/archives/2022/12/index.html","hash":"f5b27921f6d99260ca26f84c236b2c4b671829ee","modified":1702283024204},{"_id":"public/archives/2023/index.html","hash":"7f8e4ef812d163dd3c6fb848d6b957d94f508acd","modified":1702283024204},{"_id":"public/archives/2023/page/2/index.html","hash":"77c96e9efef06850bea97f526564e0d13c4bafc8","modified":1702283024204},{"_id":"public/archives/2023/page/3/index.html","hash":"5e4d858054d09f6823e212526f6df9d236628479","modified":1702283024204},{"_id":"public/archives/2023/page/4/index.html","hash":"058083198b84e8899d42af14bcffb047b050ea33","modified":1702283024204},{"_id":"public/archives/2023/01/index.html","hash":"a7390007d2062b308240f170bff1104eb3f97a7d","modified":1702283024204},{"_id":"public/archives/2023/02/index.html","hash":"57964ae782230b70e5109f5181933fd3955dde26","modified":1702283024204},{"_id":"public/archives/2023/03/index.html","hash":"d9e37568cdc6aeb0973446c6fec3a0a71f814152","modified":1702283024204},{"_id":"public/archives/2023/04/index.html","hash":"0f119e7ca1d94b5cf39571566b93db82a53e922b","modified":1702283024204},{"_id":"public/archives/2023/05/index.html","hash":"387f398976d2d37e77bcf68dcd5b377a158484ec","modified":1702283024204},{"_id":"public/archives/2023/06/index.html","hash":"9b1bc2e4eb048de400a60c23e0ac73acec8cde63","modified":1702283024204},{"_id":"public/archives/2023/07/index.html","hash":"7b0e30cb20af55acc18acae284039dec1ca46b51","modified":1702283024204},{"_id":"public/archives/2023/10/index.html","hash":"e2e5edd6419e2c166db4d453b73f8f5b96e21c3e","modified":1702283024204},{"_id":"public/archives/2023/11/index.html","hash":"a46e47b1020a478d8720fe3e834ea7c62a86f0f3","modified":1702283024204},{"_id":"public/tags/blockchain/index.html","hash":"605b7443aa03098a54c699a64b34cfa7433bd9e5","modified":1702283024204},{"_id":"public/tags/geth/index.html","hash":"406147d619dc774321bdc87321ff010d51d6b41e","modified":1702283024204},{"_id":"public/tags/cryptography/index.html","hash":"eda4cb9f15d750071a66e3e80ff3077b73724d41","modified":1702283024204},{"_id":"public/tags/cryptography/page/2/index.html","hash":"720ba7f6af1c9d5f1e0659ae25279242a453e588","modified":1702283024204},{"_id":"public/tags/number-theory/index.html","hash":"cba28d18c31c4f60dfd8e43d9388631403d28677","modified":1702283024204},{"_id":"public/tags/ecdsa/index.html","hash":"6f20a96f5acb6129a88691615852def0e03a392f","modified":1702283024204},{"_id":"public/tags/mpc/index.html","hash":"5a3d7b14d0dace715b361a3a8b01bc1c6d50c6cf","modified":1702283024204},{"_id":"public/tags/golang/index.html","hash":"b94e6c2904965f51843aa28a032049d8910b3732","modified":1702283024204},{"_id":"public/tags/rust/index.html","hash":"3216f068db451dbc63411eb0eaaf9f91075f59fe","modified":1702283024204},{"_id":"public/tags/rust/page/2/index.html","hash":"7ce94a328bed5e8c1d65d27e3031d760d949eef3","modified":1702283024204},{"_id":"public/tags/cargo/index.html","hash":"a2d1c2272183f7b95d702c3a2541b135872f4e95","modified":1702283024204},{"_id":"public/tags/zkp/index.html","hash":"27819920865a27f49e99ca9c9a4740028b664135","modified":1702283024204},{"_id":"public/tags/tool/index.html","hash":"7c615230d6b0c1d6f0e12af03e9b3d2eaf368044","modified":1702283024204},{"_id":"public/tags/math/index.html","hash":"f5155a76b420d4fe35fe439598a3b6a199298a9d","modified":1702283024204},{"_id":"public/tags/rust-crate/index.html","hash":"4552987c640338e92045a2057a6a023cf7583029","modified":1702283024204},{"_id":"public/tags/rust-std/index.html","hash":"570d7d79db7206044e18b28fddaca063347ea4a3","modified":1702283024204},{"_id":"public/2023/11/03/zkp/stark-101/index.html","hash":"e728dd7a064ca2a44ea93dfe966a6d3ea712a741","modified":1700633599407},{"_id":"public/2023/07/01/cryptography/zkp/zkp-under-the-hood/index.html","hash":"d933dc2b2a1bdbac038788c6a46c92ca84dbfc8c","modified":1702283024204},{"_id":"public/2023/06/27/cryptography/zkp/zkp-a-brief-understanding/index.html","hash":"a77bb9f26766c2b1e395224835fd58ee349bcc18","modified":1702283024204},{"_id":"public/2023/06/27/cryptography/elliptic_curve/elliptic-curve-pairing/index.html","hash":"51958e19ed40c90beecc6d61f4799bcab58df9a6","modified":1702283024204},{"_id":"public/2023/06/20/cryptography/cryptography-04-digital-signature/index.html","hash":"87cf1670d877edcc277f4d5f3d933048f308996a","modified":1702283024204},{"_id":"public/2023/06/10/cryptography/cryptography-02-rsa/index.html","hash":"1d5a9fea7aa5802731353d904150e40f91f1e1bd","modified":1702283024204},{"_id":"public/2023/06/08/rust/rust_std/rust-std-sync/index.html","hash":"650436c8268d58da5dc2fe3c96b79994b43a3a7a","modified":1702283024204},{"_id":"public/2023/06/03/rust/rust_std/rust-smart-pointer-and-internal-mutibility/index.html","hash":"344f77470ff6545261f4eb8e8470dff1e3943a39","modified":1702283024204},{"_id":"public/2023/06/03/cryptography/cryptography-01-primitive-group-and-field/index.html","hash":"65dc137c0f12ec31960957d8977a0aebcbfddcd7","modified":1702283024204},{"_id":"public/2023/05/02/rust/rust_std/rust-std-data-structure-2/index.html","hash":"8245ebda94413720d2d5e94f97d8d399ca0b87ca","modified":1702283024204},{"_id":"public/2023/05/01/rust/rust_std/rust-std-data-structure-1/index.html","hash":"e67fa2e108149cc8f5e831654eea689555520ff0","modified":1702283024204},{"_id":"public/2023/03/18/geth/tech_docs/geth.sync.mode/index.html","hash":"551b172dc9de61761bbf5c47f5ec5ec5c86f94e7","modified":1702283024204},{"_id":"public/2023/03/15/geth/tech_docs/geth.v1.10.0/index.html","hash":"38ba66c44c60fa43a77e3a5738ba30134c13251d","modified":1702283024204},{"_id":"public/2023/03/02/golang/go-reflect/index.html","hash":"daf45c37c6c7cbd8e7a19be3b65615b2f0184851","modified":1702283024204},{"_id":"public/2023/02/23/cryptography/paillier-encryption/index.html","hash":"aefa9d39d385385d948764d30775539bcd7384ff","modified":1702283024204},{"_id":"public/2023/01/15/blockchain.kademlia/index.html","hash":"8d45243f395519f93e18a554bd3348857edf6a06","modified":1702283024204},{"_id":"public/2023/01/13/rust/rust-async/index.html","hash":"d5df4ab53ae2e7ff67cdd5119c3fdca5950117d6","modified":1702283024204},{"_id":"public/2023/01/08/geth/code_analysis/geth.evm/index.html","hash":"7d9eef3e1be997eb8442fd2f4bbf3cdd1de3eb0f","modified":1702283024204},{"_id":"public/2022/12/28/rust/rust-07-macro/index.html","hash":"1a42ab081cdce0734c933a0f26f1ebdfed83550d","modified":1702283024204},{"_id":"public/2022/12/06/rust/rust-cargo-all-in-one/index.html","hash":"e2bb545576b9ac935f2e5c8d9eddf789c296af39","modified":1702283024204},{"_id":"public/2022/11/23/rust/rust-similar-concepts-comparison/index.html","hash":"1cb7caf668d1e2dededda346732965afc882fb42","modified":1702283024204},{"_id":"public/2022/11/17/rust/rust-sugar/index.html","hash":"beec4813cb5271a73e43c44d7364a7827f3e1753","modified":1702283024204},{"_id":"public/2022/11/06/rust/rust-08-project-management/index.html","hash":"d57e92577442da4c3c3274ea58949b8fc96eb265","modified":1702283024204},{"_id":"public/2022/11/01/geth/code_analysis/geth.0.get.start/index.html","hash":"ef8ed821727c55e7f4fddbf15f6cabbf7705d494","modified":1702283024204},{"_id":"public/2022/10/30/rust/rust-06-smart-pointer/index.html","hash":"e2d6b952bd2f4ea72bc6d5051df98da2ceb89c42","modified":1702283024204},{"_id":"public/2022/10/18/rust/rust-04-lifetime/index.html","hash":"f6af715f33cdb8ebdbb94a16c9517ca6858d5a47","modified":1702283024204},{"_id":"public/2022/10/11/rust/rust-03-ownership/index.html","hash":"b1636058e9a9e6b89e104caad2f787f892e9950f","modified":1702283024204},{"_id":"public/2022/10/04/rust/rust-02-basics/index.html","hash":"4dedce4d393796b62e4582c13016999a29796776","modified":1702283024204},{"_id":"public/index.html","hash":"cf5dd165d63af746c113ffb7d64ccb339c0185f0","modified":1702352970990},{"_id":"public/page/2/index.html","hash":"19e3f9ac543ede1e1703d6d5ad5d4f5cc3afcaed","modified":1702364965612},{"_id":"public/page/3/index.html","hash":"dbfddfe01ae4b74a7774232e24ed29e733ca3ba9","modified":1702283024204},{"_id":"public/page/4/index.html","hash":"73ad2b1f5db5437178cf2a58c20880739997bede","modified":1702283024204},{"_id":"public/page/5/index.html","hash":"13c4940f12e6f0b05ec51f1ebe3b7a42f2b6f183","modified":1702283024204},{"_id":"public/page/6/index.html","hash":"9c6507867088d15e8f9cc5416b072da144ddcf6e","modified":1702283024204},{"_id":"public/images/zkp/plonk/plonk_circuit_to_trace.png","hash":"d341c0439212156becdc0f337d1a12c8627f4592","modified":1699667020264},{"_id":"source/_posts/math/goldilocks-field.md","hash":"0d28d6301c8bd16bcc6e126e751af964ecea4128","modified":1700834519863},{"_id":"source/_posts/cuda/inline-ptx-assembly-in-cuda.md","hash":"a2eb51caa1376faaa14e33e8ec527ae0d26b0042","modified":1700619356342},{"_id":"source/_posts/cpp/cpp-notes.md","hash":"3fae4c9374b0ebea986d77d96f1347697f1ebca9","modified":1700019450981},{"_id":"source/images/zkp/plonk/zero_test.png","hash":"23686fabe59c9bec98ff658ce1782886fbd5c5b3","modified":1699668979336},{"_id":"public/2023/11/18/math/goldilocks-field/index.html","hash":"2e06f4f6cfa64d42c201cba3d65db662e9518fe7","modified":1701138383617},{"_id":"public/2023/11/15/cpp/cpp-notes/index.html","hash":"bfa9974c0641ff74b5845671c5df8b1da374952d","modified":1700553651994},{"_id":"public/2023/11/15/cuda/inline-ptx-assembly-in-cuda/index.html","hash":"ca85357b31121e56a4cf31f85392b122576e7a07","modified":1702283024204},{"_id":"public/images/zkp/plonk/zero_test.png","hash":"23686fabe59c9bec98ff658ce1782886fbd5c5b3","modified":1700277062226},{"_id":"source/_posts/cpp/generics_and_macros.md","hash":"35f6098a6a30a9fcdca9995fb324fea72d6933d2","modified":1700706616168},{"_id":"source/_posts/cpp/key_words.md","hash":"33ffe7f7a6d05de363fd530dea3c46d49b0080a1","modified":1700723138353},{"_id":"source/_posts/cpp/types.md","hash":"6af7e4b22d1f762de43cb1e8639d69cf873100d3","modified":1700569616654},{"_id":"source/_posts/cpp/std.md","hash":"e0c17144c4eeb4af63face571c963de8fb081d82","modified":1700571950461},{"_id":"public/2023/11/21/cpp/generics_and_macros/index.html","hash":"7a4275b62559015405568a8ee6e8cce000d833f9","modified":1702283024204},{"_id":"public/2023/11/21/cpp/key_words/index.html","hash":"ba2cd56f969ea04d401db7af8383100a585033cd","modified":1702283024204},{"_id":"public/2023/11/21/cpp/std/index.html","hash":"e0254a767a4556cac5fe94b814a452d6f346498f","modified":1702283024204},{"_id":"public/2023/11/15/cpp/types/index.html","hash":"da88c592cd346be4592a71b46fd32f8342927fb9","modified":1702283024204},{"_id":"public/archives/2023/page/5/index.html","hash":"d65fe390b8df690364e3b3900db925b68a162455","modified":1702283024204},{"_id":"public/archives/2023/11/page/2/index.html","hash":"19d8717ae0ca381e1b53e3db2b5045bade73c094","modified":1702283024204},{"_id":"public/tags/cpp/index.html","hash":"b9921797e58a7ef9f9185a8b3dfb4fbad6f3818f","modified":1702283024204},{"_id":"source/_posts/cuda/concepts_and_keywords.md","hash":"d1fe9b6dd6dbfba1eef681c639df5d49a33e911f","modified":1700724603676},{"_id":"source/_posts/cuda/cuda_api_and_tools.md","hash":"845fef81ac8e65bdc6b6c18f1795ea248e368b4d","modified":1700725285289},{"_id":"source/images/cuda/cuda_stream_example.png","hash":"cb321f532bac228ef187e82210ad2c2586292b0e","modified":1700620752287},{"_id":"source/_posts/arithmatic/efficient_implementations.md","hash":"5a2d092c2fefd8ba70212e9f75697045089600b9","modified":1700637615345},{"_id":"source/_posts/arithmatic/tricks.md","hash":"b51d383cef8fa22b5fae0af5ecae419589bf9a8f","modified":1702016313813},{"_id":"public/2023/11/22/cuda/cuda_api_and_tools/index.html","hash":"bc7785310e3fd6b66d5ab9f07ec56be34f99a523","modified":1702283024204},{"_id":"public/2023/11/22/arithmatic/efficient_implementations/index.html","hash":"327bf3b78c8a02f955d0b6f5b493cfe2a41cbb8f","modified":1702283024204},{"_id":"public/2023/11/22/arithmatic/tricks/index.html","hash":"cce5cddf2b571bf7b01a629452a162ad7cfeb1f7","modified":1702283024204},{"_id":"public/archives/page/7/index.html","hash":"f3584f6a8de495c89d934d6a5ccecda31282b0f1","modified":1702283024204},{"_id":"public/tags/cuda/index.html","hash":"6dda011c800cea67d6028029d4d224a19fb11d3c","modified":1702283024204},{"_id":"public/tags/arithmatic/index.html","hash":"75a16681f4f1231f7559cdf7ceabdbe9e2d976df","modified":1702283024204},{"_id":"public/2023/11/22/cuda/concepts_and_keywords/index.html","hash":"b4cddfafe726dcee8cc0a89b800f20272ff9600a","modified":1702283024204},{"_id":"public/page/7/index.html","hash":"eae3b9f46a731c38357915ac651086a4cf0af55a","modified":1702283024204},{"_id":"public/images/cuda/cuda_stream_example.png","hash":"cb321f532bac228ef187e82210ad2c2586292b0e","modified":1700633599407},{"_id":"source/images/arithmatic/multiple_precision_addition.png","hash":"9f1f24e8f95d44e62e98277b48bac948ae91285e","modified":1700635373358},{"_id":"source/images/arithmatic/radix_b_repesentation.png","hash":"9ce9e52f7a9a2ecd8b416e1cf8d11308bb604a8a","modified":1700634163557},{"_id":"public/images/arithmatic/multiple_precision_addition.png","hash":"9f1f24e8f95d44e62e98277b48bac948ae91285e","modified":1700745044711},{"_id":"public/images/arithmatic/radix_b_repesentation.png","hash":"9ce9e52f7a9a2ecd8b416e1cf8d11308bb604a8a","modified":1700745044711},{"_id":"source/_posts/cryptography/zkp/sgx_multi_prover.md","hash":"7cb2dec1b24e1f31a79ef8ed292dec2920733581","modified":1700834715647},{"_id":"source/_posts/arithmatic/montgomery-multiplication.md","hash":"067016492be4574514e2aa6b773f87af55b35442","modified":1702003544046},{"_id":"source/_posts/cryptography/zkp/stark-101.md","hash":"867f0c912e4f6abe1512848408f7afebae04b536","modified":1699157905566},{"_id":"source/_posts/cryptography/zkp/plonky2.md","hash":"31d92b6cfa6cd6db20f8474144f740c57a0c7e9a","modified":1701676923117},{"_id":"source/_posts/cryptography/zkp/zk-bench.md","hash":"595402d134d08adef02d80ba28de119e7a84947e","modified":1700834726446},{"_id":"source/_posts/cryptography/zkp/understanding-plonk.md","hash":"0fdbd1e6ae465944722bff3124e587af1ec31576","modified":1701228285714},{"_id":"public/2023/11/24/cryptography/zkp/zk-bench/index.html","hash":"c90de35829e35478749d060aa8f31790de9a68ce","modified":1702283024204},{"_id":"public/2023/11/24/cryptography/zkp/sgx_multi_prover/index.html","hash":"17af084e22fd2e7c4f43b1c285092c7caa614a2a","modified":1702283024204},{"_id":"public/2023/11/06/cryptography/zkp/plonky2/index.html","hash":"bc87f9693d8e7142e0a6697ab0315800511d175b","modified":1701962639979},{"_id":"public/2023/11/01/arithmatic/montgomery-multiplication/index.html","hash":"11e20bf895491889fdbaae51754c6370b39f5805","modified":1701661940748},{"_id":"public/2023/11/01/cryptography/zkp/understanding-plonk/index.html","hash":"b840bb54b8f6746718ba169c352e38333336e049","modified":1702283024204},{"_id":"public/2023/11/03/cryptography/zkp/stark-101/index.html","hash":"1ecd81fc0dad55c2c3c15c58b63027faeef4ab75","modified":1702283024204},{"_id":"source/_posts/arithmatic/goldilocks-field.md","hash":"00d471ce337f6d98e577533d2f400bfc3b2ef0ea","modified":1701164677469},{"_id":"source/images/zkp/plonk/gate_evaluation_zero_test.png","hash":"a62a302d7dd13cc9d19913819fa4e950b483ed89","modified":1701141838168},{"_id":"source/images/zkp/plonk/permutation_check.png","hash":"04d7e45bd261bb24a0e0e41beb3e53ded818b43b","modified":1701142333439},{"_id":"public/2023/11/18/arithmatic/goldilocks-field/index.html","hash":"4ea47edb4de9299e305dcbd1cb59ae77437067cc","modified":1702283024204},{"_id":"public/images/zkp/plonk/gate_evaluation_zero_test.png","hash":"a62a302d7dd13cc9d19913819fa4e950b483ed89","modified":1701158595615},{"_id":"public/images/zkp/plonk/permutation_check.png","hash":"04d7e45bd261bb24a0e0e41beb3e53ded818b43b","modified":1701158595615},{"_id":"source/images/zkp/plonk/copy_constraint_example.png","hash":"ccace19afe01e3991b688eef27da922f13b69365","modified":1701185887943},{"_id":"source/images/zkp/plonk/custom_gate.png","hash":"35c2febb4bd8044f48cf5d42744ccc508503d436","modified":1701227590315},{"_id":"source/images/zkp/plonk/prod_check_lemma.png","hash":"e6e29ac8e9eae8cdeff6e8b9909904c612d6ccb4","modified":1701165709852},{"_id":"source/images/zkp/plonk/prod_check_prove_verify.png","hash":"e6de2efae2fdbb98cc49d6bae5965ed311412e4a","modified":1701165956061},{"_id":"source/images/zkp/plonk/prescribed_perm_check_problem_prove_verify.png","hash":"58c1e25df25c2ed3cfe5da6eefe4f543890a3036","modified":1701167756501},{"_id":"source/images/zkp/plonk/prescribed_perm_check_problem_complete.png","hash":"10bf6b8740b39541a2062928bc4a48d57d4a4d96","modified":1701167850819},{"_id":"source/images/zkp/plonk/prescribed_perm_check_problem_reduce.png","hash":"1abcac63be58b40a4191defa3a7335632ffdc3da","modified":1701167649980},{"_id":"source/images/zkp/plonk/prescribed_perm_check_problem_quadratic.png","hash":"ed9cfcc17dd706abcf381f05578d00c00dab3214","modified":1701167336035},{"_id":"source/images/zkp/plonk/prescribed_perm_check_problem.png","hash":"26b7e37f8ca360fef0504099e7e6a8fb5423fe53","modified":1701167178038},{"_id":"public/images/zkp/plonk/copy_constraint_example.png","hash":"ccace19afe01e3991b688eef27da922f13b69365","modified":1701227621497},{"_id":"public/images/zkp/plonk/prod_check_lemma.png","hash":"e6e29ac8e9eae8cdeff6e8b9909904c612d6ccb4","modified":1701227621497},{"_id":"public/images/zkp/plonk/custom_gate.png","hash":"35c2febb4bd8044f48cf5d42744ccc508503d436","modified":1701227621497},{"_id":"public/images/zkp/plonk/prod_check_prove_verify.png","hash":"e6de2efae2fdbb98cc49d6bae5965ed311412e4a","modified":1701227621497},{"_id":"public/images/zkp/plonk/prescribed_perm_check_problem_prove_verify.png","hash":"58c1e25df25c2ed3cfe5da6eefe4f543890a3036","modified":1701227621497},{"_id":"public/images/zkp/plonk/prescribed_perm_check_problem_complete.png","hash":"10bf6b8740b39541a2062928bc4a48d57d4a4d96","modified":1701227621497},{"_id":"public/images/zkp/plonk/prescribed_perm_check_problem_reduce.png","hash":"1abcac63be58b40a4191defa3a7335632ffdc3da","modified":1701227621497},{"_id":"public/images/zkp/plonk/prescribed_perm_check_problem_quadratic.png","hash":"ed9cfcc17dd706abcf381f05578d00c00dab3214","modified":1701227621497},{"_id":"public/images/zkp/plonk/prescribed_perm_check_problem.png","hash":"26b7e37f8ca360fef0504099e7e6a8fb5423fe53","modified":1701227621497},{"_id":"source/_posts/cryptography/elliptic_curve/bls12-381.md","hash":"be92f8e12ea7890f02772cb2ee19f0b460ed06e7","modified":1701575637439},{"_id":"source/_posts/cryptography/zkp/kzg-commitment.md","hash":"acbf08b806cc9ff247eab5eb676e6b0deff10edd","modified":1702289247075},{"_id":"public/2023/11/05/cryptography/zkp/kzg-commitment/index.html","hash":"d8c8710aec26c17a2b5dfe12e1f6c0bf7c7d685d","modified":1701575782113},{"_id":"public/tags/ec/index.html","hash":"f49c00f9fe13c32261802fb18a11b46205016cb8","modified":1702283024204},{"_id":"public/archives/2023/12/index.html","hash":"fdc18dd671388cb2bd5cd64eee0b9d0ccfde8fd4","modified":1702283024204},{"_id":"public/2023/12/01/cryptography/elliptic_curve/bls12-381/index.html","hash":"ef8d5bc46ca215ec0f308cb8a885c04e43e632e6","modified":1702283024204},{"_id":"public/2023/12/03/cryptography/zkp/kzg-commitment/index.html","hash":"c40ae91dda0c79a493a2e1919a1e7a0c007cf51f","modified":1702352970990},{"_id":"public/2023/12/07/arithmatic/montgomery-multiplication/index.html","hash":"f329b0ab8951d3abb3d8bab03100b4aec5ac7269","modified":1702283024204},{"_id":"source/_posts/cryptography/zkp/polygon_zkevm.md","hash":"54c0dbd616412eb600eea236afaa6340abda0a23","modified":1702283699530},{"_id":"source/_posts/cryptography/zkp/plonky2-code-analysis.md","hash":"fb63c993aaa60cde80d6578c2ea1ea5549898ad3","modified":1702364804541},{"_id":"source/images/zkp/zkevm/polygon/state_machines.png","hash":"d367f14e982243b0db74d15ec1f71cc4c91a6b0d","modified":1702282985940},{"_id":"public/2023/12/11/cryptography/zkp/polygon_zkevm/index.html","hash":"0707191693ef92757d38a27f3b788b5dd6f1b286","modified":1702352970990},{"_id":"public/2023/11/06/cryptography/zkp/plonky2-code-analysis/index.html","hash":"27aede931040d6370f9bfdf462d839999e794db0","modified":1702364965612},{"_id":"public/tags/zkp/page/2/index.html","hash":"8e6da933936be3a044a678518ddf863c4645ec7a","modified":1702283024204},{"_id":"public/images/zkp/zkevm/polygon/state_machines.png","hash":"d367f14e982243b0db74d15ec1f71cc4c91a6b0d","modified":1702283024204},{"_id":"source/images/zkp/zkevm/polygon/micro_processor.png","hash":"0aef9a284fe051e40d79c80d0b79021f34aaadde","modified":1702283676469},{"_id":"public/images/zkp/zkevm/polygon/micro_processor.png","hash":"0aef9a284fe051e40d79c80d0b79021f34aaadde","modified":1702352970990}],"Category":[],"Data":[],"Page":[],"Post":[{"title":"MPT","date":"2023-01-22T09:25:36.000Z","_content":"\n# trie\na trie, also called prefix tree, is a type of k-ary search tree. These keys are most often strings, with links between nodes defined not by the entire key, but by individual characters. In order to access a key, the trie is traversed depth-first, following the links between nodes, which represent each character in the key.\n\n![prefix trie](/images/trie.prefix.png)\nIn general, the nodes of a Trie look like this:\n```\n[ [Ia, Ib, … I*], value]\n```\n[Ia, Ib, ... I*] is the index array of the node, which takes the next character in the key as the index, and each element I* points to the corresponding child node. value represents the value\n\n# MPT\nEach block of Ethereum contains three MPT trees, respectively\n\n- Transaction tree\n- Receipt tree\n- State tree\n\nIn the figure below are two block headers, where state root, tx root receipt root stores the roots of the three trees, and the second block shows when the data of account 175 changes (27 -> 45). Only need to store 3 nodes related to this account, and the data in the old block can still be accessed normally. (This is somewhat similar to the implementation of an immutable data structure in a functional programming language.) The detailed structure is\n\n![state reference](/images/mpt.state.ref.png)\n\n- use []byte as key, other than string\n- nibble: the smallest unit of the key type (4 bit)\n- Use hashes to refer to nodes instead of memory pointers\n\nthere are two types of node: full nodes (fullNode) and short nodes (shortNode). Full nodes have 17 elements, while shortNode nodes have two elements. Their schematic expressions are as follows\n```\nfullNode: [i0, i1, i2, … i15, hash] \nshortNode： [ [k0, k1, … kn], hash ] // first element is an array\n```\nif the hash pointing to a value, it is a leaf node; if pointing another node, a non leaf node. shortNode contains extension and leaf node. full node is branch node.\n\n![mpt](/images/mpt.png)\n\nUse the upper 4 bits of the first byte of the []byte value composed of nibbles as storage flag. The 0th bit stores the parity information, and the 1st bit stores the type represented by the value\n|hex char| bits | pointing to | odd/even | 2nd niddle padding |\n| -----|:----:|:----:|:----:|-------|\n|0| 0000 | node | even | no |\n|1| 0001 | node | odd | yes |\n|2| 0010 | value | even | no |\n|3| 0011 | value | odd | yes |\n\nthis encoding method is only used when accessing the database. After reading into memory, the key is directly stored in []byte type\n\nIn the trie module, there is a `Database` object, which you can understand as a cache layer of the underlying database. In actual use, the Trie object uses the Database as a database. However, the important function of Database is not caching, but the reference counting of node data during caching, and provides Database.Reference and Database.Dereference to reference and dereference a trie node. If the reference count of a node becomes 0, the node will be deleted from memory, so it will not be written to the real database\n\n# reference\n- [github](https://github.com/agiletechvn/go-ethereum-code-analysis/blob/master/trie-analysis.md)\n- [yangzhe's blog](http://yangzhe.me/2019/01/12/ethereum-trie-part-1/)\n- [yangzhe's blod](http://yangzhe.me/2019/01/18/ethereum-trie-part-2/)","source":"_posts/MPT.md","raw":"---\ntitle: MPT\ndate: 2023-01-22 17:25:36\ntags: [blockchain, geth]\n---\n\n# trie\na trie, also called prefix tree, is a type of k-ary search tree. These keys are most often strings, with links between nodes defined not by the entire key, but by individual characters. In order to access a key, the trie is traversed depth-first, following the links between nodes, which represent each character in the key.\n\n![prefix trie](/images/trie.prefix.png)\nIn general, the nodes of a Trie look like this:\n```\n[ [Ia, Ib, … I*], value]\n```\n[Ia, Ib, ... I*] is the index array of the node, which takes the next character in the key as the index, and each element I* points to the corresponding child node. value represents the value\n\n# MPT\nEach block of Ethereum contains three MPT trees, respectively\n\n- Transaction tree\n- Receipt tree\n- State tree\n\nIn the figure below are two block headers, where state root, tx root receipt root stores the roots of the three trees, and the second block shows when the data of account 175 changes (27 -> 45). Only need to store 3 nodes related to this account, and the data in the old block can still be accessed normally. (This is somewhat similar to the implementation of an immutable data structure in a functional programming language.) The detailed structure is\n\n![state reference](/images/mpt.state.ref.png)\n\n- use []byte as key, other than string\n- nibble: the smallest unit of the key type (4 bit)\n- Use hashes to refer to nodes instead of memory pointers\n\nthere are two types of node: full nodes (fullNode) and short nodes (shortNode). Full nodes have 17 elements, while shortNode nodes have two elements. Their schematic expressions are as follows\n```\nfullNode: [i0, i1, i2, … i15, hash] \nshortNode： [ [k0, k1, … kn], hash ] // first element is an array\n```\nif the hash pointing to a value, it is a leaf node; if pointing another node, a non leaf node. shortNode contains extension and leaf node. full node is branch node.\n\n![mpt](/images/mpt.png)\n\nUse the upper 4 bits of the first byte of the []byte value composed of nibbles as storage flag. The 0th bit stores the parity information, and the 1st bit stores the type represented by the value\n|hex char| bits | pointing to | odd/even | 2nd niddle padding |\n| -----|:----:|:----:|:----:|-------|\n|0| 0000 | node | even | no |\n|1| 0001 | node | odd | yes |\n|2| 0010 | value | even | no |\n|3| 0011 | value | odd | yes |\n\nthis encoding method is only used when accessing the database. After reading into memory, the key is directly stored in []byte type\n\nIn the trie module, there is a `Database` object, which you can understand as a cache layer of the underlying database. In actual use, the Trie object uses the Database as a database. However, the important function of Database is not caching, but the reference counting of node data during caching, and provides Database.Reference and Database.Dereference to reference and dereference a trie node. If the reference count of a node becomes 0, the node will be deleted from memory, so it will not be written to the real database\n\n# reference\n- [github](https://github.com/agiletechvn/go-ethereum-code-analysis/blob/master/trie-analysis.md)\n- [yangzhe's blog](http://yangzhe.me/2019/01/12/ethereum-trie-part-1/)\n- [yangzhe's blod](http://yangzhe.me/2019/01/18/ethereum-trie-part-2/)","slug":"MPT","published":1,"updated":"2023-11-05T04:21:13.732Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8dh0000qwsj6ebuelv1","content":"

trie

a trie, also called prefix tree, is a type of k-ary search tree. These keys are most often strings, with links between nodes defined not by the entire key, but by individual characters. In order to access a key, the trie is traversed depth-first, following the links between nodes, which represent each character in the key.

\n

\"prefix
In general, the nodes of a Trie look like this:

\n
1
[ [Ia, Ib, … I*], value]
\n

[Ia, Ib, … I*] is the index array of the node, which takes the next character in the key as the index, and each element I* points to the corresponding child node. value represents the value

\n

MPT

Each block of Ethereum contains three MPT trees, respectively

\n\n

In the figure below are two block headers, where state root, tx root receipt root stores the roots of the three trees, and the second block shows when the data of account 175 changes (27 -> 45). Only need to store 3 nodes related to this account, and the data in the old block can still be accessed normally. (This is somewhat similar to the implementation of an immutable data structure in a functional programming language.) The detailed structure is

\n

\"state

\n\n

there are two types of node: full nodes (fullNode) and short nodes (shortNode). Full nodes have 17 elements, while shortNode nodes have two elements. Their schematic expressions are as follows

\n
1
2
fullNode: [i0, i1, i2, … i15, hash]  
shortNode： [ [k0, k1, … kn], hash ] // first element is an array
\n

if the hash pointing to a value, it is a leaf node; if pointing another node, a non leaf node. shortNode contains extension and leaf node. full node is branch node.

\n

\"mpt\"

\n

Use the upper 4 bits of the first byte of the []byte value composed of nibbles as storage flag. The 0th bit stores the parity information, and the 1st bit stores the type represented by the value

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
hex charbitspointing toodd/even2nd niddle padding
00000nodeevenno
10001nodeoddyes
20010valueevenno
30011valueoddyes
\n

this encoding method is only used when accessing the database. After reading into memory, the key is directly stored in []byte type

\n

In the trie module, there is a Database object, which you can understand as a cache layer of the underlying database. In actual use, the Trie object uses the Database as a database. However, the important function of Database is not caching, but the reference counting of node data during caching, and provides Database.Reference and Database.Dereference to reference and dereference a trie node. If the reference count of a node becomes 0, the node will be deleted from memory, so it will not be written to the real database

\n

reference

\n","site":{"data":{}},"excerpt":"","more":"

trie

a trie, also called prefix tree, is a type of k-ary search tree. These keys are most often strings, with links between nodes defined not by the entire key, but by individual characters. In order to access a key, the trie is traversed depth-first, following the links between nodes, which represent each character in the key.

\n

\"prefix
In general, the nodes of a Trie look like this:

\n
1
[ [Ia, Ib, … I*], value]
\n

[Ia, Ib, … I*] is the index array of the node, which takes the next character in the key as the index, and each element I* points to the corresponding child node. value represents the value

\n

MPT

Each block of Ethereum contains three MPT trees, respectively

\n\n

In the figure below are two block headers, where state root, tx root receipt root stores the roots of the three trees, and the second block shows when the data of account 175 changes (27 -> 45). Only need to store 3 nodes related to this account, and the data in the old block can still be accessed normally. (This is somewhat similar to the implementation of an immutable data structure in a functional programming language.) The detailed structure is

\n

\"state

\n\n

there are two types of node: full nodes (fullNode) and short nodes (shortNode). Full nodes have 17 elements, while shortNode nodes have two elements. Their schematic expressions are as follows

\n
1
2
fullNode: [i0, i1, i2, … i15, hash]  
shortNode： [ [k0, k1, … kn], hash ] // first element is an array
\n

if the hash pointing to a value, it is a leaf node; if pointing another node, a non leaf node. shortNode contains extension and leaf node. full node is branch node.

\n

\"mpt\"

\n

Use the upper 4 bits of the first byte of the []byte value composed of nibbles as storage flag. The 0th bit stores the parity information, and the 1st bit stores the type represented by the value

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
hex charbitspointing toodd/even2nd niddle padding
00000nodeevenno
10001nodeoddyes
20010valueevenno
30011valueoddyes
\n

this encoding method is only used when accessing the database. After reading into memory, the key is directly stored in []byte type

\n

In the trie module, there is a Database object, which you can understand as a cache layer of the underlying database. In actual use, the Trie object uses the Database as a database. However, the important function of Database is not caching, but the reference counting of node data during caching, and provides Database.Reference and Database.Dereference to reference and dereference a trie node. If the reference count of a node becomes 0, the node will be deleted from memory, so it will not be written to the real database

\n

reference

\n"},{"title":"understanding of kademlia protocol","date":"2023-01-15T03:00:30.000Z","_content":"\n# Introduction\nKademlia, a peer-to-peer distributed hash table(DHT). Some other DHT techniques are Chord. In the Kad network, each node has a unique 160-bit ID (ETH account address is 20 bytes, which is 160 bits) pairs are stored in nodes whose ID is 'close' to the key for some notion of closeness. \n\n# system description\nKad effectively treats nodes as leaves in a binary tree, with each nodes's position determined by the shortest unique prefix of its ID. Figure 1 shows the position of a node with unique prefix 0011 in an example.\n\n![Figure 1](/images/kademlia.subtree.png)\nfor any given node, the binary tree is divided into a series of successively lower subtrees that don't contain the node.\n The highest-level subtree is composed of the other half of the whole tree that does not contain itself; the next level of subtree is composed of the remaining half that does not contain itself; and so on, until the complete tree is split into n subtrees. As shown in the figure, the part contained by the dotted line is the subtree of node 0011.\n\nif there is at least one node knwon in each subtree (in total, at least n nodes), a recursive routing algorithm can be used to reach any node within the binary tree. Figure 2 shows an example of node 0011 locating node 1110 by successively querying the best node it knows of to find contacts in lower and lower subtrees; finaly the lookup converges to the target node.\n\n![Figure 2](/images/kademlia.locating.png)\n\n## node distance: XOR metric\nNode's id is 160 bit. Keys are also 160 bit. The Kad algorithm uses an XOR operation to calculate the distance between nodes.\nd(x,y) = x XOR y\n\n## node state\nfor each 0 <= i < 160, every node keeps k triples (as a list) for nodes of distance between 2^i and 2^i+1 from itself. it is called k-buckets.\neach k-bucket is kept sorted by time last seen--least recently seen node at the head, most-recently seen at the tail.\nfor small values of i, the k-buckets will generally be empty (as no approriate nodes will exist).\n\n The K value mentioned here is a system-level constant (must be an even number). It is set by the software system using Kad (for example, the Kad network used by BT download, K is set to 8). \n\n## update of k bucket\nThere are mainly the following ways to update the K bucket:\n\n- Actively collect nodes: Any node can initiate a FIND_NODE (query node) request to refresh the node information in the K-bucket.\n- Passive collection node: When receiving requests from other nodes (such as: FIND_NODE, FIND_VALUE), it will add the node ID of the other party to a certain K-bucket.\n- Detect invalid nodes: By initiating a PING request, determine whether a node in the K-bucket is online, and then clean up which offline nodes in the K-bucket.\n\nThe update principle is to insert the latest node into the tail of the queue, and not to remove the online nodes in the queue.\n\nAccording to the K-bucket update principle, nodes with a long online time are more likely to remain in the K-bucket list. Therefore, by keeping the nodes with a long online time in the K-bucket, Kad will significantly increase the number of nodes in the K-bucket. it can defend against DOS attacks to a certain extent, because only when the old node fails, Kad will update the K bucket information, which avoids flooding routing information through the addition of new nodes\n![probability of continuous online agains onlie duration](/images/kademlia.onlineprob.png)\n\n## RPC method\nThe Kademlia protocol includes four remote RPC operations: PING, STORE, FIND_NODE, FIND_VALUE.\n\n- PING probes a node to see if it is online.\n- STORE instructs a node to store a pair for later retrieval.\n- FIND_NODE takes a 160bit ID as an argument. The receiver of this operation returns the (IP address, UDP port, Node ID) information of K nodes that it knows are closer to the target ID. The information of these nodes can be obtained from a single K-bucket, or from multiple K-buckets. In either case, the receiver will return the information of K nodes to the operation initiator. \n- The FIND_VALUE operation is similar to the FIND_NODE operation, with one exception. if the RPC receipients has received a STORE RPC for the key, it just returns the stored value.\n\n## node lookup\nKad participant must locate the k closest nodes to some given node ID. Kad employs a recursive algorithm for node lookups. It recursively send FIND_NODE requests to \\alpha (out of k) closest nodes it knows of. \n\n## find a pair\nto find a pair, a node starts by performing a lookup to find the k nodes with IDs closest to the key. However, value lookups use FIND_VLAUE rather than FIND_NODE RPCs.\n\n## node join\nto join the network, a node u must have a contact to an already participating node w.\n\n## routing table\n\n# references\n- [kademlia paper](https://pdos.csail.mit.edu/~petar/papers/maymounkov-kademlia-lncs.pdf)\n- [go implementation](https://github.com/libp2p/go-libp2p-kad-dht)\n- [kademlia stanford](https://codethechange.stanford.edu/guides/guide_kademlia.html)\n- [zhihu](https://zhuanlan.zhihu.com/p/388994038)","source":"_posts/blockchain.kademlia.md","raw":"---\ntitle: understanding of kademlia protocol\ndate: 2023-01-15 11:00:30\ntags: [blockchain]\n---\n\n# Introduction\nKademlia, a peer-to-peer distributed hash table(DHT). Some other DHT techniques are Chord. In the Kad network, each node has a unique 160-bit ID (ETH account address is 20 bytes, which is 160 bits) pairs are stored in nodes whose ID is 'close' to the key for some notion of closeness. \n\n# system description\nKad effectively treats nodes as leaves in a binary tree, with each nodes's position determined by the shortest unique prefix of its ID. Figure 1 shows the position of a node with unique prefix 0011 in an example.\n\n![Figure 1](/images/kademlia.subtree.png)\nfor any given node, the binary tree is divided into a series of successively lower subtrees that don't contain the node.\n The highest-level subtree is composed of the other half of the whole tree that does not contain itself; the next level of subtree is composed of the remaining half that does not contain itself; and so on, until the complete tree is split into n subtrees. As shown in the figure, the part contained by the dotted line is the subtree of node 0011.\n\nif there is at least one node knwon in each subtree (in total, at least n nodes), a recursive routing algorithm can be used to reach any node within the binary tree. Figure 2 shows an example of node 0011 locating node 1110 by successively querying the best node it knows of to find contacts in lower and lower subtrees; finaly the lookup converges to the target node.\n\n![Figure 2](/images/kademlia.locating.png)\n\n## node distance: XOR metric\nNode's id is 160 bit. Keys are also 160 bit. The Kad algorithm uses an XOR operation to calculate the distance between nodes.\nd(x,y) = x XOR y\n\n## node state\nfor each 0 <= i < 160, every node keeps k triples (as a list) for nodes of distance between 2^i and 2^i+1 from itself. it is called k-buckets.\neach k-bucket is kept sorted by time last seen--least recently seen node at the head, most-recently seen at the tail.\nfor small values of i, the k-buckets will generally be empty (as no approriate nodes will exist).\n\n The K value mentioned here is a system-level constant (must be an even number). It is set by the software system using Kad (for example, the Kad network used by BT download, K is set to 8). \n\n## update of k bucket\nThere are mainly the following ways to update the K bucket:\n\n- Actively collect nodes: Any node can initiate a FIND_NODE (query node) request to refresh the node information in the K-bucket.\n- Passive collection node: When receiving requests from other nodes (such as: FIND_NODE, FIND_VALUE), it will add the node ID of the other party to a certain K-bucket.\n- Detect invalid nodes: By initiating a PING request, determine whether a node in the K-bucket is online, and then clean up which offline nodes in the K-bucket.\n\nThe update principle is to insert the latest node into the tail of the queue, and not to remove the online nodes in the queue.\n\nAccording to the K-bucket update principle, nodes with a long online time are more likely to remain in the K-bucket list. Therefore, by keeping the nodes with a long online time in the K-bucket, Kad will significantly increase the number of nodes in the K-bucket. it can defend against DOS attacks to a certain extent, because only when the old node fails, Kad will update the K bucket information, which avoids flooding routing information through the addition of new nodes\n![probability of continuous online agains onlie duration](/images/kademlia.onlineprob.png)\n\n## RPC method\nThe Kademlia protocol includes four remote RPC operations: PING, STORE, FIND_NODE, FIND_VALUE.\n\n- PING probes a node to see if it is online.\n- STORE instructs a node to store a pair for later retrieval.\n- FIND_NODE takes a 160bit ID as an argument. The receiver of this operation returns the (IP address, UDP port, Node ID) information of K nodes that it knows are closer to the target ID. The information of these nodes can be obtained from a single K-bucket, or from multiple K-buckets. In either case, the receiver will return the information of K nodes to the operation initiator. \n- The FIND_VALUE operation is similar to the FIND_NODE operation, with one exception. if the RPC receipients has received a STORE RPC for the key, it just returns the stored value.\n\n## node lookup\nKad participant must locate the k closest nodes to some given node ID. Kad employs a recursive algorithm for node lookups. It recursively send FIND_NODE requests to \\alpha (out of k) closest nodes it knows of. \n\n## find a pair\nto find a pair, a node starts by performing a lookup to find the k nodes with IDs closest to the key. However, value lookups use FIND_VLAUE rather than FIND_NODE RPCs.\n\n## node join\nto join the network, a node u must have a contact to an already participating node w.\n\n## routing table\n\n# references\n- [kademlia paper](https://pdos.csail.mit.edu/~petar/papers/maymounkov-kademlia-lncs.pdf)\n- [go implementation](https://github.com/libp2p/go-libp2p-kad-dht)\n- [kademlia stanford](https://codethechange.stanford.edu/guides/guide_kademlia.html)\n- [zhihu](https://zhuanlan.zhihu.com/p/388994038)","slug":"blockchain.kademlia","published":1,"updated":"2023-11-05T04:21:13.732Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8dj0001qwsj9waw412a","content":"

Introduction

Kademlia, a peer-to-peer distributed hash table(DHT). Some other DHT techniques are Chord. In the Kad network, each node has a unique 160-bit ID (ETH account address is 20 bytes, which is 160 bits) <key,value> pairs are stored in nodes whose ID is ‘close’ to the key for some notion of closeness.

\n

system description

Kad effectively treats nodes as leaves in a binary tree, with each nodes’s position determined by the shortest unique prefix of its ID. Figure 1 shows the position of a node with unique prefix 0011 in an example.

\n

\"Figure
for any given node, the binary tree is divided into a series of successively lower subtrees that don’t contain the node.
The highest-level subtree is composed of the other half of the whole tree that does not contain itself; the next level of subtree is composed of the remaining half that does not contain itself; and so on, until the complete tree is split into n subtrees. As shown in the figure, the part contained by the dotted line is the subtree of node 0011.

\n

if there is at least one node knwon in each subtree (in total, at least n nodes), a recursive routing algorithm can be used to reach any node within the binary tree. Figure 2 shows an example of node 0011 locating node 1110 by successively querying the best node it knows of to find contacts in lower and lower subtrees; finaly the lookup converges to the target node.

\n

\"Figure

\n

node distance: XOR metric

Node’s id is 160 bit. Keys are also 160 bit. The Kad algorithm uses an XOR operation to calculate the distance between nodes.
d(x,y) = x XOR y

\n

node state

for each 0 <= i < 160, every node keeps k <IP address, UDP port, node ID> triples (as a list) for nodes of distance between 2^i and 2^i+1 from itself. it is called k-buckets.
each k-bucket is kept sorted by time last seen–least recently seen node at the head, most-recently seen at the tail.
for small values of i, the k-buckets will generally be empty (as no approriate nodes will exist).

\n

The K value mentioned here is a system-level constant (must be an even number). It is set by the software system using Kad (for example, the Kad network used by BT download, K is set to 8).

\n

update of k bucket

There are mainly the following ways to update the K bucket:

\n
    \n
  • Actively collect nodes: Any node can initiate a FIND_NODE (query node) request to refresh the node information in the K-bucket.
    • \n
  • Passive collection node: When receiving requests from other nodes (such as: FIND_NODE, FIND_VALUE), it will add the node ID of the other party to a certain K-bucket.
    • \n
  • Detect invalid nodes: By initiating a PING request, determine whether a node in the K-bucket is online, and then clean up which offline nodes in the K-bucket.
    • \n
\n

The update principle is to insert the latest node into the tail of the queue, and not to remove the online nodes in the queue.

\n

According to the K-bucket update principle, nodes with a long online time are more likely to remain in the K-bucket list. Therefore, by keeping the nodes with a long online time in the K-bucket, Kad will significantly increase the number of nodes in the K-bucket. it can defend against DOS attacks to a certain extent, because only when the old node fails, Kad will update the K bucket information, which avoids flooding routing information through the addition of new nodes
\"probability

\n

RPC method

The Kademlia protocol includes four remote RPC operations: PING, STORE, FIND_NODE, FIND_VALUE.

\n
    \n
  • PING probes a node to see if it is online.
    • \n
  • STORE instructs a node to store a <key,value> pair for later retrieval.
    • \n
  • FIND_NODE takes a 160bit ID as an argument. The receiver of this operation returns the (IP address, UDP port, Node ID) information of K nodes that it knows are closer to the target ID. The information of these nodes can be obtained from a single K-bucket, or from multiple K-buckets. In either case, the receiver will return the information of K nodes to the operation initiator.
    • \n
  • The FIND_VALUE operation is similar to the FIND_NODE operation, with one exception. if the RPC receipients has received a STORE RPC for the key, it just returns the stored value.
    • \n
\n

node lookup

Kad participant must locate the k closest nodes to some given node ID. Kad employs a recursive algorithm for node lookups. It recursively send FIND_NODE requests to \\alpha (out of k) closest nodes it knows of.

\n

find a <key,value> pair

to find a <key,value> pair, a node starts by performing a lookup to find the k nodes with IDs closest to the key. However, value lookups use FIND_VLAUE rather than FIND_NODE RPCs.

\n

node join

to join the network, a node u must have a contact to an already participating node w.

\n

routing table

references

\n","site":{"data":{}},"excerpt":"","more":"

Introduction

Kademlia, a peer-to-peer distributed hash table(DHT). Some other DHT techniques are Chord. In the Kad network, each node has a unique 160-bit ID (ETH account address is 20 bytes, which is 160 bits) <key,value> pairs are stored in nodes whose ID is ‘close’ to the key for some notion of closeness.

\n

system description

Kad effectively treats nodes as leaves in a binary tree, with each nodes’s position determined by the shortest unique prefix of its ID. Figure 1 shows the position of a node with unique prefix 0011 in an example.

\n

\"Figure
for any given node, the binary tree is divided into a series of successively lower subtrees that don’t contain the node.
The highest-level subtree is composed of the other half of the whole tree that does not contain itself; the next level of subtree is composed of the remaining half that does not contain itself; and so on, until the complete tree is split into n subtrees. As shown in the figure, the part contained by the dotted line is the subtree of node 0011.

\n

if there is at least one node knwon in each subtree (in total, at least n nodes), a recursive routing algorithm can be used to reach any node within the binary tree. Figure 2 shows an example of node 0011 locating node 1110 by successively querying the best node it knows of to find contacts in lower and lower subtrees; finaly the lookup converges to the target node.

\n

\"Figure

\n

node distance: XOR metric

Node’s id is 160 bit. Keys are also 160 bit. The Kad algorithm uses an XOR operation to calculate the distance between nodes.
d(x,y) = x XOR y

\n

node state

for each 0 <= i < 160, every node keeps k <IP address, UDP port, node ID> triples (as a list) for nodes of distance between 2^i and 2^i+1 from itself. it is called k-buckets.
each k-bucket is kept sorted by time last seen–least recently seen node at the head, most-recently seen at the tail.
for small values of i, the k-buckets will generally be empty (as no approriate nodes will exist).

\n

The K value mentioned here is a system-level constant (must be an even number). It is set by the software system using Kad (for example, the Kad network used by BT download, K is set to 8).

\n

update of k bucket

There are mainly the following ways to update the K bucket:

\n
    \n
  • Actively collect nodes: Any node can initiate a FIND_NODE (query node) request to refresh the node information in the K-bucket.
    • \n
  • Passive collection node: When receiving requests from other nodes (such as: FIND_NODE, FIND_VALUE), it will add the node ID of the other party to a certain K-bucket.
    • \n
  • Detect invalid nodes: By initiating a PING request, determine whether a node in the K-bucket is online, and then clean up which offline nodes in the K-bucket.
    • \n
\n

The update principle is to insert the latest node into the tail of the queue, and not to remove the online nodes in the queue.

\n

According to the K-bucket update principle, nodes with a long online time are more likely to remain in the K-bucket list. Therefore, by keeping the nodes with a long online time in the K-bucket, Kad will significantly increase the number of nodes in the K-bucket. it can defend against DOS attacks to a certain extent, because only when the old node fails, Kad will update the K bucket information, which avoids flooding routing information through the addition of new nodes
\"probability

\n

RPC method

The Kademlia protocol includes four remote RPC operations: PING, STORE, FIND_NODE, FIND_VALUE.

\n
    \n
  • PING probes a node to see if it is online.
    • \n
  • STORE instructs a node to store a <key,value> pair for later retrieval.
    • \n
  • FIND_NODE takes a 160bit ID as an argument. The receiver of this operation returns the (IP address, UDP port, Node ID) information of K nodes that it knows are closer to the target ID. The information of these nodes can be obtained from a single K-bucket, or from multiple K-buckets. In either case, the receiver will return the information of K nodes to the operation initiator.
    • \n
  • The FIND_VALUE operation is similar to the FIND_NODE operation, with one exception. if the RPC receipients has received a STORE RPC for the key, it just returns the stored value.
    • \n
\n

node lookup

Kad participant must locate the k closest nodes to some given node ID. Kad employs a recursive algorithm for node lookups. It recursively send FIND_NODE requests to \\alpha (out of k) closest nodes it knows of.

\n

find a <key,value> pair

to find a <key,value> pair, a node starts by performing a lookup to find the k nodes with IDs closest to the key. However, value lookups use FIND_VLAUE rather than FIND_NODE RPCs.

\n

node join

to join the network, a node u must have a contact to an already participating node w.

\n

routing table

references

\n"},{"title":"geth_fine_tune","date":"2023-01-01T08:29:43.000Z","_content":"\n\nIf we're a full node on mainnet without --cache specified, bump default cache allowance\nctx.Set(utils.CacheFlag.Name, strconv.Itoa(4096))","source":"_posts/geth-fine-tune.md","raw":"---\ntitle: geth_fine_tune\ndate: 2023-01-01 16:29:43\ntags:\n---\n\n\nIf we're a full node on mainnet without --cache specified, bump default cache allowance\nctx.Set(utils.CacheFlag.Name, strconv.Itoa(4096))","slug":"geth-fine-tune","published":1,"updated":"2023-11-05T04:21:13.733Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8dl0003qwsj0la65sux","content":"

If we’re a full node on mainnet without –cache specified, bump default cache allowance
ctx.Set(utils.CacheFlag.Name, strconv.Itoa(4096))

\n","site":{"data":{}},"excerpt":"","more":"

If we’re a full node on mainnet without –cache specified, bump default cache allowance
ctx.Set(utils.CacheFlag.Name, strconv.Itoa(4096))

\n"},{"title":"how to use hexo","date":"2022-11-27T03:47:56.000Z","_content":"Welcome to [Hexo](https://hexo.io/)! This is your very first post. Check [documentation](https://hexo.io/docs/) for more info. If you get any problems when using Hexo, you can find the answer in [troubleshooting](https://hexo.io/docs/troubleshooting.html) or you can ask me on [GitHub](https://github.com/hexojs/hexo/issues).\n\n## Quick Start\n\n### Create a new post\n\n``` bash\n$ hexo new \"My New Post\"\n```\n\nMore info: [Writing](https://hexo.io/docs/writing.html)\n\n### Run server\n\n``` bash\n$ hexo server\n```\n\nMore info: [Server](https://hexo.io/docs/server.html)\n\n### Generate static files\n\n``` bash\n$ hexo generate\n```\n\nMore info: [Generating](https://hexo.io/docs/generating.html)\n\n### Deploy to remote sites\n\n``` bash\n$ hexo deploy\n```\n\nMore info: [Deployment](https://hexo.io/docs/one-command-deployment.html)\n","source":"_posts/hello-world.md","raw":"---\ntitle: how to use hexo\ndate: 2022-11-27 11:47:56\n---\nWelcome to [Hexo](https://hexo.io/)! This is your very first post. Check [documentation](https://hexo.io/docs/) for more info. If you get any problems when using Hexo, you can find the answer in [troubleshooting](https://hexo.io/docs/troubleshooting.html) or you can ask me on [GitHub](https://github.com/hexojs/hexo/issues).\n\n## Quick Start\n\n### Create a new post\n\n``` bash\n$ hexo new \"My New Post\"\n```\n\nMore info: [Writing](https://hexo.io/docs/writing.html)\n\n### Run server\n\n``` bash\n$ hexo server\n```\n\nMore info: [Server](https://hexo.io/docs/server.html)\n\n### Generate static files\n\n``` bash\n$ hexo generate\n```\n\nMore info: [Generating](https://hexo.io/docs/generating.html)\n\n### Deploy to remote sites\n\n``` bash\n$ hexo deploy\n```\n\nMore info: [Deployment](https://hexo.io/docs/one-command-deployment.html)\n","slug":"hello-world","published":1,"updated":"2023-11-05T04:21:13.733Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8dl0004qwsj06vjfxz2","content":"

Welcome to Hexo! This is your very first post. Check documentation for more info. If you get any problems when using Hexo, you can find the answer in troubleshooting or you can ask me on GitHub.

\n

Quick Start

Create a new post

1
$ hexo new "My New Post"
\n\n

More info: Writing

\n

Run server

1
$ hexo server
\n\n

More info: Server

\n

Generate static files

1
$ hexo generate
\n\n

More info: Generating

\n

Deploy to remote sites

1
$ hexo deploy
\n\n

More info: Deployment

\n","site":{"data":{}},"excerpt":"","more":"

Welcome to Hexo! This is your very first post. Check documentation for more info. If you get any problems when using Hexo, you can find the answer in troubleshooting or you can ask me on GitHub.

\n

Quick Start

Create a new post

1
$ hexo new "My New Post"
\n\n

More info: Writing

\n

Run server

1
$ hexo server
\n\n

More info: Server

\n

Generate static files

1
$ hexo generate
\n\n

More info: Generating

\n

Deploy to remote sites

1
$ hexo deploy
\n\n

More info: Deployment

\n"},{"title":"danksharding","date":"2023-10-12T06:36:58.000Z","_content":"\n\n\n\n## introduction\nDanksharding is central to Ethereum’s rollup-centric roadmap. The idea is to provide space for large blobs of data, which are verifiable and available, without attempting to interpret them.\n\n\n## Preliminaries\nAll references to EC, in this report, refer to BLS12_381 [1]. We are mainly concerned with the EC Abelian-group \\\\(\\mathbb{G_{1}}\\\\) and the EC scalar-field \\\\(F_r\\\\). Both are of size \\\\(r < 2256\\\\). Note that \\\\(2^{32}\\\\) divides the size of the multiplicative-group in \\\\(F_r\\\\)\n\n## Data organization\nThe input data consists of n = 256 shard-blobs. Each shard-blob is a vector of m = 4096 field-elements referred-to as symbols. The data symbols are organized in an n × m input\nmatrix\n\\\\[\n \\tag{1}\n D_{n x m}^{in} =\n\\begin{bmatrix}\n\\displaylines{\n d(0,0) & d(0,1) & \\dots & d(0,m-1)\\\\\\\\\n d(1,0) & d(1,1) & \\dots & d(1,m-1)\\\\\\\\\n \\vdots & \\vdots & \\ddots & \\vdots\\\\\\\\\n d(n-1,0) & d(n-1,1) & \\dots & d(n-1,m-1)\n}\n\\end{bmatrix}\n\\\\]\nwhere each row is one of the shard-blob vectors [2]\nIn order to enable interpolation and polynomial-commitment to the data, we will pro-\nceed to treat the data symbols as polynomial evaluations.\n\nLet us thus associate each domain location in the input matrix with a field-element pair \\\\(u_{\\mu}, \\omega_{\\eta}\\\\), where \\\\(\\mu \\in [0, n−1], \\eta \\in [0, m−1]\\\\) correspond to the row and column indexes, respectively.The row field-element is defined as \\\\( u_{\\mu} \\equiv u^{rbo(\\mu)}\\\\), where u is a 2n’th root-of-unity such that \\\\(u^{2n} = 1\\\\). The column field-element is defined as \\\\( \\omega_{\\eta} \\equiv \\omega^{rbo(\\eta)}\\\\), where \\\\(\\omega\\\\) is a 2m’th root-of-unity such that \\\\(\\omega^{2m} = 1\\\\). _Using reverse-bit-order ordering rather than natural-ordering allows accessing cosets in block (consecutive) rather than interleaved manner_\n\n## coefficients extraction\nTaking the data symbols to be evaluations of a 2D-polynomial or 1D-product-polynomials with row degree n−1 and column degree m−1 uniquely defines the polynomials’ coefficients.\n\n### 2D coeficients extraction\nThe 2D-polynomial representing the input data can be expressed as\n\\\\[\\tag{2} d(x,y) \\equiv \\sum_{i=0}^{n-1}\\sum_{j=0}^{m-1} \\hat{c}[i,j] x^{i}y^{j}\\\\]\nPlugging an evaluation from (1) into (2) results in the following:\n\n\\\\[\\tag{3} d(u_{\\mu},\\omega_{\\eta}) \\equiv \\sum_{i=0}^{n-1}\\sum_{j=0}^{m-1} \\hat{c}[i,j] {u_{\\mu}}^{i}{\\omega_{\\eta}}^{j}\\\\]\n## references\n[1] Ben Edgington. Bls12-381 for the rest of us. https://hackmd.io/@benjaminion/bls12-381.\n[2] Dankrad Feist. Data availability encoding. https://notes.ethereum.org/@dankrad/danksharding_encoding","source":"_posts/blockchain/danksharding.md","raw":"---\ntitle: danksharding\ndate: 2023-10-12 14:36:58\ntags: [blockchain]\n---\n\n\n\n\n## introduction\nDanksharding is central to Ethereum’s rollup-centric roadmap. The idea is to provide space for large blobs of data, which are verifiable and available, without attempting to interpret them.\n\n\n## Preliminaries\nAll references to EC, in this report, refer to BLS12_381 [1]. We are mainly concerned with the EC Abelian-group \\\\(\\mathbb{G_{1}}\\\\) and the EC scalar-field \\\\(F_r\\\\). Both are of size \\\\(r < 2256\\\\). Note that \\\\(2^{32}\\\\) divides the size of the multiplicative-group in \\\\(F_r\\\\)\n\n## Data organization\nThe input data consists of n = 256 shard-blobs. Each shard-blob is a vector of m = 4096 field-elements referred-to as symbols. The data symbols are organized in an n × m input\nmatrix\n\\\\[\n \\tag{1}\n D_{n x m}^{in} =\n\\begin{bmatrix}\n\\displaylines{\n d(0,0) & d(0,1) & \\dots & d(0,m-1)\\\\\\\\\n d(1,0) & d(1,1) & \\dots & d(1,m-1)\\\\\\\\\n \\vdots & \\vdots & \\ddots & \\vdots\\\\\\\\\n d(n-1,0) & d(n-1,1) & \\dots & d(n-1,m-1)\n}\n\\end{bmatrix}\n\\\\]\nwhere each row is one of the shard-blob vectors [2]\nIn order to enable interpolation and polynomial-commitment to the data, we will pro-\nceed to treat the data symbols as polynomial evaluations.\n\nLet us thus associate each domain location in the input matrix with a field-element pair \\\\(u_{\\mu}, \\omega_{\\eta}\\\\), where \\\\(\\mu \\in [0, n−1], \\eta \\in [0, m−1]\\\\) correspond to the row and column indexes, respectively.The row field-element is defined as \\\\( u_{\\mu} \\equiv u^{rbo(\\mu)}\\\\), where u is a 2n’th root-of-unity such that \\\\(u^{2n} = 1\\\\). The column field-element is defined as \\\\( \\omega_{\\eta} \\equiv \\omega^{rbo(\\eta)}\\\\), where \\\\(\\omega\\\\) is a 2m’th root-of-unity such that \\\\(\\omega^{2m} = 1\\\\). _Using reverse-bit-order ordering rather than natural-ordering allows accessing cosets in block (consecutive) rather than interleaved manner_\n\n## coefficients extraction\nTaking the data symbols to be evaluations of a 2D-polynomial or 1D-product-polynomials with row degree n−1 and column degree m−1 uniquely defines the polynomials’ coefficients.\n\n### 2D coeficients extraction\nThe 2D-polynomial representing the input data can be expressed as\n\\\\[\\tag{2} d(x,y) \\equiv \\sum_{i=0}^{n-1}\\sum_{j=0}^{m-1} \\hat{c}[i,j] x^{i}y^{j}\\\\]\nPlugging an evaluation from (1) into (2) results in the following:\n\n\\\\[\\tag{3} d(u_{\\mu},\\omega_{\\eta}) \\equiv \\sum_{i=0}^{n-1}\\sum_{j=0}^{m-1} \\hat{c}[i,j] {u_{\\mu}}^{i}{\\omega_{\\eta}}^{j}\\\\]\n## references\n[1] Ben Edgington. Bls12-381 for the rest of us. https://hackmd.io/@benjaminion/bls12-381.\n[2] Dankrad Feist. Data availability encoding. https://notes.ethereum.org/@dankrad/danksharding_encoding","slug":"blockchain/danksharding","published":1,"updated":"2023-11-05T04:21:17.033Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8dm0005qwsja95zaawe","content":"\n\n\n

introduction

Danksharding is central to Ethereum’s rollup-centric roadmap. The idea is to provide space for large blobs of data, which are verifiable and available, without attempting to interpret them.

\n

Preliminaries

All references to EC, in this report, refer to BLS12_381 [1]. We are mainly concerned with the EC Abelian-group \\(\\mathbb{G_{1}}\\) and the EC scalar-field \\(F_r\\). Both are of size \\(r < 2256\\). Note that \\(2^{32}\\) divides the size of the multiplicative-group in \\(F_r\\)

\n

Data organization

The input data consists of n = 256 shard-blobs. Each shard-blob is a vector of m = 4096 field-elements referred-to as symbols. The data symbols are organized in an n × m input
matrix
\\[
\\tag{1}
D_{n x m}^{in} =
\\begin{bmatrix}
\\displaylines{
d(0,0) & d(0,1) & \\dots & d(0,m-1)\\\\
d(1,0) & d(1,1) & \\dots & d(1,m-1)\\\\
\\vdots & \\vdots & \\ddots & \\vdots\\\\
d(n-1,0) & d(n-1,1) & \\dots & d(n-1,m-1)
}
\\end{bmatrix}
\\]
where each row is one of the shard-blob vectors [2]
In order to enable interpolation and polynomial-commitment to the data, we will pro-
ceed to treat the data symbols as polynomial evaluations.

\n

Let us thus associate each domain location in the input matrix with a field-element pair \\(u_{\\mu}, \\omega_{\\eta}\\), where \\(\\mu \\in [0, n−1], \\eta \\in [0, m−1]\\) correspond to the row and column indexes, respectively.The row field-element is defined as \\( u_{\\mu} \\equiv u^{rbo(\\mu)}\\), where u is a 2n’th root-of-unity such that \\(u^{2n} = 1\\). The column field-element is defined as \\( \\omega_{\\eta} \\equiv \\omega^{rbo(\\eta)}\\), where \\(\\omega\\) is a 2m’th root-of-unity such that \\(\\omega^{2m} = 1\\). Using reverse-bit-order ordering rather than natural-ordering allows accessing cosets in block (consecutive) rather than interleaved manner

\n

coefficients extraction

Taking the data symbols to be evaluations of a 2D-polynomial or 1D-product-polynomials with row degree n−1 and column degree m−1 uniquely defines the polynomials’ coefficients.

\n

2D coeficients extraction

The 2D-polynomial representing the input data can be expressed as
\\[\\tag{2} d(x,y) \\equiv \\sum_{i=0}^{n-1}\\sum_{j=0}^{m-1} \\hat{c}[i,j] x^{i}y^{j}\\]
Plugging an evaluation from (1) into (2) results in the following:

\n

\\[\\tag{3} d(u_{\\mu},\\omega_{\\eta}) \\equiv \\sum_{i=0}^{n-1}\\sum_{j=0}^{m-1} \\hat{c}[i,j] {u_{\\mu}}^{i}{\\omega_{\\eta}}^{j}\\]

\n

references

[1] Ben Edgington. Bls12-381 for the rest of us. https://hackmd.io/@benjaminion/bls12-381.
[2] Dankrad Feist. Data availability encoding. https://notes.ethereum.org/@dankrad/danksharding_encoding

\n","site":{"data":{}},"excerpt":"","more":"\n\n\n

introduction

Danksharding is central to Ethereum’s rollup-centric roadmap. The idea is to provide space for large blobs of data, which are verifiable and available, without attempting to interpret them.

\n

Preliminaries

All references to EC, in this report, refer to BLS12_381 [1]. We are mainly concerned with the EC Abelian-group \\(\\mathbb{G_{1}}\\) and the EC scalar-field \\(F_r\\). Both are of size \\(r < 2256\\). Note that \\(2^{32}\\) divides the size of the multiplicative-group in \\(F_r\\)

\n

Data organization

The input data consists of n = 256 shard-blobs. Each shard-blob is a vector of m = 4096 field-elements referred-to as symbols. The data symbols are organized in an n × m input
matrix
\\[
\\tag{1}
D_{n x m}^{in} =
\\begin{bmatrix}
\\displaylines{
d(0,0) & d(0,1) & \\dots & d(0,m-1)\\\\
d(1,0) & d(1,1) & \\dots & d(1,m-1)\\\\
\\vdots & \\vdots & \\ddots & \\vdots\\\\
d(n-1,0) & d(n-1,1) & \\dots & d(n-1,m-1)
}
\\end{bmatrix}
\\]
where each row is one of the shard-blob vectors [2]
In order to enable interpolation and polynomial-commitment to the data, we will pro-
ceed to treat the data symbols as polynomial evaluations.

\n

Let us thus associate each domain location in the input matrix with a field-element pair \\(u_{\\mu}, \\omega_{\\eta}\\), where \\(\\mu \\in [0, n−1], \\eta \\in [0, m−1]\\) correspond to the row and column indexes, respectively.The row field-element is defined as \\( u_{\\mu} \\equiv u^{rbo(\\mu)}\\), where u is a 2n’th root-of-unity such that \\(u^{2n} = 1\\). The column field-element is defined as \\( \\omega_{\\eta} \\equiv \\omega^{rbo(\\eta)}\\), where \\(\\omega\\) is a 2m’th root-of-unity such that \\(\\omega^{2m} = 1\\). Using reverse-bit-order ordering rather than natural-ordering allows accessing cosets in block (consecutive) rather than interleaved manner

\n

coefficients extraction

Taking the data symbols to be evaluations of a 2D-polynomial or 1D-product-polynomials with row degree n−1 and column degree m−1 uniquely defines the polynomials’ coefficients.

\n

2D coeficients extraction

The 2D-polynomial representing the input data can be expressed as
\\[\\tag{2} d(x,y) \\equiv \\sum_{i=0}^{n-1}\\sum_{j=0}^{m-1} \\hat{c}[i,j] x^{i}y^{j}\\]
Plugging an evaluation from (1) into (2) results in the following:

\n

\\[\\tag{3} d(u_{\\mu},\\omega_{\\eta}) \\equiv \\sum_{i=0}^{n-1}\\sum_{j=0}^{m-1} \\hat{c}[i,j] {u_{\\mu}}^{i}{\\omega_{\\eta}}^{j}\\]

\n

references

[1] Ben Edgington. Bls12-381 for the rest of us. https://hackmd.io/@benjaminion/bls12-381.
[2] Dankrad Feist. Data availability encoding. https://notes.ethereum.org/@dankrad/danksharding_encoding

\n"},{"title":"cryptography (1) group & field","date":"2023-06-03T06:29:26.000Z","_content":"\n\n\n\n\n## Group\na group is a set of elements \\\\( G \\\\) together with an operation \\\\( \\circ \\\\). a group has the following properties\n1. the group operation \\\\( \\circ \\\\) is closed. that is, for all \\\\( a,b, \\in G \\\\), it holds that \\\\( a \\circ b = c \\in G \\\\)\n2. the group operation is associative. That is, \\\\( a \\circ (b \\circ c) = (a \\circ b) \\circ c \\\\) for all \\\\( a,b,c \\in G \\\\)\n3. there is an element \\\\( 1 \\in G \\\\), called the neutral element (or identity element), such that \\\\( a \\circ \\ = 1 \\circ a\\\\) for all \\\\( a \\in G \\\\)\n4. For each \\\\( a \\in G \\\\) there exists an element \\\\( a^{-1} \\in G\\\\), called the inverse of a, such that \\\\( a \\circ a^{-1} = 1 \\\\).\n5. a group G is abelian (or commutative) if, furthermore, \\\\( a \\circ b = b \\circ a \\\\) for all \\\\( a, b \\in G \\\\) \n\n### Example of Group\nthe set of integers \\\\( Z_{m} = 0,1,...,m-1 \\\\) and the operation addition modulo \\\\( m \\\\) forma group with the neutral element 0. every element \\\\( a \\\\) has an inverse \\\\( -a \\\\). note that this set does not form a group with the operation multiplication gecause mose eleents \\\\( a \\\\) do not have an inverse such that \\\\( a a^{-1} = 1 \\bmod m\\\\).\n\nIn order to have all four basic arithmetic operations (i.e. addition, subtraction, multiplication, division) in one structure, we need a set which contains an additive and a multiplicative group. this is what we call a field.\n\n## Field\nA field is a set of elements with the following properties\n- all elements of \\\\( F\\\\) form an additive group with the group operation \\\\( + \\\\) and the neutral element \\\\( 0 \\\\)\n- all element of \\\\( F \\\\) except \\\\( 0 \\\\) form a multiplicative group with the gorup operation \\\\( x \\\\) and the neutral element \\\\( 1 \\\\) .\n- when the two group operations are mixed, the distributivety law holds, i.e., for all \\\\( a,b,c \\in F: a(b+c) = (ab) + (ac) \\\\) \n\n### Example of Field\nthe set \\\\( R \\\\) of real numbers is a field with the neutral element 0 for the additive group and the neutral element 1 for the multiplicative group. every real number \\\\( a \\\\) has an additive inverse, namely \\\\( -a \\\\), and every nonzero element \\\\( a \\\\) has a multiplicative inverse \\\\( 1 \\div a \\\\)\n\n## Galois Field\nIn cryptography, we are almose always interested in fields with a **finite** number of elements, which we call finite fields or `Galois field`. the number of elements in the field is called the `order` or `cardinality` of the field. Of fundamental importance is the following theorem\n***\na field with order `m` only exists if `m` is a prime power, i.e, \\\\( m = p^{n} \\\\), for some positive integer `n` and prime integer `p`. `p` is called the characteristic of the finite field\n***\n\nthe theorem implies that there are, for instance, finite fields with 11 elements (\\\\( 11^{1} \\\\)), or with 81 elements \\\\( 81 = 3^{4} \\\\) or with 256 elements (\\\\( 256 = 2^{8} \\\\)). \n\n## Prime Field\nthe most intuitive examples of finite fields are fields of prime order, i.e., fields with \\\\( n =1 \\\\). elements of the field \\\\( GF(p) \\\\) can be represented by integers \\\\( 0,1,..., p-1 \\\\).\n\n## Extension Field `GF(2^m)`\nIn AES the finite field contains 256 elements and is denoted as \\\\( GF(2^8) \\\\).\nHowever, if the order of a finite field is not prime, and \\\\( 2^8 \\\\) is clearly not a prime, the addition and multiplication operation cannot be represented by addition and multiplication of integers modulo \\\\( 2^8 \\\\). such fields with \\\\( m >1 \\\\) are called `extension field`. in order to deal with extension fields we need (1) a different notation for field elements and (2) different rules for performing arithmetic with the elements. elements of extension fields can be represented as `polynomials`, and computation in the extension field is achieved by performing a certain type of `polynomial arithmetic`.\n\nIn the field \\\\( GF(2^8) \\\\), which is used in AES, each element \\\\( A \\in GF(2^8) \\\\) is thus represented as: \n\\\\[ A(x) = a_{7}x^7 + ...+a_{1}x+a_{0}, a_{i} \\in GF(2) = {0,1} \\\\]\nIt is also important to observe that every polynomial can simply be stored in digital form as an 8-bit vector\n\\\\[ A = (a_7,a_6,a_5,a_4,a_3,a_2,a_1,a_0) \\\\]\n\n### addition and subtraction in `GF(2^m)`\naddition and subtraction are simply achieved by performing polynomial addition and subtraction. note that we perform modulo 2 addition (or subtraction) with the coefficients.\n\\\\[ A(x) = x^7 + x^6 + x^4 + 1 \\\\]\n\\\\[ B(x) = x^4 + x^2+ 1 \\\\]\n\\\\[ A(x) + B(x) = x^7 + x^6+ x^2 \\\\]\n\n### multiplication in `GF(2^m)`\nfirstly, two elements (represented by their polynomials) of a finite field `GF(2^m)` are multiplied usig the standard polynomial multiplication rule \\\\[ A(x) \\dot B(x) = C(x) \\\\]. In general, the product polynomial \\\\[ C(x) \\\\] will have a degree higher than `m-1` and has to be reduced. to do that, the product of the multiplication is divided by a certain polyhomial, and we consider only the remainder after the polynomial division. we need **irreducible** polynomials for the module reduction. irreducible polynomials are roughly comparable to prime numbers. their only factors are 1 and polynomial itself.\nThus, every field `GF(2^m)` requires an irreducible polynomial `P(x)` of degree `m`. \nFor AES, the irreducible polynomial is \n\\\\[ P(x) = x^8 + x^4+ x^3 +x + 1 \\\\]\nthe main algorithm for computing multiplicative inverse is the extended Euclidean algorithm, which is introduced in other posts.","source":"_posts/cryptography/cryptography-01-primitive-group-and-field.md","raw":"---\ntitle: cryptography (1) group & field\ndate: 2023-06-03 14:29:26\ntags: [cryptography]\n---\n\n\n\n\n\n## Group\na group is a set of elements \\\\( G \\\\) together with an operation \\\\( \\circ \\\\). a group has the following properties\n1. the group operation \\\\( \\circ \\\\) is closed. that is, for all \\\\( a,b, \\in G \\\\), it holds that \\\\( a \\circ b = c \\in G \\\\)\n2. the group operation is associative. That is, \\\\( a \\circ (b \\circ c) = (a \\circ b) \\circ c \\\\) for all \\\\( a,b,c \\in G \\\\)\n3. there is an element \\\\( 1 \\in G \\\\), called the neutral element (or identity element), such that \\\\( a \\circ \\ = 1 \\circ a\\\\) for all \\\\( a \\in G \\\\)\n4. For each \\\\( a \\in G \\\\) there exists an element \\\\( a^{-1} \\in G\\\\), called the inverse of a, such that \\\\( a \\circ a^{-1} = 1 \\\\).\n5. a group G is abelian (or commutative) if, furthermore, \\\\( a \\circ b = b \\circ a \\\\) for all \\\\( a, b \\in G \\\\) \n\n### Example of Group\nthe set of integers \\\\( Z_{m} = 0,1,...,m-1 \\\\) and the operation addition modulo \\\\( m \\\\) forma group with the neutral element 0. every element \\\\( a \\\\) has an inverse \\\\( -a \\\\). note that this set does not form a group with the operation multiplication gecause mose eleents \\\\( a \\\\) do not have an inverse such that \\\\( a a^{-1} = 1 \\bmod m\\\\).\n\nIn order to have all four basic arithmetic operations (i.e. addition, subtraction, multiplication, division) in one structure, we need a set which contains an additive and a multiplicative group. this is what we call a field.\n\n## Field\nA field is a set of elements with the following properties\n- all elements of \\\\( F\\\\) form an additive group with the group operation \\\\( + \\\\) and the neutral element \\\\( 0 \\\\)\n- all element of \\\\( F \\\\) except \\\\( 0 \\\\) form a multiplicative group with the gorup operation \\\\( x \\\\) and the neutral element \\\\( 1 \\\\) .\n- when the two group operations are mixed, the distributivety law holds, i.e., for all \\\\( a,b,c \\in F: a(b+c) = (ab) + (ac) \\\\) \n\n### Example of Field\nthe set \\\\( R \\\\) of real numbers is a field with the neutral element 0 for the additive group and the neutral element 1 for the multiplicative group. every real number \\\\( a \\\\) has an additive inverse, namely \\\\( -a \\\\), and every nonzero element \\\\( a \\\\) has a multiplicative inverse \\\\( 1 \\div a \\\\)\n\n## Galois Field\nIn cryptography, we are almose always interested in fields with a **finite** number of elements, which we call finite fields or `Galois field`. the number of elements in the field is called the `order` or `cardinality` of the field. Of fundamental importance is the following theorem\n***\na field with order `m` only exists if `m` is a prime power, i.e, \\\\( m = p^{n} \\\\), for some positive integer `n` and prime integer `p`. `p` is called the characteristic of the finite field\n***\n\nthe theorem implies that there are, for instance, finite fields with 11 elements (\\\\( 11^{1} \\\\)), or with 81 elements \\\\( 81 = 3^{4} \\\\) or with 256 elements (\\\\( 256 = 2^{8} \\\\)). \n\n## Prime Field\nthe most intuitive examples of finite fields are fields of prime order, i.e., fields with \\\\( n =1 \\\\). elements of the field \\\\( GF(p) \\\\) can be represented by integers \\\\( 0,1,..., p-1 \\\\).\n\n## Extension Field `GF(2^m)`\nIn AES the finite field contains 256 elements and is denoted as \\\\( GF(2^8) \\\\).\nHowever, if the order of a finite field is not prime, and \\\\( 2^8 \\\\) is clearly not a prime, the addition and multiplication operation cannot be represented by addition and multiplication of integers modulo \\\\( 2^8 \\\\). such fields with \\\\( m >1 \\\\) are called `extension field`. in order to deal with extension fields we need (1) a different notation for field elements and (2) different rules for performing arithmetic with the elements. elements of extension fields can be represented as `polynomials`, and computation in the extension field is achieved by performing a certain type of `polynomial arithmetic`.\n\nIn the field \\\\( GF(2^8) \\\\), which is used in AES, each element \\\\( A \\in GF(2^8) \\\\) is thus represented as: \n\\\\[ A(x) = a_{7}x^7 + ...+a_{1}x+a_{0}, a_{i} \\in GF(2) = {0,1} \\\\]\nIt is also important to observe that every polynomial can simply be stored in digital form as an 8-bit vector\n\\\\[ A = (a_7,a_6,a_5,a_4,a_3,a_2,a_1,a_0) \\\\]\n\n### addition and subtraction in `GF(2^m)`\naddition and subtraction are simply achieved by performing polynomial addition and subtraction. note that we perform modulo 2 addition (or subtraction) with the coefficients.\n\\\\[ A(x) = x^7 + x^6 + x^4 + 1 \\\\]\n\\\\[ B(x) = x^4 + x^2+ 1 \\\\]\n\\\\[ A(x) + B(x) = x^7 + x^6+ x^2 \\\\]\n\n### multiplication in `GF(2^m)`\nfirstly, two elements (represented by their polynomials) of a finite field `GF(2^m)` are multiplied usig the standard polynomial multiplication rule \\\\[ A(x) \\dot B(x) = C(x) \\\\]. In general, the product polynomial \\\\[ C(x) \\\\] will have a degree higher than `m-1` and has to be reduced. to do that, the product of the multiplication is divided by a certain polyhomial, and we consider only the remainder after the polynomial division. we need **irreducible** polynomials for the module reduction. irreducible polynomials are roughly comparable to prime numbers. their only factors are 1 and polynomial itself.\nThus, every field `GF(2^m)` requires an irreducible polynomial `P(x)` of degree `m`. \nFor AES, the irreducible polynomial is \n\\\\[ P(x) = x^8 + x^4+ x^3 +x + 1 \\\\]\nthe main algorithm for computing multiplicative inverse is the extended Euclidean algorithm, which is introduced in other posts.","slug":"cryptography/cryptography-01-primitive-group-and-field","published":1,"updated":"2023-11-05T04:21:13.732Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8dm0009qwsj4fr9fx90","content":"\n\n\n\n\n

Group

a group is a set of elements \\( G \\) together with an operation \\( \\circ \\). a group has the following properties

\n
    \n
  1. the group operation \\( \\circ \\) is closed. that is, for all \\( a,b, \\in G \\), it holds that \\( a \\circ b = c \\in G \\)
    2. \n
  2. the group operation is associative. That is, \\( a \\circ (b \\circ c) = (a \\circ b) \\circ c \\) for all \\( a,b,c \\in G \\)
    3. \n
  3. there is an element \\( 1 \\in G \\), called the neutral element (or identity element), such that \\( a \\circ \\ = 1 \\circ a\\) for all \\( a \\in G \\)
    4. \n
  4. For each \\( a \\in G \\) there exists an element \\( a^{-1} \\in G\\), called the inverse of a, such that \\( a \\circ a^{-1} = 1 \\).
    5. \n
  5. a group G is abelian (or commutative) if, furthermore, \\( a \\circ b = b \\circ a \\) for all \\( a, b \\in G \\)
    6. \n
\n

Example of Group

the set of integers \\( Z_{m} = 0,1,…,m-1 \\) and the operation addition modulo \\( m \\) forma group with the neutral element 0. every element \\( a \\) has an inverse \\( -a \\). note that this set does not form a group with the operation multiplication gecause mose eleents \\( a \\) do not have an inverse such that \\( a a^{-1} = 1 \\bmod m\\).

\n

In order to have all four basic arithmetic operations (i.e. addition, subtraction, multiplication, division) in one structure, we need a set which contains an additive and a multiplicative group. this is what we call a field.

\n

Field

A field is a set of elements with the following properties

\n
    \n
  • all elements of \\( F\\) form an additive group with the group operation \\( + \\) and the neutral element \\( 0 \\)
    • \n
  • all element of \\( F \\) except \\( 0 \\) form a multiplicative group with the gorup operation \\( x \\) and the neutral element \\( 1 \\) .
    • \n
  • when the two group operations are mixed, the distributivety law holds, i.e., for all \\( a,b,c \\in F: a(b+c) = (ab) + (ac) \\)
    • \n
\n

Example of Field

the set \\( R \\) of real numbers is a field with the neutral element 0 for the additive group and the neutral element 1 for the multiplicative group. every real number \\( a \\) has an additive inverse, namely \\( -a \\), and every nonzero element \\( a \\) has a multiplicative inverse \\( 1 \\div a \\)

\n

Galois Field

In cryptography, we are almose always interested in fields with a finite number of elements, which we call finite fields or Galois field. the number of elements in the field is called the order or cardinality of the field. Of fundamental importance is the following theorem

\n
\n

a field with order m only exists if m is a prime power, i.e, \\( m = p^{n} \\), for some positive integer n and prime integer p. p is called the characteristic of the finite field

\n
\n

the theorem implies that there are, for instance, finite fields with 11 elements (\\( 11^{1} \\)), or with 81 elements \\( 81 = 3^{4} \\) or with 256 elements (\\( 256 = 2^{8} \\)).

\n

Prime Field

the most intuitive examples of finite fields are fields of prime order, i.e., fields with \\( n =1 \\). elements of the field \\( GF(p) \\) can be represented by integers \\( 0,1,…, p-1 \\).

\n

Extension Field GF(2^m)

In AES the finite field contains 256 elements and is denoted as \\( GF(2^8) \\).
However, if the order of a finite field is not prime, and \\( 2^8 \\) is clearly not a prime, the addition and multiplication operation cannot be represented by addition and multiplication of integers modulo \\( 2^8 \\). such fields with \\( m >1 \\) are called extension field. in order to deal with extension fields we need (1) a different notation for field elements and (2) different rules for performing arithmetic with the elements. elements of extension fields can be represented as polynomials, and computation in the extension field is achieved by performing a certain type of polynomial arithmetic.

\n

In the field \\( GF(2^8) \\), which is used in AES, each element \\( A \\in GF(2^8) \\) is thus represented as:
\\[ A(x) = a_{7}x^7 + …+a_{1}x+a_{0}, a_{i} \\in GF(2) = {0,1} \\]
It is also important to observe that every polynomial can simply be stored in digital form as an 8-bit vector
\\[ A = (a_7,a_6,a_5,a_4,a_3,a_2,a_1,a_0) \\]

\n

addition and subtraction in GF(2^m)

addition and subtraction are simply achieved by performing polynomial addition and subtraction. note that we perform modulo 2 addition (or subtraction) with the coefficients.
\\[ A(x) = x^7 + x^6 + x^4 + 1 \\]
\\[ B(x) = x^4 + x^2+ 1 \\]
\\[ A(x) + B(x) = x^7 + x^6+ x^2 \\]

\n

multiplication in GF(2^m)

firstly, two elements (represented by their polynomials) of a finite field GF(2^m) are multiplied usig the standard polynomial multiplication rule \\[ A(x) \\dot B(x) = C(x) \\]. In general, the product polynomial \\[ C(x) \\] will have a degree higher than m-1 and has to be reduced. to do that, the product of the multiplication is divided by a certain polyhomial, and we consider only the remainder after the polynomial division. we need irreducible polynomials for the module reduction. irreducible polynomials are roughly comparable to prime numbers. their only factors are 1 and polynomial itself.
Thus, every field GF(2^m) requires an irreducible polynomial P(x) of degree m.
For AES, the irreducible polynomial is
\\[ P(x) = x^8 + x^4+ x^3 +x + 1 \\]
the main algorithm for computing multiplicative inverse is the extended Euclidean algorithm, which is introduced in other posts.

\n","site":{"data":{}},"excerpt":"","more":"\n\n\n\n\n

Group

a group is a set of elements \\( G \\) together with an operation \\( \\circ \\). a group has the following properties

\n
    \n
  1. the group operation \\( \\circ \\) is closed. that is, for all \\( a,b, \\in G \\), it holds that \\( a \\circ b = c \\in G \\)
    2. \n
  2. the group operation is associative. That is, \\( a \\circ (b \\circ c) = (a \\circ b) \\circ c \\) for all \\( a,b,c \\in G \\)
    3. \n
  3. there is an element \\( 1 \\in G \\), called the neutral element (or identity element), such that \\( a \\circ \\ = 1 \\circ a\\) for all \\( a \\in G \\)
    4. \n
  4. For each \\( a \\in G \\) there exists an element \\( a^{-1} \\in G\\), called the inverse of a, such that \\( a \\circ a^{-1} = 1 \\).
    5. \n
  5. a group G is abelian (or commutative) if, furthermore, \\( a \\circ b = b \\circ a \\) for all \\( a, b \\in G \\)
    6. \n
\n

Example of Group

the set of integers \\( Z_{m} = 0,1,…,m-1 \\) and the operation addition modulo \\( m \\) forma group with the neutral element 0. every element \\( a \\) has an inverse \\( -a \\). note that this set does not form a group with the operation multiplication gecause mose eleents \\( a \\) do not have an inverse such that \\( a a^{-1} = 1 \\bmod m\\).

\n

In order to have all four basic arithmetic operations (i.e. addition, subtraction, multiplication, division) in one structure, we need a set which contains an additive and a multiplicative group. this is what we call a field.

\n

Field

A field is a set of elements with the following properties

\n
    \n
  • all elements of \\( F\\) form an additive group with the group operation \\( + \\) and the neutral element \\( 0 \\)
    • \n
  • all element of \\( F \\) except \\( 0 \\) form a multiplicative group with the gorup operation \\( x \\) and the neutral element \\( 1 \\) .
    • \n
  • when the two group operations are mixed, the distributivety law holds, i.e., for all \\( a,b,c \\in F: a(b+c) = (ab) + (ac) \\)
    • \n
\n

Example of Field

the set \\( R \\) of real numbers is a field with the neutral element 0 for the additive group and the neutral element 1 for the multiplicative group. every real number \\( a \\) has an additive inverse, namely \\( -a \\), and every nonzero element \\( a \\) has a multiplicative inverse \\( 1 \\div a \\)

\n

Galois Field

In cryptography, we are almose always interested in fields with a finite number of elements, which we call finite fields or Galois field. the number of elements in the field is called the order or cardinality of the field. Of fundamental importance is the following theorem

\n
\n

a field with order m only exists if m is a prime power, i.e, \\( m = p^{n} \\), for some positive integer n and prime integer p. p is called the characteristic of the finite field

\n
\n

the theorem implies that there are, for instance, finite fields with 11 elements (\\( 11^{1} \\)), or with 81 elements \\( 81 = 3^{4} \\) or with 256 elements (\\( 256 = 2^{8} \\)).

\n

Prime Field

the most intuitive examples of finite fields are fields of prime order, i.e., fields with \\( n =1 \\). elements of the field \\( GF(p) \\) can be represented by integers \\( 0,1,…, p-1 \\).

\n

Extension Field GF(2^m)

In AES the finite field contains 256 elements and is denoted as \\( GF(2^8) \\).
However, if the order of a finite field is not prime, and \\( 2^8 \\) is clearly not a prime, the addition and multiplication operation cannot be represented by addition and multiplication of integers modulo \\( 2^8 \\). such fields with \\( m >1 \\) are called extension field. in order to deal with extension fields we need (1) a different notation for field elements and (2) different rules for performing arithmetic with the elements. elements of extension fields can be represented as polynomials, and computation in the extension field is achieved by performing a certain type of polynomial arithmetic.

\n

In the field \\( GF(2^8) \\), which is used in AES, each element \\( A \\in GF(2^8) \\) is thus represented as:
\\[ A(x) = a_{7}x^7 + …+a_{1}x+a_{0}, a_{i} \\in GF(2) = {0,1} \\]
It is also important to observe that every polynomial can simply be stored in digital form as an 8-bit vector
\\[ A = (a_7,a_6,a_5,a_4,a_3,a_2,a_1,a_0) \\]

\n

addition and subtraction in GF(2^m)

addition and subtraction are simply achieved by performing polynomial addition and subtraction. note that we perform modulo 2 addition (or subtraction) with the coefficients.
\\[ A(x) = x^7 + x^6 + x^4 + 1 \\]
\\[ B(x) = x^4 + x^2+ 1 \\]
\\[ A(x) + B(x) = x^7 + x^6+ x^2 \\]

\n

multiplication in GF(2^m)

firstly, two elements (represented by their polynomials) of a finite field GF(2^m) are multiplied usig the standard polynomial multiplication rule \\[ A(x) \\dot B(x) = C(x) \\]. In general, the product polynomial \\[ C(x) \\] will have a degree higher than m-1 and has to be reduced. to do that, the product of the multiplication is divided by a certain polyhomial, and we consider only the remainder after the polynomial division. we need irreducible polynomials for the module reduction. irreducible polynomials are roughly comparable to prime numbers. their only factors are 1 and polynomial itself.
Thus, every field GF(2^m) requires an irreducible polynomial P(x) of degree m.
For AES, the irreducible polynomial is
\\[ P(x) = x^8 + x^4+ x^3 +x + 1 \\]
the main algorithm for computing multiplicative inverse is the extended Euclidean algorithm, which is introduced in other posts.

\n"},{"title":"cryptography (3) elliptic curve","date":"2023-06-17T06:29:26.000Z","_content":"\n\n\n\n## elliptic curve definition\nfor cryptographic use, we need to conside the curve not over the real numbers but over a finite field. the most popular fields `GF(p)`, where all arithmetic is performed modulo a prime p.\n***\nthe elliptic curve over \\\\( Z_{p}, p>3 \\\\), is the set of all pairs \\\\( (x,y) \\in Z_{p} \\\\) which fulfill\n \\\\[ y^2 \\equiv x^3 + a \\cdot x + b \\bmod p \\\\]\ntogether with an imaginary point of infinity \\\\( \\mathcal{O} \\\\), where\n \\\\[ a,b \\in Z_{p} \\\\]\n and the condition \\\\( 4 \\cdot a^3 + 27 \\cdot b^2 \\neq 0 \\bmod p \\\\)\n***\nthe definition of elliptic curve requires that the curve is nonsingular. Geometrically speaking, this means that the plot has no self-intersections or vertices, which is achieved if the discriminant of the curve \\\\( -16(4a^3) + 27b^2 \\\\) is nonzero.\n\n## operations on elliptic curve\n![point addition](/images/cryptography/elliptic_curve/point_addition.webp)\nlet's denote the group operation with the addition symbol `+`. \"addition\" means that given two points and their coordinates, say \\\\( P = (x_1, y_1) \\\\) and \\\\( Q = (x_2, y_2) \\\\), we have to compute the coordidnate of a third point \\\\( R (x_3, y_3) \\\\). the construction works as follows: draw a line through P and Q and obtain a third point of intersection between the elliptic curve and the line (-R). Mirror this third intersection point along the x-axis. this mirored point is, by definition, the point R. point doubling `(P+P = 2P)` is a tangent line through the point P, and the rest is same.\nthe fomulae for Point Addition (P+Q) and Point Doubling (2P) is as below \n***\n \\\\[ x_3 = s^2 - x_1 -x_2 \\bmod p \\\\]\n \\\\[ y_3 = s(x_1 - x_3) - y_1 \\bmod p\\\\]\n where \n\\begin{equation}\ns = \n\\begin{cases}\n\\frac{y_2-y_1}{x_2-x_1} \\bmod p & if P \\neq Q (\\text{point addition}) \\cr\n\\frac{3x_{1}^{2} + a}{2y_1} \\bmod p & if P =Q (\\text{point doubling})\n\\end{cases}\n\\end{equation}\n***\nnote that the parameter s is the slope of the line through P and Q in the case of point addition, or the slope of the tangent through P in the case of point doubling.\nFurthmore, we define an abstract point at infinity as the neutral element \\\\( \\mathcal{O} \\\\). \n\\\\[ P + \\mathcal{O} = P\\\\]\naccording the group definition, we can now also define the inverse `-P` of any group element P as\n\\\\[ P + (-P) = \\mathcal{O}\\\\]\nTherefore, the inverse of \\\\( P = (x_p, y_p) \\\\) is just \\\\( P = (x_p, -y_p)\\\\). since \\\\( -y_p \\equiv p - y_p\\\\), hence\n\\\\[ -P = (x_p, p-y_p) \\\\]\n\n## DLP(discrete log problem) with Elliptic curve\nto set up DL cryptosystems it is important to know the order of the group.\nHasse's theorem\n***\ngiven an elliptic curve E modulo p, the number of points on the curve is denoted by #E and is bounded by:\n\\\\[ p+1-2\\sqrt{p} \\leq \\\\#E \\leq p+1+2\\sqrt{p} \\\\]\n***\nElliptic Curved Discrete Logarithm Problem (ECDLP)\n***\nGiven an elliptic curve E. We consider a primitive elemnt P and another element T. The DL problem is finding the integer d, where \\\\( 1 \\leq d \\leq \\\\#E \\\\), such that:\n\\\\[ P + P + ....+ P = dP = T \\\\]\n***\nIn cryptosystems, d is the private key which is an integer, while the public key T is a point on the curve with coordinates \\\\( T = (x_T, y_T)\\\\)\nUsually a **square-and-multiply** algorithm is used to calculate point multiplication (the algorithm detail is not coverred in this post). \n","source":"_posts/cryptography/cryptography-03-elliptic-curve.md","raw":"---\ntitle: cryptography (3) elliptic curve\ndate: 2023-06-17 14:29:26\ntags: [cryptography]\n---\n\n\n\n\n## elliptic curve definition\nfor cryptographic use, we need to conside the curve not over the real numbers but over a finite field. the most popular fields `GF(p)`, where all arithmetic is performed modulo a prime p.\n***\nthe elliptic curve over \\\\( Z_{p}, p>3 \\\\), is the set of all pairs \\\\( (x,y) \\in Z_{p} \\\\) which fulfill\n \\\\[ y^2 \\equiv x^3 + a \\cdot x + b \\bmod p \\\\]\ntogether with an imaginary point of infinity \\\\( \\mathcal{O} \\\\), where\n \\\\[ a,b \\in Z_{p} \\\\]\n and the condition \\\\( 4 \\cdot a^3 + 27 \\cdot b^2 \\neq 0 \\bmod p \\\\)\n***\nthe definition of elliptic curve requires that the curve is nonsingular. Geometrically speaking, this means that the plot has no self-intersections or vertices, which is achieved if the discriminant of the curve \\\\( -16(4a^3) + 27b^2 \\\\) is nonzero.\n\n## operations on elliptic curve\n![point addition](/images/cryptography/elliptic_curve/point_addition.webp)\nlet's denote the group operation with the addition symbol `+`. \"addition\" means that given two points and their coordinates, say \\\\( P = (x_1, y_1) \\\\) and \\\\( Q = (x_2, y_2) \\\\), we have to compute the coordidnate of a third point \\\\( R (x_3, y_3) \\\\). the construction works as follows: draw a line through P and Q and obtain a third point of intersection between the elliptic curve and the line (-R). Mirror this third intersection point along the x-axis. this mirored point is, by definition, the point R. point doubling `(P+P = 2P)` is a tangent line through the point P, and the rest is same.\nthe fomulae for Point Addition (P+Q) and Point Doubling (2P) is as below \n***\n \\\\[ x_3 = s^2 - x_1 -x_2 \\bmod p \\\\]\n \\\\[ y_3 = s(x_1 - x_3) - y_1 \\bmod p\\\\]\n where \n\\begin{equation}\ns = \n\\begin{cases}\n\\frac{y_2-y_1}{x_2-x_1} \\bmod p & if P \\neq Q (\\text{point addition}) \\cr\n\\frac{3x_{1}^{2} + a}{2y_1} \\bmod p & if P =Q (\\text{point doubling})\n\\end{cases}\n\\end{equation}\n***\nnote that the parameter s is the slope of the line through P and Q in the case of point addition, or the slope of the tangent through P in the case of point doubling.\nFurthmore, we define an abstract point at infinity as the neutral element \\\\( \\mathcal{O} \\\\). \n\\\\[ P + \\mathcal{O} = P\\\\]\naccording the group definition, we can now also define the inverse `-P` of any group element P as\n\\\\[ P + (-P) = \\mathcal{O}\\\\]\nTherefore, the inverse of \\\\( P = (x_p, y_p) \\\\) is just \\\\( P = (x_p, -y_p)\\\\). since \\\\( -y_p \\equiv p - y_p\\\\), hence\n\\\\[ -P = (x_p, p-y_p) \\\\]\n\n## DLP(discrete log problem) with Elliptic curve\nto set up DL cryptosystems it is important to know the order of the group.\nHasse's theorem\n***\ngiven an elliptic curve E modulo p, the number of points on the curve is denoted by #E and is bounded by:\n\\\\[ p+1-2\\sqrt{p} \\leq \\\\#E \\leq p+1+2\\sqrt{p} \\\\]\n***\nElliptic Curved Discrete Logarithm Problem (ECDLP)\n***\nGiven an elliptic curve E. We consider a primitive elemnt P and another element T. The DL problem is finding the integer d, where \\\\( 1 \\leq d \\leq \\\\#E \\\\), such that:\n\\\\[ P + P + ....+ P = dP = T \\\\]\n***\nIn cryptosystems, d is the private key which is an integer, while the public key T is a point on the curve with coordinates \\\\( T = (x_T, y_T)\\\\)\nUsually a **square-and-multiply** algorithm is used to calculate point multiplication (the algorithm detail is not coverred in this post). \n","slug":"cryptography/cryptography-03-elliptic-curve","published":1,"updated":"2023-11-05T04:21:17.033Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8dn000bqwsj9gt33o5h","content":"\n\n\n\n

elliptic curve definition

for cryptographic use, we need to conside the curve not over the real numbers but over a finite field. the most popular fields GF(p), where all arithmetic is performed modulo a prime p.

\n
\n

the elliptic curve over \\( Z_{p}, p>3 \\), is the set of all pairs \\( (x,y) \\in Z_{p} \\) which fulfill
\\[ y^2 \\equiv x^3 + a \\cdot x + b \\bmod p \\]
together with an imaginary point of infinity \\( \\mathcal{O} \\), where
\\[ a,b \\in Z_{p} \\]
and the condition \\( 4 \\cdot a^3 + 27 \\cdot b^2 \\neq 0 \\bmod p \\)

\n
\n

the definition of elliptic curve requires that the curve is nonsingular. Geometrically speaking, this means that the plot has no self-intersections or vertices, which is achieved if the discriminant of the curve \\( -16(4a^3) + 27b^2 \\) is nonzero.

\n

operations on elliptic curve

\"point
let’s denote the group operation with the addition symbol +. “addition” means that given two points and their coordinates, say \\( P = (x_1, y_1) \\) and \\( Q = (x_2, y_2) \\), we have to compute the coordidnate of a third point \\( R (x_3, y_3) \\). the construction works as follows: draw a line through P and Q and obtain a third point of intersection between the elliptic curve and the line (-R). Mirror this third intersection point along the x-axis. this mirored point is, by definition, the point R. point doubling (P+P = 2P) is a tangent line through the point P, and the rest is same.
the fomulae for Point Addition (P+Q) and Point Doubling (2P) is as below

\n
\n

\\[ x_3 = s^2 - x_1 -x_2 \\bmod p \\]
\\[ y_3 = s(x_1 - x_3) - y_1 \\bmod p\\]
where
\\begin{equation}
s =
\\begin{cases}
\\frac{y_2-y_1}{x_2-x_1} \\bmod p & if P \\neq Q (\\text{point addition}) \\cr
\\frac{3x_{1}^{2} + a}{2y_1} \\bmod p & if P =Q (\\text{point doubling})
\\end{cases}
\\end{equation}

\n
\n

note that the parameter s is the slope of the line through P and Q in the case of point addition, or the slope of the tangent through P in the case of point doubling.
Furthmore, we define an abstract point at infinity as the neutral element \\( \\mathcal{O} \\).
\\[ P + \\mathcal{O} = P\\]
according the group definition, we can now also define the inverse -P of any group element P as
\\[ P + (-P) = \\mathcal{O}\\]
Therefore, the inverse of \\( P = (x_p, y_p) \\) is just \\( P = (x_p, -y_p)\\). since \\( -y_p \\equiv p - y_p\\), hence
\\[ -P = (x_p, p-y_p) \\]

\n

DLP(discrete log problem) with Elliptic curve

to set up DL cryptosystems it is important to know the order of the group.
Hasse’s theorem

\n
\n

given an elliptic curve E modulo p, the number of points on the curve is denoted by #E and is bounded by:
\\[ p+1-2\\sqrt{p} \\leq \\#E \\leq p+1+2\\sqrt{p} \\]

\n
\n

Elliptic Curved Discrete Logarithm Problem (ECDLP)

\n
\n

Given an elliptic curve E. We consider a primitive elemnt P and another element T. The DL problem is finding the integer d, where \\( 1 \\leq d \\leq \\#E \\), such that:
\\[ P + P + ….+ P = dP = T \\]

\n
\n

In cryptosystems, d is the private key which is an integer, while the public key T is a point on the curve with coordinates \\( T = (x_T, y_T)\\)
Usually a square-and-multiply algorithm is used to calculate point multiplication (the algorithm detail is not coverred in this post).

\n","site":{"data":{}},"excerpt":"","more":"\n\n\n\n

elliptic curve definition

for cryptographic use, we need to conside the curve not over the real numbers but over a finite field. the most popular fields GF(p), where all arithmetic is performed modulo a prime p.

\n
\n

the elliptic curve over \\( Z_{p}, p>3 \\), is the set of all pairs \\( (x,y) \\in Z_{p} \\) which fulfill
\\[ y^2 \\equiv x^3 + a \\cdot x + b \\bmod p \\]
together with an imaginary point of infinity \\( \\mathcal{O} \\), where
\\[ a,b \\in Z_{p} \\]
and the condition \\( 4 \\cdot a^3 + 27 \\cdot b^2 \\neq 0 \\bmod p \\)

\n
\n

the definition of elliptic curve requires that the curve is nonsingular. Geometrically speaking, this means that the plot has no self-intersections or vertices, which is achieved if the discriminant of the curve \\( -16(4a^3) + 27b^2 \\) is nonzero.

\n

operations on elliptic curve

\"point
let’s denote the group operation with the addition symbol +. “addition” means that given two points and their coordinates, say \\( P = (x_1, y_1) \\) and \\( Q = (x_2, y_2) \\), we have to compute the coordidnate of a third point \\( R (x_3, y_3) \\). the construction works as follows: draw a line through P and Q and obtain a third point of intersection between the elliptic curve and the line (-R). Mirror this third intersection point along the x-axis. this mirored point is, by definition, the point R. point doubling (P+P = 2P) is a tangent line through the point P, and the rest is same.
the fomulae for Point Addition (P+Q) and Point Doubling (2P) is as below

\n
\n

\\[ x_3 = s^2 - x_1 -x_2 \\bmod p \\]
\\[ y_3 = s(x_1 - x_3) - y_1 \\bmod p\\]
where
\\begin{equation}
s =
\\begin{cases}
\\frac{y_2-y_1}{x_2-x_1} \\bmod p & if P \\neq Q (\\text{point addition}) \\cr
\\frac{3x_{1}^{2} + a}{2y_1} \\bmod p & if P =Q (\\text{point doubling})
\\end{cases}
\\end{equation}

\n
\n

note that the parameter s is the slope of the line through P and Q in the case of point addition, or the slope of the tangent through P in the case of point doubling.
Furthmore, we define an abstract point at infinity as the neutral element \\( \\mathcal{O} \\).
\\[ P + \\mathcal{O} = P\\]
according the group definition, we can now also define the inverse -P of any group element P as
\\[ P + (-P) = \\mathcal{O}\\]
Therefore, the inverse of \\( P = (x_p, y_p) \\) is just \\( P = (x_p, -y_p)\\). since \\( -y_p \\equiv p - y_p\\), hence
\\[ -P = (x_p, p-y_p) \\]

\n

DLP(discrete log problem) with Elliptic curve

to set up DL cryptosystems it is important to know the order of the group.
Hasse’s theorem

\n
\n

given an elliptic curve E modulo p, the number of points on the curve is denoted by #E and is bounded by:
\\[ p+1-2\\sqrt{p} \\leq \\#E \\leq p+1+2\\sqrt{p} \\]

\n
\n

Elliptic Curved Discrete Logarithm Problem (ECDLP)

\n
\n

Given an elliptic curve E. We consider a primitive elemnt P and another element T. The DL problem is finding the integer d, where \\( 1 \\leq d \\leq \\#E \\), such that:
\\[ P + P + ….+ P = dP = T \\]

\n
\n

In cryptosystems, d is the private key which is an integer, while the public key T is a point on the curve with coordinates \\( T = (x_T, y_T)\\)
Usually a square-and-multiply algorithm is used to calculate point multiplication (the algorithm detail is not coverred in this post).

\n"},{"title":"cryptography (4) digital signature","date":"2023-06-20T06:29:26.000Z","_content":"\n\n\n## Elgamal Digital Signature Scheme\nThe Elgammal signature scheme is based on the difficulty of computing discrete logarithms. Unlike RSA, where encryption and digital signature are almoste identical operations, the Elgamal digital signature is quite different from the encryption scheme with teh same name.\n\n### key generation\n***\n1. Choose a large prime \\\\(p\\\\).\n2. Choose a primitive element \\\\(\\alpha\\\\) of \\\\(Z_{p}^{\\ast}\\\\), or a subgroup of \\\\(Z_{p}^{\\ast}\\\\).\n3. Choose a random integer \\\\(d \\in {2,3,...,p-2}\\\\)\n4. Compute \\\\(\\beta = \\alpha^{d} \\bmod p\\\\)\n***\nThe public key is now formed by \\\\(k_{pub} = (p, \\alpha, \\beta)\\\\), and the private key by \\\\(k_{pr}=d\\\\)\n\\\\(Z_{p}^{\\ast}\\\\) is the set of integers who are smaller than \\\\(p\\\\) and coprime to \\\\(p\\\\)\n\n### signature and verification\nUsign the private ey and parameters of the public key, the signature\n\\\\[sig_{k_{pr}}(x, k_{E}) = (r,s)\\\\]\n\\\\(x\\\\) is the message. \\\\(k_{E}\\\\) is a random value, which forms an ephemeral private key\n***\n**Elgamal Signature Generation**\n1. choose a random ephemeral key \\\\(k_{E} \\in {0,1,2,..,p-2}\\\\) such that \\\\(gcd(k_{E}, p-1) = 1\\\\)\n2. compute the signatue parameters:\n\\\\[r \\equiv \\alpha^{k_{E}} \\bmod p\\\\]\n\\\\[s \\equiv (x - d \\cdot r)k_{E}^{-1} \\bmod p-1\\\\]\n***\non the receiving side, the signature is verified as \\\\(ver_{k_{pub}}(x,(r,s))\\\\) using the public key, the signature and the message.\n***\n**Elgamal Signature Verification**\n1. comput the value \n\\\\[t \\equiv \\beta^{r} \\cdot r^s \\bmod p\\\\]\n2. the verification follows form\n\\begin{equation}\nt = \n\\begin{cases}\n\\equiv \\alpha^{x} \\bmod p & => \\text{valid signature} \\cr\n\\not\\equiv \\alpha^{x} \\bmod p & => \\text{invalid signature}\n\\end{cases}\n\\end{equation}\n***\n\n## Digital Signature Algorithm (DSA)\nThe native Elgamal signature algorithm described above is rarely used in practice. Instead, a much more popular variant is used, known as the Digital Signature Algorithm (DSA). It is a federal US government standard for digital signatures and was proposed by NIST (National Institute of Standards and Technology). Its main advantages over the Elgamal signature scheme are that the signature is only 320-bit long and that some of the attacks\n that can threaten the Elgamal scheme are not applicable.\n### key generation\n***\n**key Generation for DSA**\n1. Generate a prime \\\\(p\\\\) with \\\\(2^1023 < p < 2^1024\\\\)\n2. find a prime divisor \\\\(q\\\\) of \\\\(p-1\\\\) \\\\(2^159 < q < 2^160\\\\)\n3. Find an element \\\\(\\alpha\\\\) with \\\\( ord(\\alpha) = q\\\\), i.e., \\alpha genertes the subgroup with \\\\(q\\\\) elements.\n4. choose a random integer \\\\(d\\\\) with \\\\(0 < d < q\\\\).\n5. compute \\\\(\\beta \\equiv \\alpha^{d} \\bmod p\\\\).\nthe keys are now:\n\\\\(k_{pub} = (p, q, \\alpha, \\beta)\\\\)\n\\\\(k_{pr}= (d)\\\\)\n***\nThe central idea of DSA is that there are two cyclic groups involved. One is the large cyclic group \\\\(Z_{p}*{\\ast}\\\\), the order of which has bit length of 1024 bit. The second one is in the 160-bit subgroup of \\\\(Z_{p}^{\\ast}\\\\). this set-up yields shorter signature.\n\n### Signature and Verification\nAs in the Elgamal signatue scheme, the DSA signature consists of a pair of integers \\\\((r,s)\\\\). Since each of the two parameters is only 160-bit long, the total signature length is 320 bit. \n***\n**DSA signature generation**\n1. choose an integer as random ephemeral key \\\\(k_{E}\\\\) with \\\\(0 < k_{E} < q\\\\).\n2. compute \\\\(r \\equiv (\\alpha^{k_{E}} \\bmod p) \\bmod q\\\\)\n3. compute \\\\(s \\equiv (SHA(x) + d \\cdot r)k_{E}^{-1} \\bmod q \\\\)\n***\n\nThe signature verification process is as follows:\n***\n**DSA signature verification**\n1. compute auxilary value \\\\(w \\equiv s^{-1} \\bmod q\\\\).\n2. compute auxilary value \\\\(u_{1} \\equiv w \\cdot SHA(x) \\bmod q\\\\).\n3. compute auxilary value \\\\(u_{2} \\equiv w \\cdot r \\bmod q\\\\).\n4. compute \\\\(v \\equiv (\\alpha^{u_1} \\cdot \\beta^{u_2} \\bmod p) \\bmod q\\\\).\n5. the verification \\\\(ver_{k_{pub}}(x, (r,s))\\\\) folows from\n\\begin{equation}\nv = \n\\begin{cases}\n\\equiv r \\bmod q & => \\text{valid signature} \\cr\n\\not\\equiv r \\bmod q & => \\text{invalid signature}\n\\end{cases}\n\\end{equation}\n***\n\n## Elliptic Curve Digital Signature Algorithm (ECDSA)\nElliptic curves have several advantages over RSA and over DL schemes like Elgamal or DSA. In particular, in abscence of strong attacks against elliptic curve cryptosystems (ECC), bit lengths in the range of 160-256 bit can be chosen which provide security equivalent to 1024-3072 bit RSA and DL scheme. The shorter bit length of ECC often results in shorter processing time and in shorter signatures. \nThe ECDSA standard is defined for elliptic curves over prime fields \\\\(Z_{p}\\\\) adn Galois fields \\\\(GF(2^m)\\\\). the former is often preferred in practice, and we only introduce this one in what follows\n### key generation\n***\n**Key Generation for ECDSA**\n1. Use and elliptic curve E with \n- modulus p\n- coefficients a and b\n- a point A which generates a cyclic group of prime order q.\n2. choose a random integer d with \\\\(0 < d < q\\\\)\n3. compute \\\\(B = dA\\\\).\nThe keys are now\n\\\\(k_{pub} = (p,a,b,q,A,B)\\\\)\n\\\\(k_{pr} = (d)\\\\)\n***\n\n### Signature and Verification\n***\n**ECDSA Signature Generation**\n1. choose an integer as random ephemeral key \\\\(k_{E}\\\\) with \\\\( 0 < k_{E} < q\\\\).\n2. compute \\\\(R = k_{E}A\\\\)\n3. Let \\\\(r = x_{R}\\\\)\n4. compute \\\\(s \\equiv (h(x) + d \\cdot r)k_{E}^{-1} \\bmod q\\\\)\n***\nthe signature verification process is as follows\n***\n**ECDSA Signature Verification**\n1. Compute auxiliary value \\\\(w \\equiv s^{-1} \\bmod q\\\\)\n2. compute auxilary value \\\\(u_1 \\equiv w \\cdot h(x) \\bmod q\\\\)\n3. compute auxiliary value \\\\(u_2 = w \\cdot r \\bmod q\\\\)\n4. compute \\\\(P = u_1 A + u_2 B\\\\).\n5. the verification \\\\(ver{k_{pub}}(x, (r,s))\\\\) follows from\n\\begin{equation}\nx_{P} = \n\\begin{cases}\n\\equiv r \\bmod q & => \\text{valid signature} \\cr\n\\not\\equiv r \\bmod q & => \\text{invalid signature}\n\\end{cases}\n\\end{equation}\n***\nThe point multiplication, which is in most cases by the far the most arithmetic intensive operation, can be precomputed by choosing the ephemeral key ahead of time.","source":"_posts/cryptography/cryptography-04-digital-signature.md","raw":"---\ntitle: cryptography (4) digital signature\ndate: 2023-06-20 14:29:26\ntags: [cryptography]\n---\n\n\n\n## Elgamal Digital Signature Scheme\nThe Elgammal signature scheme is based on the difficulty of computing discrete logarithms. Unlike RSA, where encryption and digital signature are almoste identical operations, the Elgamal digital signature is quite different from the encryption scheme with teh same name.\n\n### key generation\n***\n1. Choose a large prime \\\\(p\\\\).\n2. Choose a primitive element \\\\(\\alpha\\\\) of \\\\(Z_{p}^{\\ast}\\\\), or a subgroup of \\\\(Z_{p}^{\\ast}\\\\).\n3. Choose a random integer \\\\(d \\in {2,3,...,p-2}\\\\)\n4. Compute \\\\(\\beta = \\alpha^{d} \\bmod p\\\\)\n***\nThe public key is now formed by \\\\(k_{pub} = (p, \\alpha, \\beta)\\\\), and the private key by \\\\(k_{pr}=d\\\\)\n\\\\(Z_{p}^{\\ast}\\\\) is the set of integers who are smaller than \\\\(p\\\\) and coprime to \\\\(p\\\\)\n\n### signature and verification\nUsign the private ey and parameters of the public key, the signature\n\\\\[sig_{k_{pr}}(x, k_{E}) = (r,s)\\\\]\n\\\\(x\\\\) is the message. \\\\(k_{E}\\\\) is a random value, which forms an ephemeral private key\n***\n**Elgamal Signature Generation**\n1. choose a random ephemeral key \\\\(k_{E} \\in {0,1,2,..,p-2}\\\\) such that \\\\(gcd(k_{E}, p-1) = 1\\\\)\n2. compute the signatue parameters:\n\\\\[r \\equiv \\alpha^{k_{E}} \\bmod p\\\\]\n\\\\[s \\equiv (x - d \\cdot r)k_{E}^{-1} \\bmod p-1\\\\]\n***\non the receiving side, the signature is verified as \\\\(ver_{k_{pub}}(x,(r,s))\\\\) using the public key, the signature and the message.\n***\n**Elgamal Signature Verification**\n1. comput the value \n\\\\[t \\equiv \\beta^{r} \\cdot r^s \\bmod p\\\\]\n2. the verification follows form\n\\begin{equation}\nt = \n\\begin{cases}\n\\equiv \\alpha^{x} \\bmod p & => \\text{valid signature} \\cr\n\\not\\equiv \\alpha^{x} \\bmod p & => \\text{invalid signature}\n\\end{cases}\n\\end{equation}\n***\n\n## Digital Signature Algorithm (DSA)\nThe native Elgamal signature algorithm described above is rarely used in practice. Instead, a much more popular variant is used, known as the Digital Signature Algorithm (DSA). It is a federal US government standard for digital signatures and was proposed by NIST (National Institute of Standards and Technology). Its main advantages over the Elgamal signature scheme are that the signature is only 320-bit long and that some of the attacks\n that can threaten the Elgamal scheme are not applicable.\n### key generation\n***\n**key Generation for DSA**\n1. Generate a prime \\\\(p\\\\) with \\\\(2^1023 < p < 2^1024\\\\)\n2. find a prime divisor \\\\(q\\\\) of \\\\(p-1\\\\) \\\\(2^159 < q < 2^160\\\\)\n3. Find an element \\\\(\\alpha\\\\) with \\\\( ord(\\alpha) = q\\\\), i.e., \\alpha genertes the subgroup with \\\\(q\\\\) elements.\n4. choose a random integer \\\\(d\\\\) with \\\\(0 < d < q\\\\).\n5. compute \\\\(\\beta \\equiv \\alpha^{d} \\bmod p\\\\).\nthe keys are now:\n\\\\(k_{pub} = (p, q, \\alpha, \\beta)\\\\)\n\\\\(k_{pr}= (d)\\\\)\n***\nThe central idea of DSA is that there are two cyclic groups involved. One is the large cyclic group \\\\(Z_{p}*{\\ast}\\\\), the order of which has bit length of 1024 bit. The second one is in the 160-bit subgroup of \\\\(Z_{p}^{\\ast}\\\\). this set-up yields shorter signature.\n\n### Signature and Verification\nAs in the Elgamal signatue scheme, the DSA signature consists of a pair of integers \\\\((r,s)\\\\). Since each of the two parameters is only 160-bit long, the total signature length is 320 bit. \n***\n**DSA signature generation**\n1. choose an integer as random ephemeral key \\\\(k_{E}\\\\) with \\\\(0 < k_{E} < q\\\\).\n2. compute \\\\(r \\equiv (\\alpha^{k_{E}} \\bmod p) \\bmod q\\\\)\n3. compute \\\\(s \\equiv (SHA(x) + d \\cdot r)k_{E}^{-1} \\bmod q \\\\)\n***\n\nThe signature verification process is as follows:\n***\n**DSA signature verification**\n1. compute auxilary value \\\\(w \\equiv s^{-1} \\bmod q\\\\).\n2. compute auxilary value \\\\(u_{1} \\equiv w \\cdot SHA(x) \\bmod q\\\\).\n3. compute auxilary value \\\\(u_{2} \\equiv w \\cdot r \\bmod q\\\\).\n4. compute \\\\(v \\equiv (\\alpha^{u_1} \\cdot \\beta^{u_2} \\bmod p) \\bmod q\\\\).\n5. the verification \\\\(ver_{k_{pub}}(x, (r,s))\\\\) folows from\n\\begin{equation}\nv = \n\\begin{cases}\n\\equiv r \\bmod q & => \\text{valid signature} \\cr\n\\not\\equiv r \\bmod q & => \\text{invalid signature}\n\\end{cases}\n\\end{equation}\n***\n\n## Elliptic Curve Digital Signature Algorithm (ECDSA)\nElliptic curves have several advantages over RSA and over DL schemes like Elgamal or DSA. In particular, in abscence of strong attacks against elliptic curve cryptosystems (ECC), bit lengths in the range of 160-256 bit can be chosen which provide security equivalent to 1024-3072 bit RSA and DL scheme. The shorter bit length of ECC often results in shorter processing time and in shorter signatures. \nThe ECDSA standard is defined for elliptic curves over prime fields \\\\(Z_{p}\\\\) adn Galois fields \\\\(GF(2^m)\\\\). the former is often preferred in practice, and we only introduce this one in what follows\n### key generation\n***\n**Key Generation for ECDSA**\n1. Use and elliptic curve E with \n- modulus p\n- coefficients a and b\n- a point A which generates a cyclic group of prime order q.\n2. choose a random integer d with \\\\(0 < d < q\\\\)\n3. compute \\\\(B = dA\\\\).\nThe keys are now\n\\\\(k_{pub} = (p,a,b,q,A,B)\\\\)\n\\\\(k_{pr} = (d)\\\\)\n***\n\n### Signature and Verification\n***\n**ECDSA Signature Generation**\n1. choose an integer as random ephemeral key \\\\(k_{E}\\\\) with \\\\( 0 < k_{E} < q\\\\).\n2. compute \\\\(R = k_{E}A\\\\)\n3. Let \\\\(r = x_{R}\\\\)\n4. compute \\\\(s \\equiv (h(x) + d \\cdot r)k_{E}^{-1} \\bmod q\\\\)\n***\nthe signature verification process is as follows\n***\n**ECDSA Signature Verification**\n1. Compute auxiliary value \\\\(w \\equiv s^{-1} \\bmod q\\\\)\n2. compute auxilary value \\\\(u_1 \\equiv w \\cdot h(x) \\bmod q\\\\)\n3. compute auxiliary value \\\\(u_2 = w \\cdot r \\bmod q\\\\)\n4. compute \\\\(P = u_1 A + u_2 B\\\\).\n5. the verification \\\\(ver{k_{pub}}(x, (r,s))\\\\) follows from\n\\begin{equation}\nx_{P} = \n\\begin{cases}\n\\equiv r \\bmod q & => \\text{valid signature} \\cr\n\\not\\equiv r \\bmod q & => \\text{invalid signature}\n\\end{cases}\n\\end{equation}\n***\nThe point multiplication, which is in most cases by the far the most arithmetic intensive operation, can be precomputed by choosing the ephemeral key ahead of time.","slug":"cryptography/cryptography-04-digital-signature","published":1,"updated":"2023-11-05T04:21:17.033Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8do000dqwsj2vh07e7k","content":"\n\n\n

Elgamal Digital Signature Scheme

The Elgammal signature scheme is based on the difficulty of computing discrete logarithms. Unlike RSA, where encryption and digital signature are almoste identical operations, the Elgamal digital signature is quite different from the encryption scheme with teh same name.

\n

key generation

\n
    \n
  1. Choose a large prime \\(p\\).
    2. \n
  2. Choose a primitive element \\(\\alpha\\) of \\(Z_{p}^{\\ast}\\), or a subgroup of \\(Z_{p}^{\\ast}\\).
    3. \n
  3. Choose a random integer \\(d \\in {2,3,…,p-2}\\)
    4. \n
  4. Compute \\(\\beta = \\alpha^{d} \\bmod p\\)
    5. \n
\n
\n

The public key is now formed by \\(k_{pub} = (p, \\alpha, \\beta)\\), and the private key by \\(k_{pr}=d\\)
\\(Z_{p}^{\\ast}\\) is the set of integers who are smaller than \\(p\\) and coprime to \\(p\\)

\n

signature and verification

Usign the private ey and parameters of the public key, the signature
\\[sig_{k_{pr}}(x, k_{E}) = (r,s)\\]
\\(x\\) is the message. \\(k_{E}\\) is a random value, which forms an ephemeral private key

\n
\n

Elgamal Signature Generation

\n
    \n
  1. choose a random ephemeral key \\(k_{E} \\in {0,1,2,..,p-2}\\) such that \\(gcd(k_{E}, p-1) = 1\\)
    2. \n
  2. compute the signatue parameters:
    \\[r \\equiv \\alpha^{k_{E}} \\bmod p\\]
    \\[s \\equiv (x - d \\cdot r)k_{E}^{-1} \\bmod p-1\\]
    3. \n
\n
\n

on the receiving side, the signature is verified as \\(ver_{k_{pub}}(x,(r,s))\\) using the public key, the signature and the message.

\n
\n

Elgamal Signature Verification

\n
    \n
  1. comput the value
    \\[t \\equiv \\beta^{r} \\cdot r^s \\bmod p\\]
    2. \n
  2. the verification follows form
    \\begin{equation}
    t =
    \\begin{cases}
    \\equiv \\alpha^{x} \\bmod p & => \\text{valid signature} \\cr
    \\not\\equiv \\alpha^{x} \\bmod p & => \\text{invalid signature}
    \\end{cases}
    \\end{equation}
    3. \n
\n
\n

Digital Signature Algorithm (DSA)

The native Elgamal signature algorithm described above is rarely used in practice. Instead, a much more popular variant is used, known as the Digital Signature Algorithm (DSA). It is a federal US government standard for digital signatures and was proposed by NIST (National Institute of Standards and Technology). Its main advantages over the Elgamal signature scheme are that the signature is only 320-bit long and that some of the attacks
that can threaten the Elgamal scheme are not applicable.

\n

key generation

\n

key Generation for DSA

\n
    \n
  1. Generate a prime \\(p\\) with \\(2^1023 < p < 2^1024\\)
    2. \n
  2. find a prime divisor \\(q\\) of \\(p-1\\) \\(2^159 < q < 2^160\\)
    3. \n
  3. Find an element \\(\\alpha\\) with \\( ord(\\alpha) = q\\), i.e., \\alpha genertes the subgroup with \\(q\\) elements.
    4. \n
  4. choose a random integer \\(d\\) with \\(0 < d < q\\).
    5. \n
  5. compute \\(\\beta \\equiv \\alpha^{d} \\bmod p\\).
    the keys are now:
    \\(k_{pub} = (p, q, \\alpha, \\beta)\\)
    \\(k_{pr}= (d)\\)
    6. \n
\n
\n

The central idea of DSA is that there are two cyclic groups involved. One is the large cyclic group \\(Z_{p}*{\\ast}\\), the order of which has bit length of 1024 bit. The second one is in the 160-bit subgroup of \\(Z_{p}^{\\ast}\\). this set-up yields shorter signature.

\n

Signature and Verification

As in the Elgamal signatue scheme, the DSA signature consists of a pair of integers \\((r,s)\\). Since each of the two parameters is only 160-bit long, the total signature length is 320 bit.

\n
\n

DSA signature generation

\n
    \n
  1. choose an integer as random ephemeral key \\(k_{E}\\) with \\(0 < k_{E} < q\\).
    2. \n
  2. compute \\(r \\equiv (\\alpha^{k_{E}} \\bmod p) \\bmod q\\)
    3. \n
  3. compute \\(s \\equiv (SHA(x) + d \\cdot r)k_{E}^{-1} \\bmod q \\)
    4. \n
\n
\n

The signature verification process is as follows:

\n
\n

DSA signature verification

\n
    \n
  1. compute auxilary value \\(w \\equiv s^{-1} \\bmod q\\).
    2. \n
  2. compute auxilary value \\(u_{1} \\equiv w \\cdot SHA(x) \\bmod q\\).
    3. \n
  3. compute auxilary value \\(u_{2} \\equiv w \\cdot r \\bmod q\\).
    4. \n
  4. compute \\(v \\equiv (\\alpha^{u_1} \\cdot \\beta^{u_2} \\bmod p) \\bmod q\\).
    5. \n
  5. the verification \\(ver_{k_{pub}}(x, (r,s))\\) folows from
    \\begin{equation}
    v =
    \\begin{cases}
    \\equiv r \\bmod q & => \\text{valid signature} \\cr
    \\not\\equiv r \\bmod q & => \\text{invalid signature}
    \\end{cases}
    \\end{equation}
    6. \n
\n
\n

Elliptic Curve Digital Signature Algorithm (ECDSA)

Elliptic curves have several advantages over RSA and over DL schemes like Elgamal or DSA. In particular, in abscence of strong attacks against elliptic curve cryptosystems (ECC), bit lengths in the range of 160-256 bit can be chosen which provide security equivalent to 1024-3072 bit RSA and DL scheme. The shorter bit length of ECC often results in shorter processing time and in shorter signatures.
The ECDSA standard is defined for elliptic curves over prime fields \\(Z_{p}\\) adn Galois fields \\(GF(2^m)\\). the former is often preferred in practice, and we only introduce this one in what follows

\n

key generation

\n

Key Generation for ECDSA

\n
    \n
  1. Use and elliptic curve E with
    2. \n
\n
    \n
  • modulus p
    • \n
  • coefficients a and b
    • \n
  • a point A which generates a cyclic group of prime order q.
    • \n
\n
    \n
  1. choose a random integer d with \\(0 < d < q\\)
    2. \n
  2. compute \\(B = dA\\).
    The keys are now
    \\(k_{pub} = (p,a,b,q,A,B)\\)
    \\(k_{pr} = (d)\\)
    3. \n
\n
\n

Signature and Verification

\n

ECDSA Signature Generation

\n
    \n
  1. choose an integer as random ephemeral key \\(k_{E}\\) with \\( 0 < k_{E} < q\\).
    2. \n
  2. compute \\(R = k_{E}A\\)
    3. \n
  3. Let \\(r = x_{R}\\)
    4. \n
  4. compute \\(s \\equiv (h(x) + d \\cdot r)k_{E}^{-1} \\bmod q\\)
    5. \n
\n
\n

the signature verification process is as follows

\n
\n

ECDSA Signature Verification

\n
    \n
  1. Compute auxiliary value \\(w \\equiv s^{-1} \\bmod q\\)
    2. \n
  2. compute auxilary value \\(u_1 \\equiv w \\cdot h(x) \\bmod q\\)
    3. \n
  3. compute auxiliary value \\(u_2 = w \\cdot r \\bmod q\\)
    4. \n
  4. compute \\(P = u_1 A + u_2 B\\).
    5. \n
  5. the verification \\(ver{k_{pub}}(x, (r,s))\\) follows from
    \\begin{equation}
    x_{P} =
    \\begin{cases}
    \\equiv r \\bmod q & => \\text{valid signature} \\cr
    \\not\\equiv r \\bmod q & => \\text{invalid signature}
    \\end{cases}
    \\end{equation}
    6. \n
\n
\n

The point multiplication, which is in most cases by the far the most arithmetic intensive operation, can be precomputed by choosing the ephemeral key ahead of time.

\n","site":{"data":{}},"excerpt":"","more":"\n\n\n

Elgamal Digital Signature Scheme

The Elgammal signature scheme is based on the difficulty of computing discrete logarithms. Unlike RSA, where encryption and digital signature are almoste identical operations, the Elgamal digital signature is quite different from the encryption scheme with teh same name.

\n

key generation

\n
    \n
  1. Choose a large prime \\(p\\).
    2. \n
  2. Choose a primitive element \\(\\alpha\\) of \\(Z_{p}^{\\ast}\\), or a subgroup of \\(Z_{p}^{\\ast}\\).
    3. \n
  3. Choose a random integer \\(d \\in {2,3,…,p-2}\\)
    4. \n
  4. Compute \\(\\beta = \\alpha^{d} \\bmod p\\)
    5. \n
\n
\n

The public key is now formed by \\(k_{pub} = (p, \\alpha, \\beta)\\), and the private key by \\(k_{pr}=d\\)
\\(Z_{p}^{\\ast}\\) is the set of integers who are smaller than \\(p\\) and coprime to \\(p\\)

\n

signature and verification

Usign the private ey and parameters of the public key, the signature
\\[sig_{k_{pr}}(x, k_{E}) = (r,s)\\]
\\(x\\) is the message. \\(k_{E}\\) is a random value, which forms an ephemeral private key

\n
\n

Elgamal Signature Generation

\n
    \n
  1. choose a random ephemeral key \\(k_{E} \\in {0,1,2,..,p-2}\\) such that \\(gcd(k_{E}, p-1) = 1\\)
    2. \n
  2. compute the signatue parameters:
    \\[r \\equiv \\alpha^{k_{E}} \\bmod p\\]
    \\[s \\equiv (x - d \\cdot r)k_{E}^{-1} \\bmod p-1\\]
    3. \n
\n
\n

on the receiving side, the signature is verified as \\(ver_{k_{pub}}(x,(r,s))\\) using the public key, the signature and the message.

\n
\n

Elgamal Signature Verification

\n
    \n
  1. comput the value
    \\[t \\equiv \\beta^{r} \\cdot r^s \\bmod p\\]
    2. \n
  2. the verification follows form
    \\begin{equation}
    t =
    \\begin{cases}
    \\equiv \\alpha^{x} \\bmod p & => \\text{valid signature} \\cr
    \\not\\equiv \\alpha^{x} \\bmod p & => \\text{invalid signature}
    \\end{cases}
    \\end{equation}
    3. \n
\n
\n

Digital Signature Algorithm (DSA)

The native Elgamal signature algorithm described above is rarely used in practice. Instead, a much more popular variant is used, known as the Digital Signature Algorithm (DSA). It is a federal US government standard for digital signatures and was proposed by NIST (National Institute of Standards and Technology). Its main advantages over the Elgamal signature scheme are that the signature is only 320-bit long and that some of the attacks
that can threaten the Elgamal scheme are not applicable.

\n

key generation

\n

key Generation for DSA

\n
    \n
  1. Generate a prime \\(p\\) with \\(2^1023 < p < 2^1024\\)
    2. \n
  2. find a prime divisor \\(q\\) of \\(p-1\\) \\(2^159 < q < 2^160\\)
    3. \n
  3. Find an element \\(\\alpha\\) with \\( ord(\\alpha) = q\\), i.e., \\alpha genertes the subgroup with \\(q\\) elements.
    4. \n
  4. choose a random integer \\(d\\) with \\(0 < d < q\\).
    5. \n
  5. compute \\(\\beta \\equiv \\alpha^{d} \\bmod p\\).
    the keys are now:
    \\(k_{pub} = (p, q, \\alpha, \\beta)\\)
    \\(k_{pr}= (d)\\)
    6. \n
\n
\n

The central idea of DSA is that there are two cyclic groups involved. One is the large cyclic group \\(Z_{p}*{\\ast}\\), the order of which has bit length of 1024 bit. The second one is in the 160-bit subgroup of \\(Z_{p}^{\\ast}\\). this set-up yields shorter signature.

\n

Signature and Verification

As in the Elgamal signatue scheme, the DSA signature consists of a pair of integers \\((r,s)\\). Since each of the two parameters is only 160-bit long, the total signature length is 320 bit.

\n
\n

DSA signature generation

\n
    \n
  1. choose an integer as random ephemeral key \\(k_{E}\\) with \\(0 < k_{E} < q\\).
    2. \n
  2. compute \\(r \\equiv (\\alpha^{k_{E}} \\bmod p) \\bmod q\\)
    3. \n
  3. compute \\(s \\equiv (SHA(x) + d \\cdot r)k_{E}^{-1} \\bmod q \\)
    4. \n
\n
\n

The signature verification process is as follows:

\n
\n

DSA signature verification

\n
    \n
  1. compute auxilary value \\(w \\equiv s^{-1} \\bmod q\\).
    2. \n
  2. compute auxilary value \\(u_{1} \\equiv w \\cdot SHA(x) \\bmod q\\).
    3. \n
  3. compute auxilary value \\(u_{2} \\equiv w \\cdot r \\bmod q\\).
    4. \n
  4. compute \\(v \\equiv (\\alpha^{u_1} \\cdot \\beta^{u_2} \\bmod p) \\bmod q\\).
    5. \n
  5. the verification \\(ver_{k_{pub}}(x, (r,s))\\) folows from
    \\begin{equation}
    v =
    \\begin{cases}
    \\equiv r \\bmod q & => \\text{valid signature} \\cr
    \\not\\equiv r \\bmod q & => \\text{invalid signature}
    \\end{cases}
    \\end{equation}
    6. \n
\n
\n

Elliptic Curve Digital Signature Algorithm (ECDSA)

Elliptic curves have several advantages over RSA and over DL schemes like Elgamal or DSA. In particular, in abscence of strong attacks against elliptic curve cryptosystems (ECC), bit lengths in the range of 160-256 bit can be chosen which provide security equivalent to 1024-3072 bit RSA and DL scheme. The shorter bit length of ECC often results in shorter processing time and in shorter signatures.
The ECDSA standard is defined for elliptic curves over prime fields \\(Z_{p}\\) adn Galois fields \\(GF(2^m)\\). the former is often preferred in practice, and we only introduce this one in what follows

\n

key generation

\n

Key Generation for ECDSA

\n
    \n
  1. Use and elliptic curve E with
    2. \n
\n
    \n
  • modulus p
    • \n
  • coefficients a and b
    • \n
  • a point A which generates a cyclic group of prime order q.
    • \n
\n
    \n
  1. choose a random integer d with \\(0 < d < q\\)
    2. \n
  2. compute \\(B = dA\\).
    The keys are now
    \\(k_{pub} = (p,a,b,q,A,B)\\)
    \\(k_{pr} = (d)\\)
    3. \n
\n
\n

Signature and Verification

\n

ECDSA Signature Generation

\n
    \n
  1. choose an integer as random ephemeral key \\(k_{E}\\) with \\( 0 < k_{E} < q\\).
    2. \n
  2. compute \\(R = k_{E}A\\)
    3. \n
  3. Let \\(r = x_{R}\\)
    4. \n
  4. compute \\(s \\equiv (h(x) + d \\cdot r)k_{E}^{-1} \\bmod q\\)
    5. \n
\n
\n

the signature verification process is as follows

\n
\n

ECDSA Signature Verification

\n
    \n
  1. Compute auxiliary value \\(w \\equiv s^{-1} \\bmod q\\)
    2. \n
  2. compute auxilary value \\(u_1 \\equiv w \\cdot h(x) \\bmod q\\)
    3. \n
  3. compute auxiliary value \\(u_2 = w \\cdot r \\bmod q\\)
    4. \n
  4. compute \\(P = u_1 A + u_2 B\\).
    5. \n
  5. the verification \\(ver{k_{pub}}(x, (r,s))\\) follows from
    \\begin{equation}
    x_{P} =
    \\begin{cases}
    \\equiv r \\bmod q & => \\text{valid signature} \\cr
    \\not\\equiv r \\bmod q & => \\text{invalid signature}
    \\end{cases}
    \\end{equation}
    6. \n
\n
\n

The point multiplication, which is in most cases by the far the most arithmetic intensive operation, can be precomputed by choosing the ephemeral key ahead of time.

\n"},{"title":"cryptography (2) RSA cryptosystem","date":"2023-06-10T06:29:26.000Z","_content":"\n\n\n\n## introduction\nthe security of RSA relies on the difficulty of factoring a product of two large primes(the integer factorization problem). it is firstly presented in 1978 in [1].\n\n## Euclidean Algorithm\nthe gcd of two positive integers \\\\(r_0\\\\) and \\\\(r_1\\\\) is denoted by \n\\\\[gcd(r_0, r_1)\\\\]\nthe Eucliedean algorithm is used to calculate gcd, based on as simple observation that\n\\\\[gcd(r_0, r_1) = gcd(r_0-r_1, r_1)\\\\]\nwhere we assume \\\\(r_0 > r_1\\\\)\nThis property can easily be proven: Let \\\\(gcd(r_0, r_1) = g\\\\), since \\\\(g\\\\) divides both \\\\(r_0\\\\) and \\\\(r_1\\\\), we can write \\\\(r_0 = g \\cdot x\\\\) and \\\\(r_1 = g \\cdot y\\\\), where \\\\(x>y\\\\) and \\\\(x\\\\) and \\\\(y\\\\) are coprime integers, i.e., they do not have common factors. Moreover, it is easy to show that \\\\((x-y)\\\\) and \\\\(y\\\\) are also coprime. it follows from here that\n\\\\[gcd(r_0 - r_1, r_1) = gcd(g \\cdot (x-y), g\\cdot y) = g\\\\]\n\nit also follow immediately that we can apply the process iteratively:\n\\\\[gcd(r_0 - r_1, r_1) = gcd(r_0 - mr_1, r_1) \\\\]\nas long as \\\\((r_0 - mr_1) >0\\\\). the algorithm use the fewest number of steps if we choose maximum value of \\\\(m\\\\). this is the case if we compute:\n\\\\[gcd(r_0, r_1) = gcd( r_1, r_0 \\bmod r_1) \\\\]\nthis process can be applied recursively until we obtain finally \\\\[gcd(r_l, 0) = r_l \\\\]\nthe euclidean algorithm is very efficient, even with the very long numbers. the number of iterations is close to the number of digits of the input operands. that means, for instance, that the number of iterations of a gcd involving 1024-bit nubmers is 1024 times a constant.\n## Extended Euclidean Algorithm\nan extension of the Euclidean algorithm allows us to compute ***modular inverse***. in addition to computing the gcd, the ***extended Euclidean algorithm*** computes a linear combination of the form\n\\\\[gcd(r_0, r_1) = s \\cdot r_0 + t\\cdot r_1 \\\\]\nwhere s and t are integer coefficients. this equation is ofthen referred to as ***Diophantine equation***\n\nthe detail of the algorithm can be foud in section 6.3.2 of the book understanding cryptography by Christof Paar. Here presents the general idea by using an example.\nlet \\\\(r_0 =973 , r_1 = 301\\\\). during the steps of Euclidean Algorithm, we obtain \\\\(973 = 3\\cdot301 + 70\\\\)\nwhich is \\\\[r_0 = q_1 \\cdot r_1 + r_2\\\\] \nrearrange:\n\\\\[r_2 = r_0 + (-q_1) \\cdot r_1\\\\] \nreplacing (r_0, r_1) and iteratively by (r_1, r_2), ... (r_{i-1}, r_{i}), util \\\\(r_{i+1} = 0\\\\)\nthen \\\\(r_{i}\\\\) is \\\\(gcd(r_0,r_1)\\\\), and can have a representation of \n\\\\[gcd(r_0, r_1) = s\\cdot r_0 + t\\cdot r_1 \\\\]. \nsince the inverse only exists if \\\\(gcd(r_0, r_1)=1\\\\). we obtain\n\\\\[ s\\cdot r_0 + t\\cdot r_1 = 1\\\\]\ntaking this equation modulo \\\\(r_0\\\\) we obtain\n\\\\[ s\\cdot 0 + t\\cdot r_1 \\equiv 1 \\bmod r_0\\\\]\n\\\\[ t\\cdot r_1 \\equiv 1 \\bmod r_0\\\\]\nt is the definition of the inverse of \\\\(r_1\\\\)\n\nThus, if we need to compute an inverse \\\\(a^{-1} \\bmod m\\\\), we apply EEA with the input parameters \\\\(m\\\\) and \\\\(a\\\\)\n## Euler's Phi Function\nwe consider the ring \\\\( Z_m\\\\) i.e., the set of integers \\\\({0,1,...,m-1}\\\\). we are interested in teh problem of knowing how many numbers in this set are relatively prime to m. this quantity is given by Euler's phi function, which is \\\\(\\Phi(m)\\\\)\n\n***\nlet m have the following canonical factorization\n\\\\[ m = p_{1}^{e_1} \\cdot p_{2}^{e_2} \\cdot ... \\cdot p_{n}^{e_n}\\\\]\nwhere the \\\\(p_i\\\\) are distinct prime numbers and \\\\( e_i\\\\) are positive integers, then\n\n\\\\[ \\Phi(m) = \\prod_{i=1}^{n}(p_{i}^{e_i} - p_{i}^{e_i -1} ) \\\\]\n***\nit is important to stress that we need to know the factoorization of m in order to calculate Euler's phi function.\n\n## Fermat's little theorem\nFermat's little theorem states that if p is a prime number, then for any integer a, the number \n\\\\(a^{p}-a \\\\) is an integer multiple of p. In the notation of modular arithmetic, this is expressed as\n\\\\[ a^{p} \\equiv a \\bmod p\\\\]\nthe theorem can be stated in the form also,\n\\\\[ a^{p-1} \\equiv 1 \\bmod p\\\\]\nthen the inverse of an integer is,\n\\\\[ a^{-1} \\equiv a^{p-2} \\bmod p\\\\]\nperforming the above formulate (involving exponentiation) to find inverse is usually slower than using extended Euclidean algorithm. However, there are situations where it is advantageous to use Fermat's Little Theorem, e.g., on smart cards or other devices which have a hardware accelerator for fast exponentiation anyway.\n\na generatlization of Fermat's little Theorem to any integer moduli, i.e., moduli that are not necessarily primes, is Euler's theorem.\n***\n**Euler's Theorem**\nlet \\\\(a\\\\) and \\\\(m\\\\) be integers with \\\\(gcd(a,m) = 1\\\\), then\n\\\\[ a^{\\Phi(m)} \\equiv 1 \\bmod m\\\\]\n***\nsince it works modulo m, it is applicable to integer rings \\\\(Z_{m}\\\\)\n\n## key generation\n***\n**Output**: public key: \\\\( k_{pub} = (n,e) and private key: k_{pr} = (d) \\\\)\n1. choose two large primes p and q.\n2. compute \\\\(n = p\\cdot q\\\\)\n3. compute \\\\( \\Phi(n) = (p-1)(q-1)\\\\)\n4. select the public exponent \\\\( e \\in {1,2,...,\\Phi(n)-1} \\\\) such that \n\\\\[ gcd(e,\\Phi(n)) = 1\\\\]\n5. compute the private key d such that\n\\\\[ d \\cdot e \\equiv 1 \\bmod \\Phi(n)\\\\]\n***\nthe condition that \\\\( gcd(e,\\Phi(n)) = 1\\\\) ensures that the inverse of \\\\(e\\\\) exists modulo \\\\(\\Phi(n)\\\\), so that there is always a private key \\\\(d\\\\).\nthe computation of key keys \\\\(d\\\\) and \\\\(e\\\\) canb e doen at once using the extended Euclidean algorith. \n\n\n## Encryption and Decryption\n\n***\n**RSA Encryption** Given the privaate key \\\\( k_{pub} = (n,e) \\\\) and the plaintext \\\\(x\\\\), the encryption is:\n\\\\[ y = e_{k_{pub}}(x) \\equiv x^{e} \\bmod n\\\\]\nwhere \\\\(x,y \\in Z_{n}\\\\)\n***\n***\n**RSA Decryption** Given the public key \\\\d = k_{pr} \\\\) and the plaintext \\\\(y\\\\), the decryption is:\n\\\\[ x = d_{k_{pr}}(y) \\equiv y^{d} \\bmod n\\\\]\nwhere \\\\(x,y \\in Z_{n}\\\\)\n***\n\n## Digital signature\nthe message \\\\(x\\\\) that is being signed is in the range \\\\(1,2,...,n-1\\\\)\n![rsa digital signature](/images/cryptography/rsa/rsa_signature.png)\n## references\n- [1] [A Method for Obtaining Digital\nSignatures and Public-Key Cryptosystems](https://web.williams.edu/Mathematics/lg5/302/RSA.pdf) ","source":"_posts/cryptography/cryptography-02-rsa.md","raw":"---\ntitle: cryptography (2) RSA cryptosystem\ndate: 2023-06-10 14:29:26\ntags: [cryptography]\n---\n\n\n\n\n## introduction\nthe security of RSA relies on the difficulty of factoring a product of two large primes(the integer factorization problem). it is firstly presented in 1978 in [1].\n\n## Euclidean Algorithm\nthe gcd of two positive integers \\\\(r_0\\\\) and \\\\(r_1\\\\) is denoted by \n\\\\[gcd(r_0, r_1)\\\\]\nthe Eucliedean algorithm is used to calculate gcd, based on as simple observation that\n\\\\[gcd(r_0, r_1) = gcd(r_0-r_1, r_1)\\\\]\nwhere we assume \\\\(r_0 > r_1\\\\)\nThis property can easily be proven: Let \\\\(gcd(r_0, r_1) = g\\\\), since \\\\(g\\\\) divides both \\\\(r_0\\\\) and \\\\(r_1\\\\), we can write \\\\(r_0 = g \\cdot x\\\\) and \\\\(r_1 = g \\cdot y\\\\), where \\\\(x>y\\\\) and \\\\(x\\\\) and \\\\(y\\\\) are coprime integers, i.e., they do not have common factors. Moreover, it is easy to show that \\\\((x-y)\\\\) and \\\\(y\\\\) are also coprime. it follows from here that\n\\\\[gcd(r_0 - r_1, r_1) = gcd(g \\cdot (x-y), g\\cdot y) = g\\\\]\n\nit also follow immediately that we can apply the process iteratively:\n\\\\[gcd(r_0 - r_1, r_1) = gcd(r_0 - mr_1, r_1) \\\\]\nas long as \\\\((r_0 - mr_1) >0\\\\). the algorithm use the fewest number of steps if we choose maximum value of \\\\(m\\\\). this is the case if we compute:\n\\\\[gcd(r_0, r_1) = gcd( r_1, r_0 \\bmod r_1) \\\\]\nthis process can be applied recursively until we obtain finally \\\\[gcd(r_l, 0) = r_l \\\\]\nthe euclidean algorithm is very efficient, even with the very long numbers. the number of iterations is close to the number of digits of the input operands. that means, for instance, that the number of iterations of a gcd involving 1024-bit nubmers is 1024 times a constant.\n## Extended Euclidean Algorithm\nan extension of the Euclidean algorithm allows us to compute ***modular inverse***. in addition to computing the gcd, the ***extended Euclidean algorithm*** computes a linear combination of the form\n\\\\[gcd(r_0, r_1) = s \\cdot r_0 + t\\cdot r_1 \\\\]\nwhere s and t are integer coefficients. this equation is ofthen referred to as ***Diophantine equation***\n\nthe detail of the algorithm can be foud in section 6.3.2 of the book understanding cryptography by Christof Paar. Here presents the general idea by using an example.\nlet \\\\(r_0 =973 , r_1 = 301\\\\). during the steps of Euclidean Algorithm, we obtain \\\\(973 = 3\\cdot301 + 70\\\\)\nwhich is \\\\[r_0 = q_1 \\cdot r_1 + r_2\\\\] \nrearrange:\n\\\\[r_2 = r_0 + (-q_1) \\cdot r_1\\\\] \nreplacing (r_0, r_1) and iteratively by (r_1, r_2), ... (r_{i-1}, r_{i}), util \\\\(r_{i+1} = 0\\\\)\nthen \\\\(r_{i}\\\\) is \\\\(gcd(r_0,r_1)\\\\), and can have a representation of \n\\\\[gcd(r_0, r_1) = s\\cdot r_0 + t\\cdot r_1 \\\\]. \nsince the inverse only exists if \\\\(gcd(r_0, r_1)=1\\\\). we obtain\n\\\\[ s\\cdot r_0 + t\\cdot r_1 = 1\\\\]\ntaking this equation modulo \\\\(r_0\\\\) we obtain\n\\\\[ s\\cdot 0 + t\\cdot r_1 \\equiv 1 \\bmod r_0\\\\]\n\\\\[ t\\cdot r_1 \\equiv 1 \\bmod r_0\\\\]\nt is the definition of the inverse of \\\\(r_1\\\\)\n\nThus, if we need to compute an inverse \\\\(a^{-1} \\bmod m\\\\), we apply EEA with the input parameters \\\\(m\\\\) and \\\\(a\\\\)\n## Euler's Phi Function\nwe consider the ring \\\\( Z_m\\\\) i.e., the set of integers \\\\({0,1,...,m-1}\\\\). we are interested in teh problem of knowing how many numbers in this set are relatively prime to m. this quantity is given by Euler's phi function, which is \\\\(\\Phi(m)\\\\)\n\n***\nlet m have the following canonical factorization\n\\\\[ m = p_{1}^{e_1} \\cdot p_{2}^{e_2} \\cdot ... \\cdot p_{n}^{e_n}\\\\]\nwhere the \\\\(p_i\\\\) are distinct prime numbers and \\\\( e_i\\\\) are positive integers, then\n\n\\\\[ \\Phi(m) = \\prod_{i=1}^{n}(p_{i}^{e_i} - p_{i}^{e_i -1} ) \\\\]\n***\nit is important to stress that we need to know the factoorization of m in order to calculate Euler's phi function.\n\n## Fermat's little theorem\nFermat's little theorem states that if p is a prime number, then for any integer a, the number \n\\\\(a^{p}-a \\\\) is an integer multiple of p. In the notation of modular arithmetic, this is expressed as\n\\\\[ a^{p} \\equiv a \\bmod p\\\\]\nthe theorem can be stated in the form also,\n\\\\[ a^{p-1} \\equiv 1 \\bmod p\\\\]\nthen the inverse of an integer is,\n\\\\[ a^{-1} \\equiv a^{p-2} \\bmod p\\\\]\nperforming the above formulate (involving exponentiation) to find inverse is usually slower than using extended Euclidean algorithm. However, there are situations where it is advantageous to use Fermat's Little Theorem, e.g., on smart cards or other devices which have a hardware accelerator for fast exponentiation anyway.\n\na generatlization of Fermat's little Theorem to any integer moduli, i.e., moduli that are not necessarily primes, is Euler's theorem.\n***\n**Euler's Theorem**\nlet \\\\(a\\\\) and \\\\(m\\\\) be integers with \\\\(gcd(a,m) = 1\\\\), then\n\\\\[ a^{\\Phi(m)} \\equiv 1 \\bmod m\\\\]\n***\nsince it works modulo m, it is applicable to integer rings \\\\(Z_{m}\\\\)\n\n## key generation\n***\n**Output**: public key: \\\\( k_{pub} = (n,e) and private key: k_{pr} = (d) \\\\)\n1. choose two large primes p and q.\n2. compute \\\\(n = p\\cdot q\\\\)\n3. compute \\\\( \\Phi(n) = (p-1)(q-1)\\\\)\n4. select the public exponent \\\\( e \\in {1,2,...,\\Phi(n)-1} \\\\) such that \n\\\\[ gcd(e,\\Phi(n)) = 1\\\\]\n5. compute the private key d such that\n\\\\[ d \\cdot e \\equiv 1 \\bmod \\Phi(n)\\\\]\n***\nthe condition that \\\\( gcd(e,\\Phi(n)) = 1\\\\) ensures that the inverse of \\\\(e\\\\) exists modulo \\\\(\\Phi(n)\\\\), so that there is always a private key \\\\(d\\\\).\nthe computation of key keys \\\\(d\\\\) and \\\\(e\\\\) canb e doen at once using the extended Euclidean algorith. \n\n\n## Encryption and Decryption\n\n***\n**RSA Encryption** Given the privaate key \\\\( k_{pub} = (n,e) \\\\) and the plaintext \\\\(x\\\\), the encryption is:\n\\\\[ y = e_{k_{pub}}(x) \\equiv x^{e} \\bmod n\\\\]\nwhere \\\\(x,y \\in Z_{n}\\\\)\n***\n***\n**RSA Decryption** Given the public key \\\\d = k_{pr} \\\\) and the plaintext \\\\(y\\\\), the decryption is:\n\\\\[ x = d_{k_{pr}}(y) \\equiv y^{d} \\bmod n\\\\]\nwhere \\\\(x,y \\in Z_{n}\\\\)\n***\n\n## Digital signature\nthe message \\\\(x\\\\) that is being signed is in the range \\\\(1,2,...,n-1\\\\)\n![rsa digital signature](/images/cryptography/rsa/rsa_signature.png)\n## references\n- [1] [A Method for Obtaining Digital\nSignatures and Public-Key Cryptosystems](https://web.williams.edu/Mathematics/lg5/302/RSA.pdf) ","slug":"cryptography/cryptography-02-rsa","published":1,"updated":"2023-11-05T04:21:17.033Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8do000gqwsj13yvh382","content":"\n\n\n\n

introduction

the security of RSA relies on the difficulty of factoring a product of two large primes(the integer factorization problem). it is firstly presented in 1978 in [1].

\n

Euclidean Algorithm

the gcd of two positive integers \\(r_0\\) and \\(r_1\\) is denoted by
\\[gcd(r_0, r_1)\\]
the Eucliedean algorithm is used to calculate gcd, based on as simple observation that
\\[gcd(r_0, r_1) = gcd(r_0-r_1, r_1)\\]
where we assume \\(r_0 > r_1\\)
This property can easily be proven: Let \\(gcd(r_0, r_1) = g\\), since \\(g\\) divides both \\(r_0\\) and \\(r_1\\), we can write \\(r_0 = g \\cdot x\\) and \\(r_1 = g \\cdot y\\), where \\(x>y\\) and \\(x\\) and \\(y\\) are coprime integers, i.e., they do not have common factors. Moreover, it is easy to show that \\((x-y)\\) and \\(y\\) are also coprime. it follows from here that
\\[gcd(r_0 - r_1, r_1) = gcd(g \\cdot (x-y), g\\cdot y) = g\\]

\n

it also follow immediately that we can apply the process iteratively:
\\[gcd(r_0 - r_1, r_1) = gcd(r_0 - mr_1, r_1) \\]
as long as \\((r_0 - mr_1) >0\\). the algorithm use the fewest number of steps if we choose maximum value of \\(m\\). this is the case if we compute:
\\[gcd(r_0, r_1) = gcd( r_1, r_0 \\bmod r_1) \\]
this process can be applied recursively until we obtain finally \\[gcd(r_l, 0) = r_l \\]
the euclidean algorithm is very efficient, even with the very long numbers. the number of iterations is close to the number of digits of the input operands. that means, for instance, that the number of iterations of a gcd involving 1024-bit nubmers is 1024 times a constant.

\n

Extended Euclidean Algorithm

an extension of the Euclidean algorithm allows us to compute modular inverse. in addition to computing the gcd, the extended Euclidean algorithm computes a linear combination of the form
\\[gcd(r_0, r_1) = s \\cdot r_0 + t\\cdot r_1 \\]
where s and t are integer coefficients. this equation is ofthen referred to as Diophantine equation

\n

the detail of the algorithm can be foud in section 6.3.2 of the book understanding cryptography by Christof Paar. Here presents the general idea by using an example.
let \\(r_0 =973 , r_1 = 301\\). during the steps of Euclidean Algorithm, we obtain \\(973 = 3\\cdot301 + 70\\)
which is \\[r_0 = q_1 \\cdot r_1 + r_2\\]
rearrange:
\\[r_2 = r_0 + (-q_1) \\cdot r_1\\]
replacing (r_0, r_1) and iteratively by (r_1, r_2), … (r_{i-1}, r_{i}), util \\(r_{i+1} = 0\\)
then \\(r_{i}\\) is \\(gcd(r_0,r_1)\\), and can have a representation of
\\[gcd(r_0, r_1) = s\\cdot r_0 + t\\cdot r_1 \\].
since the inverse only exists if \\(gcd(r_0, r_1)=1\\). we obtain
\\[ s\\cdot r_0 + t\\cdot r_1 = 1\\]
taking this equation modulo \\(r_0\\) we obtain
\\[ s\\cdot 0 + t\\cdot r_1 \\equiv 1 \\bmod r_0\\]
\\[ t\\cdot r_1 \\equiv 1 \\bmod r_0\\]
t is the definition of the inverse of \\(r_1\\)

\n

Thus, if we need to compute an inverse \\(a^{-1} \\bmod m\\), we apply EEA with the input parameters \\(m\\) and \\(a\\)

\n

Euler’s Phi Function

we consider the ring \\( Z_m\\) i.e., the set of integers \\({0,1,…,m-1}\\). we are interested in teh problem of knowing how many numbers in this set are relatively prime to m. this quantity is given by Euler’s phi function, which is \\(\\Phi(m)\\)

\n
\n

let m have the following canonical factorization
\\[ m = p_{1}^{e_1} \\cdot p_{2}^{e_2} \\cdot … \\cdot p_{n}^{e_n}\\]
where the \\(p_i\\) are distinct prime numbers and \\( e_i\\) are positive integers, then

\n

\\[ \\Phi(m) = \\prod_{i=1}^{n}(p_{i}^{e_i} - p_{i}^{e_i -1} ) \\]

\n
\n

it is important to stress that we need to know the factoorization of m in order to calculate Euler’s phi function.

\n

Fermat’s little theorem

Fermat’s little theorem states that if p is a prime number, then for any integer a, the number
\\(a^{p}-a \\) is an integer multiple of p. In the notation of modular arithmetic, this is expressed as
\\[ a^{p} \\equiv a \\bmod p\\]
the theorem can be stated in the form also,
\\[ a^{p-1} \\equiv 1 \\bmod p\\]
then the inverse of an integer is,
\\[ a^{-1} \\equiv a^{p-2} \\bmod p\\]
performing the above formulate (involving exponentiation) to find inverse is usually slower than using extended Euclidean algorithm. However, there are situations where it is advantageous to use Fermat’s Little Theorem, e.g., on smart cards or other devices which have a hardware accelerator for fast exponentiation anyway.

\n

a generatlization of Fermat’s little Theorem to any integer moduli, i.e., moduli that are not necessarily primes, is Euler’s theorem.

\n
\n

Euler’s Theorem
let \\(a\\) and \\(m\\) be integers with \\(gcd(a,m) = 1\\), then
\\[ a^{\\Phi(m)} \\equiv 1 \\bmod m\\]

\n
\n

since it works modulo m, it is applicable to integer rings \\(Z_{m}\\)

\n

key generation

\n

Output: public key: \\( k_{pub} = (n,e) and private key: k_{pr} = (d) \\)

\n
    \n
  1. choose two large primes p and q.
    2. \n
  2. compute \\(n = p\\cdot q\\)
    3. \n
  3. compute \\( \\Phi(n) = (p-1)(q-1)\\)
    4. \n
  4. select the public exponent \\( e \\in {1,2,…,\\Phi(n)-1} \\) such that
    \\[ gcd(e,\\Phi(n)) = 1\\]
    5. \n
  5. compute the private key d such that
    \\[ d \\cdot e \\equiv 1 \\bmod \\Phi(n)\\]
    6. \n
\n
\n

the condition that \\( gcd(e,\\Phi(n)) = 1\\) ensures that the inverse of \\(e\\) exists modulo \\(\\Phi(n)\\), so that there is always a private key \\(d\\).
the computation of key keys \\(d\\) and \\(e\\) canb e doen at once using the extended Euclidean algorith.

\n

Encryption and Decryption

\n

RSA Encryption Given the privaate key \\( k_{pub} = (n,e) \\) and the plaintext \\(x\\), the encryption is:
\\[ y = e_{k_{pub}}(x) \\equiv x^{e} \\bmod n\\]
where \\(x,y \\in Z_{n}\\)

\n
\n
\n

RSA Decryption Given the public key \\d = k_{pr} \\) and the plaintext \\(y\\), the decryption is:
\\[ x = d_{k_{pr}}(y) \\equiv y^{d} \\bmod n\\]
where \\(x,y \\in Z_{n}\\)

\n
\n

Digital signature

the message \\(x\\) that is being signed is in the range \\(1,2,…,n-1\\)
\"rsa

\n

references

\n","site":{"data":{}},"excerpt":"","more":"\n\n\n\n

introduction

the security of RSA relies on the difficulty of factoring a product of two large primes(the integer factorization problem). it is firstly presented in 1978 in [1].

\n

Euclidean Algorithm

the gcd of two positive integers \\(r_0\\) and \\(r_1\\) is denoted by
\\[gcd(r_0, r_1)\\]
the Eucliedean algorithm is used to calculate gcd, based on as simple observation that
\\[gcd(r_0, r_1) = gcd(r_0-r_1, r_1)\\]
where we assume \\(r_0 > r_1\\)
This property can easily be proven: Let \\(gcd(r_0, r_1) = g\\), since \\(g\\) divides both \\(r_0\\) and \\(r_1\\), we can write \\(r_0 = g \\cdot x\\) and \\(r_1 = g \\cdot y\\), where \\(x>y\\) and \\(x\\) and \\(y\\) are coprime integers, i.e., they do not have common factors. Moreover, it is easy to show that \\((x-y)\\) and \\(y\\) are also coprime. it follows from here that
\\[gcd(r_0 - r_1, r_1) = gcd(g \\cdot (x-y), g\\cdot y) = g\\]

\n

it also follow immediately that we can apply the process iteratively:
\\[gcd(r_0 - r_1, r_1) = gcd(r_0 - mr_1, r_1) \\]
as long as \\((r_0 - mr_1) >0\\). the algorithm use the fewest number of steps if we choose maximum value of \\(m\\). this is the case if we compute:
\\[gcd(r_0, r_1) = gcd( r_1, r_0 \\bmod r_1) \\]
this process can be applied recursively until we obtain finally \\[gcd(r_l, 0) = r_l \\]
the euclidean algorithm is very efficient, even with the very long numbers. the number of iterations is close to the number of digits of the input operands. that means, for instance, that the number of iterations of a gcd involving 1024-bit nubmers is 1024 times a constant.

\n

Extended Euclidean Algorithm

an extension of the Euclidean algorithm allows us to compute modular inverse. in addition to computing the gcd, the extended Euclidean algorithm computes a linear combination of the form
\\[gcd(r_0, r_1) = s \\cdot r_0 + t\\cdot r_1 \\]
where s and t are integer coefficients. this equation is ofthen referred to as Diophantine equation

\n

the detail of the algorithm can be foud in section 6.3.2 of the book understanding cryptography by Christof Paar. Here presents the general idea by using an example.
let \\(r_0 =973 , r_1 = 301\\). during the steps of Euclidean Algorithm, we obtain \\(973 = 3\\cdot301 + 70\\)
which is \\[r_0 = q_1 \\cdot r_1 + r_2\\]
rearrange:
\\[r_2 = r_0 + (-q_1) \\cdot r_1\\]
replacing (r_0, r_1) and iteratively by (r_1, r_2), … (r_{i-1}, r_{i}), util \\(r_{i+1} = 0\\)
then \\(r_{i}\\) is \\(gcd(r_0,r_1)\\), and can have a representation of
\\[gcd(r_0, r_1) = s\\cdot r_0 + t\\cdot r_1 \\].
since the inverse only exists if \\(gcd(r_0, r_1)=1\\). we obtain
\\[ s\\cdot r_0 + t\\cdot r_1 = 1\\]
taking this equation modulo \\(r_0\\) we obtain
\\[ s\\cdot 0 + t\\cdot r_1 \\equiv 1 \\bmod r_0\\]
\\[ t\\cdot r_1 \\equiv 1 \\bmod r_0\\]
t is the definition of the inverse of \\(r_1\\)

\n

Thus, if we need to compute an inverse \\(a^{-1} \\bmod m\\), we apply EEA with the input parameters \\(m\\) and \\(a\\)

\n

Euler’s Phi Function

we consider the ring \\( Z_m\\) i.e., the set of integers \\({0,1,…,m-1}\\). we are interested in teh problem of knowing how many numbers in this set are relatively prime to m. this quantity is given by Euler’s phi function, which is \\(\\Phi(m)\\)

\n
\n

let m have the following canonical factorization
\\[ m = p_{1}^{e_1} \\cdot p_{2}^{e_2} \\cdot … \\cdot p_{n}^{e_n}\\]
where the \\(p_i\\) are distinct prime numbers and \\( e_i\\) are positive integers, then

\n

\\[ \\Phi(m) = \\prod_{i=1}^{n}(p_{i}^{e_i} - p_{i}^{e_i -1} ) \\]

\n
\n

it is important to stress that we need to know the factoorization of m in order to calculate Euler’s phi function.

\n

Fermat’s little theorem

Fermat’s little theorem states that if p is a prime number, then for any integer a, the number
\\(a^{p}-a \\) is an integer multiple of p. In the notation of modular arithmetic, this is expressed as
\\[ a^{p} \\equiv a \\bmod p\\]
the theorem can be stated in the form also,
\\[ a^{p-1} \\equiv 1 \\bmod p\\]
then the inverse of an integer is,
\\[ a^{-1} \\equiv a^{p-2} \\bmod p\\]
performing the above formulate (involving exponentiation) to find inverse is usually slower than using extended Euclidean algorithm. However, there are situations where it is advantageous to use Fermat’s Little Theorem, e.g., on smart cards or other devices which have a hardware accelerator for fast exponentiation anyway.

\n

a generatlization of Fermat’s little Theorem to any integer moduli, i.e., moduli that are not necessarily primes, is Euler’s theorem.

\n
\n

Euler’s Theorem
let \\(a\\) and \\(m\\) be integers with \\(gcd(a,m) = 1\\), then
\\[ a^{\\Phi(m)} \\equiv 1 \\bmod m\\]

\n
\n

since it works modulo m, it is applicable to integer rings \\(Z_{m}\\)

\n

key generation

\n

Output: public key: \\( k_{pub} = (n,e) and private key: k_{pr} = (d) \\)

\n
    \n
  1. choose two large primes p and q.
    2. \n
  2. compute \\(n = p\\cdot q\\)
    3. \n
  3. compute \\( \\Phi(n) = (p-1)(q-1)\\)
    4. \n
  4. select the public exponent \\( e \\in {1,2,…,\\Phi(n)-1} \\) such that
    \\[ gcd(e,\\Phi(n)) = 1\\]
    5. \n
  5. compute the private key d such that
    \\[ d \\cdot e \\equiv 1 \\bmod \\Phi(n)\\]
    6. \n
\n
\n

the condition that \\( gcd(e,\\Phi(n)) = 1\\) ensures that the inverse of \\(e\\) exists modulo \\(\\Phi(n)\\), so that there is always a private key \\(d\\).
the computation of key keys \\(d\\) and \\(e\\) canb e doen at once using the extended Euclidean algorith.

\n

Encryption and Decryption

\n

RSA Encryption Given the privaate key \\( k_{pub} = (n,e) \\) and the plaintext \\(x\\), the encryption is:
\\[ y = e_{k_{pub}}(x) \\equiv x^{e} \\bmod n\\]
where \\(x,y \\in Z_{n}\\)

\n
\n
\n

RSA Decryption Given the public key \\d = k_{pr} \\) and the plaintext \\(y\\), the decryption is:
\\[ x = d_{k_{pr}}(y) \\equiv y^{d} \\bmod n\\]
where \\(x,y \\in Z_{n}\\)

\n
\n

Digital signature

the message \\(x\\) that is being signed is in the range \\(1,2,…,n-1\\)
\"rsa

\n

references

\n"},{"title":"paillier encryption","date":"2023-02-23T13:25:41.000Z","_content":"\n\n\n\n## fundamentals\n1. fundamental theorem of arighmetic\nthe fundamental theorem of arithmetic, also called the unique factorization theorem and prime factorization theorem, states that every integer greater than 1 can be represented uniquely as a product of prime numbers, up to the order of the factors [wiki](https://en.wikipedia.org/wiki/Fundamental_theorem_of_arithmetic)\n2. Euler's totient function\nIn number theory, Euler's totient function counts the positive integers up to a given integer n that are relatively prime to n. It is written using the Greek letter phi as \\\\( \\phi (n) \\\\), and may also be called Euler's phi function. In other words, it is the number of integers k in the range 1 ≤ k ≤ n for which the greatest common divisor gcd(n, k) is equal to 1. The integers k of this form are sometimes referred to as totatives of n. the collection of k is denoted by \\\\( Z_{n}^{\\ast } \\\\), and \\\\[ \\phi(n) = |Z_n^{\\ast }| \\\\]\n3. if p is prime, then \\\\( Z_p^{\\ast } = Z_p \\\\), \\\\( \\phi(p) = p-1 \\\\)\n4. if p is prime, for any integer r, then \\\\( \\begin{align} \\tag{0.1} \\phi(p^{r}) =p^{r-1}\\phi(p)=p^{r-1}(p-1)\\end{align} \\\\)\n5. Euler's totient function is a multiplicative function, meaning that if two numbers m and n are relatively prime, then \\\\(\\phi(mn) = \\phi(m)\\phi(n)\\\\)\n6. Euler's product formula, it states\n\\\\[ \\phi(n) = n \\prod_{p|n}^{}(1-\\frac{1}{p}) \\\\]\nwhere the product is over the distinct prime numbers dividing n.\n7. Euler's theorem\nif \\\\(a\\\\) and \\\\(n\\\\) are coprime positive integers, and \\\\( \\phi(n)\\\\) is Euler's totient function, then \\\\(a\\\\) raised to the power \\\\(\\phi(n)\\\\) is congruent to 1 modulo n; that is\n\\\\[a^{\\phi(n)} \\equiv 1 \\bmod n\\\\]\n8. according to 7, we have \\\\( a \\cdot a^{\\phi(n)-1} \\equiv 1 \\bmod n \\\\). then\n\\\\[ a^{-1} = a^{\\phi(n)-1} \\\\]\n9. Fermat's little theorem\nFermat's little theorem states that if p is a prime number, then for any integer a, the number \n\\\\(a^{p}-a \\\\) is an integer multiple of p. In the notation of modular arithmetic, this is expressed as\n\\\\[ a^{p} \\equiv a \\bmod p\\\\]\n10. Binomial theorem\nit states\n\\\\[ y = (1+n)^{x} = \\sum_{k=0}^{x}\\tbinom{x}{k}n^{k} = 1 + nx + \\tbinom{x}{2}n^2 + ...\\\\]\nobserve that, the higher degree could be divided by \\\\(n^2\\\\). we have\n\\\\[ \\begin{align} \\tag{0.2} (1+n)^{x} \\equiv 1 + nx \\bmod n^2 \\end{align} \\\\]\ntherefore, \\\\( y - 1 \\equiv nx \\bmod n^2 \\\\). then we have\n\\\\[ x \\equiv \\frac{y-1}{n} \\bmod n \\\\].\nIn paillier, later we define \\\\( \\begin{align} \\tag{0.3} L(y) = \\frac{y-1}{n} \\end{align} \\\\)\ntherefore\n\\\\[ L(y \\bmod n^2) \\equiv x \\bmod n \\\\]\n\n\n## Paillier\n1. key generation\n`KeyGen() -> (pk, sk)`\nrandomly select two big prime numbers \\\\(p, q\\\\). it shoud satisfy \\\\(gcd(pq, (p-1)(q-1)) =1 \\\\), \\\\(p\\\\) and \\\\(q\\\\) should have similar bit length. let \\\\( n = pq \\\\), \\\\(\\lambda = lcm(p-1, q-1)\\\\). randomly sample \\\\( g \\in Z_{n^2}^{\\ast}\\\\). to simplify, let \\\\( g = n+1\\\\). we have\n\\\\[ pk=(n,g) \\\\]\n\\\\[ sk = (\\lambda)\\\\]\n\n2. encryption\n`Enc(pk, m) -> c`\nrandomly sample \\\\( r \\in Z_{n}^{\\ast}\\\\), then also have \\\\( r \\in Z_{n^2}^{\\ast}\\\\), cypher is calculated\n\\\\[ \\begin{align} \\tag{1.1} c = g^mr^n \\bmod n^2 \\end{align} \\\\]\n\n3. Decryption\n`Dec(sk, c) -> m`\nLet \\\\(L(x) = \\frac{x-1}{n} \\\\), we have message\n\\\\[ \\begin{align} \\tag{1.2} m = \\frac{L(c^{\\lambda} \\bmod n^2)}{L(g^{\\lambda} \\bmod n^2)} \\bmod n \\end{align}\\\\]\n\n4. proof of correctness\nbased on Eq(1), we have \\\\[ \\begin{align} \\tag{1.3} c^{\\lambda} \\bmod n^2 = g^{m\\lambda}r^{n\\lambda} \\bmod n^2 \\end{align}\\\\]\nwhere \\\\( r^{n\\lambda} \\bmod n^2 \\equiv 1 \\bmod n^2\\\\), which is proved by Carmichael theorem later on. then Eq(3) becomes\n \\\\[ \\begin{align} \\tag{1.4} c^{\\lambda} \\bmod n^2 = g^{m\\lambda}\\bmod n^2 \\end{align}\\\\]\nsince \\\\( g = n+1\\\\), we have\n\\\\[ \\begin{align} \\tag{1.5} c^{\\lambda} \\bmod n^2 = (1+n)^{m\\lambda}\\bmod n^2 \\end{align}\\\\]\nAccording to Eq(0.2), we have\n\\\\[ \\begin{align} \\tag{1.6} c^{\\lambda} \\bmod n^2 = 1 + nm\\lambda \\bmod n^2 \\end{align}\\\\]\n\\\\[ \\begin{align} \\tag{1.7} g^{\\lambda} \\bmod n^2 \\equiv (1+n)^{\\lambda} \\bmod n^2 = 1 +\\lambda n \\bmod n^2 \\end{align}\\\\]\ntherefore, based on definition given by Eq(0.3) we have\n\\\\[ \\begin{align} \\tag{1.8} L(c^{\\lambda} \\bmod n^2) = \\frac{c^{\\lambda}-1}{n} \\bmod n^2 \\end{align} \\\\]\nSubstitute Eq(1.6) into Eq(1.8), we have\n\\\\[ \\begin{align} \\tag{1.9} L(c^{\\lambda} \\bmod n^2) = m\\lambda \\bmod n^2 \\end{align} \\\\]\nFurther, we have\n\\\\[ \\begin{align} \\tag{1.10} L(g^{\\lambda} \\bmod n^2) = \\frac{g^\\lambda -1}{n} \\end{align} \\\\]\nSub Eq(1.7) into Eq(1.10), we have\n\\\\[ \\begin{align} \\tag{1.11} L(g^{\\lambda} \\bmod n^2) = \\frac{\\lambda n}{n} \\equiv \\lambda \\bmod n^2\\end{align} \\\\]\nAt last, Eq(1.2) becomes （bu sub Eq1.9 and Eq1.11)\n\\\\[ \\begin{align} m = \\frac{L(c^{\\lambda} \\bmod n^2)}{L(g^{\\lambda} \\bmod n^2)} \\bmod n = \\frac{m \\lambda}{\\lambda} \\equiv m \\bmod n \\end{align}\\\\]\nproved!!!\n\n5. Carmichael theorem\nIn number theory, a branch of mathematics, the Carmichael function \\\\(λ(n)\\\\) of a positive integer n is the smallest positive integer m such that \\\\(a^{m}\\equiv 1{\\pmod {n}}\\\\) (similar but different from Euler's totient function). Carmichael's λ function, the reduced totient function, and the least universal exponent function\n![carmichael theorem](/images/paillier/carmichael_thorem.png)\n![](/images/paillier/carmichael_thorem_2.png)\nlet \\\\( n = pq\\\\), where p and q are prime numbers; \\\\( \\phi(n)\\\\) is the Euler's totient function. Let \\\\(\\lambda(n)\\\\) denotes carmichael function. We have \\\\(\\phi(n)=(p-1)(q-1)\\\\) and \\\\( \\lambda(n)=\\phi(n) = (p-1)(q-1)\\\\).\n\nSince \\\\( |Z_{n^2}^{\\ast}| = \\phi(n^2) = n \\phi(n)\\\\) (according to Eq(0.1)). Thereby, for any \\\\( w \\in Z_{n^2}^{\\ast}\\\\)\n\\\\[ \\begin{align} \\tag{1.12} w^{n\\phi(n)} \\equiv w^{n\\\\lambda} \\equiv 1 \\bmod n^2 \\end{align}\\\\]\n\n\\\\[ \\begin{align} \\tag{1.13} w^{\\lambda} \\equiv 1 \\bmod n \\end{align}\\\\]\nEq(1.13) is just Carmichael's function\n\nBased on Carmichael's theorem\n\\\\[ \\lambda(n^2) = lcm(\\lambda(q^2),\\lambda(p^2)) = lcm(\\phi(q^2),\\phi(p^2)) = lcm(q(q-1), p(p-1)) = pq(lcm(p-1, q-1)) = n\\lambda(n) \\\\] \ntherefore, we have\n\n\\\\[w^{\\lambda(n^2)} = w ^{n\\lambda} \\equiv 1 \\bmod n^2\\\\]\n\n6. Addition homomorphic\n![homomorphic addition](/images/paillier/homomorphic_addition.png)\n\n7. Multiplication homomorphic \n![homomorphic multiplication](/images/paillier/homomorphic_mul.png)\n## references\n- [csdn post](https://blog.csdn.net/qq_42328228/article/details/109349590)","source":"_posts/cryptography/paillier-encryption.md","raw":"---\ntitle: paillier encryption\ndate: 2023-02-23 21:25:41\ntags: [cryptography]\n---\n\n\n\n\n## fundamentals\n1. fundamental theorem of arighmetic\nthe fundamental theorem of arithmetic, also called the unique factorization theorem and prime factorization theorem, states that every integer greater than 1 can be represented uniquely as a product of prime numbers, up to the order of the factors [wiki](https://en.wikipedia.org/wiki/Fundamental_theorem_of_arithmetic)\n2. Euler's totient function\nIn number theory, Euler's totient function counts the positive integers up to a given integer n that are relatively prime to n. It is written using the Greek letter phi as \\\\( \\phi (n) \\\\), and may also be called Euler's phi function. In other words, it is the number of integers k in the range 1 ≤ k ≤ n for which the greatest common divisor gcd(n, k) is equal to 1. The integers k of this form are sometimes referred to as totatives of n. the collection of k is denoted by \\\\( Z_{n}^{\\ast } \\\\), and \\\\[ \\phi(n) = |Z_n^{\\ast }| \\\\]\n3. if p is prime, then \\\\( Z_p^{\\ast } = Z_p \\\\), \\\\( \\phi(p) = p-1 \\\\)\n4. if p is prime, for any integer r, then \\\\( \\begin{align} \\tag{0.1} \\phi(p^{r}) =p^{r-1}\\phi(p)=p^{r-1}(p-1)\\end{align} \\\\)\n5. Euler's totient function is a multiplicative function, meaning that if two numbers m and n are relatively prime, then \\\\(\\phi(mn) = \\phi(m)\\phi(n)\\\\)\n6. Euler's product formula, it states\n\\\\[ \\phi(n) = n \\prod_{p|n}^{}(1-\\frac{1}{p}) \\\\]\nwhere the product is over the distinct prime numbers dividing n.\n7. Euler's theorem\nif \\\\(a\\\\) and \\\\(n\\\\) are coprime positive integers, and \\\\( \\phi(n)\\\\) is Euler's totient function, then \\\\(a\\\\) raised to the power \\\\(\\phi(n)\\\\) is congruent to 1 modulo n; that is\n\\\\[a^{\\phi(n)} \\equiv 1 \\bmod n\\\\]\n8. according to 7, we have \\\\( a \\cdot a^{\\phi(n)-1} \\equiv 1 \\bmod n \\\\). then\n\\\\[ a^{-1} = a^{\\phi(n)-1} \\\\]\n9. Fermat's little theorem\nFermat's little theorem states that if p is a prime number, then for any integer a, the number \n\\\\(a^{p}-a \\\\) is an integer multiple of p. In the notation of modular arithmetic, this is expressed as\n\\\\[ a^{p} \\equiv a \\bmod p\\\\]\n10. Binomial theorem\nit states\n\\\\[ y = (1+n)^{x} = \\sum_{k=0}^{x}\\tbinom{x}{k}n^{k} = 1 + nx + \\tbinom{x}{2}n^2 + ...\\\\]\nobserve that, the higher degree could be divided by \\\\(n^2\\\\). we have\n\\\\[ \\begin{align} \\tag{0.2} (1+n)^{x} \\equiv 1 + nx \\bmod n^2 \\end{align} \\\\]\ntherefore, \\\\( y - 1 \\equiv nx \\bmod n^2 \\\\). then we have\n\\\\[ x \\equiv \\frac{y-1}{n} \\bmod n \\\\].\nIn paillier, later we define \\\\( \\begin{align} \\tag{0.3} L(y) = \\frac{y-1}{n} \\end{align} \\\\)\ntherefore\n\\\\[ L(y \\bmod n^2) \\equiv x \\bmod n \\\\]\n\n\n## Paillier\n1. key generation\n`KeyGen() -> (pk, sk)`\nrandomly select two big prime numbers \\\\(p, q\\\\). it shoud satisfy \\\\(gcd(pq, (p-1)(q-1)) =1 \\\\), \\\\(p\\\\) and \\\\(q\\\\) should have similar bit length. let \\\\( n = pq \\\\), \\\\(\\lambda = lcm(p-1, q-1)\\\\). randomly sample \\\\( g \\in Z_{n^2}^{\\ast}\\\\). to simplify, let \\\\( g = n+1\\\\). we have\n\\\\[ pk=(n,g) \\\\]\n\\\\[ sk = (\\lambda)\\\\]\n\n2. encryption\n`Enc(pk, m) -> c`\nrandomly sample \\\\( r \\in Z_{n}^{\\ast}\\\\), then also have \\\\( r \\in Z_{n^2}^{\\ast}\\\\), cypher is calculated\n\\\\[ \\begin{align} \\tag{1.1} c = g^mr^n \\bmod n^2 \\end{align} \\\\]\n\n3. Decryption\n`Dec(sk, c) -> m`\nLet \\\\(L(x) = \\frac{x-1}{n} \\\\), we have message\n\\\\[ \\begin{align} \\tag{1.2} m = \\frac{L(c^{\\lambda} \\bmod n^2)}{L(g^{\\lambda} \\bmod n^2)} \\bmod n \\end{align}\\\\]\n\n4. proof of correctness\nbased on Eq(1), we have \\\\[ \\begin{align} \\tag{1.3} c^{\\lambda} \\bmod n^2 = g^{m\\lambda}r^{n\\lambda} \\bmod n^2 \\end{align}\\\\]\nwhere \\\\( r^{n\\lambda} \\bmod n^2 \\equiv 1 \\bmod n^2\\\\), which is proved by Carmichael theorem later on. then Eq(3) becomes\n \\\\[ \\begin{align} \\tag{1.4} c^{\\lambda} \\bmod n^2 = g^{m\\lambda}\\bmod n^2 \\end{align}\\\\]\nsince \\\\( g = n+1\\\\), we have\n\\\\[ \\begin{align} \\tag{1.5} c^{\\lambda} \\bmod n^2 = (1+n)^{m\\lambda}\\bmod n^2 \\end{align}\\\\]\nAccording to Eq(0.2), we have\n\\\\[ \\begin{align} \\tag{1.6} c^{\\lambda} \\bmod n^2 = 1 + nm\\lambda \\bmod n^2 \\end{align}\\\\]\n\\\\[ \\begin{align} \\tag{1.7} g^{\\lambda} \\bmod n^2 \\equiv (1+n)^{\\lambda} \\bmod n^2 = 1 +\\lambda n \\bmod n^2 \\end{align}\\\\]\ntherefore, based on definition given by Eq(0.3) we have\n\\\\[ \\begin{align} \\tag{1.8} L(c^{\\lambda} \\bmod n^2) = \\frac{c^{\\lambda}-1}{n} \\bmod n^2 \\end{align} \\\\]\nSubstitute Eq(1.6) into Eq(1.8), we have\n\\\\[ \\begin{align} \\tag{1.9} L(c^{\\lambda} \\bmod n^2) = m\\lambda \\bmod n^2 \\end{align} \\\\]\nFurther, we have\n\\\\[ \\begin{align} \\tag{1.10} L(g^{\\lambda} \\bmod n^2) = \\frac{g^\\lambda -1}{n} \\end{align} \\\\]\nSub Eq(1.7) into Eq(1.10), we have\n\\\\[ \\begin{align} \\tag{1.11} L(g^{\\lambda} \\bmod n^2) = \\frac{\\lambda n}{n} \\equiv \\lambda \\bmod n^2\\end{align} \\\\]\nAt last, Eq(1.2) becomes （bu sub Eq1.9 and Eq1.11)\n\\\\[ \\begin{align} m = \\frac{L(c^{\\lambda} \\bmod n^2)}{L(g^{\\lambda} \\bmod n^2)} \\bmod n = \\frac{m \\lambda}{\\lambda} \\equiv m \\bmod n \\end{align}\\\\]\nproved!!!\n\n5. Carmichael theorem\nIn number theory, a branch of mathematics, the Carmichael function \\\\(λ(n)\\\\) of a positive integer n is the smallest positive integer m such that \\\\(a^{m}\\equiv 1{\\pmod {n}}\\\\) (similar but different from Euler's totient function). Carmichael's λ function, the reduced totient function, and the least universal exponent function\n![carmichael theorem](/images/paillier/carmichael_thorem.png)\n![](/images/paillier/carmichael_thorem_2.png)\nlet \\\\( n = pq\\\\), where p and q are prime numbers; \\\\( \\phi(n)\\\\) is the Euler's totient function. Let \\\\(\\lambda(n)\\\\) denotes carmichael function. We have \\\\(\\phi(n)=(p-1)(q-1)\\\\) and \\\\( \\lambda(n)=\\phi(n) = (p-1)(q-1)\\\\).\n\nSince \\\\( |Z_{n^2}^{\\ast}| = \\phi(n^2) = n \\phi(n)\\\\) (according to Eq(0.1)). Thereby, for any \\\\( w \\in Z_{n^2}^{\\ast}\\\\)\n\\\\[ \\begin{align} \\tag{1.12} w^{n\\phi(n)} \\equiv w^{n\\\\lambda} \\equiv 1 \\bmod n^2 \\end{align}\\\\]\n\n\\\\[ \\begin{align} \\tag{1.13} w^{\\lambda} \\equiv 1 \\bmod n \\end{align}\\\\]\nEq(1.13) is just Carmichael's function\n\nBased on Carmichael's theorem\n\\\\[ \\lambda(n^2) = lcm(\\lambda(q^2),\\lambda(p^2)) = lcm(\\phi(q^2),\\phi(p^2)) = lcm(q(q-1), p(p-1)) = pq(lcm(p-1, q-1)) = n\\lambda(n) \\\\] \ntherefore, we have\n\n\\\\[w^{\\lambda(n^2)} = w ^{n\\lambda} \\equiv 1 \\bmod n^2\\\\]\n\n6. Addition homomorphic\n![homomorphic addition](/images/paillier/homomorphic_addition.png)\n\n7. Multiplication homomorphic \n![homomorphic multiplication](/images/paillier/homomorphic_mul.png)\n## references\n- [csdn post](https://blog.csdn.net/qq_42328228/article/details/109349590)","slug":"cryptography/paillier-encryption","published":1,"updated":"2023-11-05T04:21:13.732Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8dp000nqwsjgbxocb3i","content":"\n\n\n

fundamentals

    \n
  1. fundamental theorem of arighmetic
    the fundamental theorem of arithmetic, also called the unique factorization theorem and prime factorization theorem, states that every integer greater than 1 can be represented uniquely as a product of prime numbers, up to the order of the factors wiki
    2. \n
  2. Euler’s totient function
    In number theory, Euler’s totient function counts the positive integers up to a given integer n that are relatively prime to n. It is written using the Greek letter phi as \\( \\phi (n) \\), and may also be called Euler’s phi function. In other words, it is the number of integers k in the range 1 ≤ k ≤ n for which the greatest common divisor gcd(n, k) is equal to 1. The integers k of this form are sometimes referred to as totatives of n. the collection of k is denoted by \\( Z_{n}^{\\ast } \\), and \\[ \\phi(n) = |Z_n^{\\ast }| \\]
    3. \n
  3. if p is prime, then \\( Z_p^{\\ast } = Z_p \\), \\( \\phi(p) = p-1 \\)
    4. \n
  4. if p is prime, for any integer r, then \\( \\begin{align} \\tag{0.1} \\phi(p^{r}) =p^{r-1}\\phi(p)=p^{r-1}(p-1)\\end{align} \\)
    5. \n
  5. Euler’s totient function is a multiplicative function, meaning that if two numbers m and n are relatively prime, then \\(\\phi(mn) = \\phi(m)\\phi(n)\\)
    6. \n
  6. Euler’s product formula, it states
    \\[ \\phi(n) = n \\prod_{p|n}^{}(1-\\frac{1}{p}) \\]
    where the product is over the distinct prime numbers dividing n.
    7. \n
  7. Euler’s theorem
    if \\(a\\) and \\(n\\) are coprime positive integers, and \\( \\phi(n)\\) is Euler’s totient function, then \\(a\\) raised to the power \\(\\phi(n)\\) is congruent to 1 modulo n; that is
    \\[a^{\\phi(n)} \\equiv 1 \\bmod n\\]
    8. \n
  8. according to 7, we have \\( a \\cdot a^{\\phi(n)-1} \\equiv 1 \\bmod n \\). then
    \\[ a^{-1} = a^{\\phi(n)-1} \\]
    9. \n
  9. Fermat’s little theorem
    Fermat’s little theorem states that if p is a prime number, then for any integer a, the number
    \\(a^{p}-a \\) is an integer multiple of p. In the notation of modular arithmetic, this is expressed as
    \\[ a^{p} \\equiv a \\bmod p\\]
    10. \n
  10. Binomial theorem
    it states
    \\[ y = (1+n)^{x} = \\sum_{k=0}^{x}\\tbinom{x}{k}n^{k} = 1 + nx + \\tbinom{x}{2}n^2 + …\\]
    observe that, the higher degree could be divided by \\(n^2\\). we have
    \\[ \\begin{align} \\tag{0.2} (1+n)^{x} \\equiv 1 + nx \\bmod n^2 \\end{align} \\]
    therefore, \\( y - 1 \\equiv nx \\bmod n^2 \\). then we have
    \\[ x \\equiv \\frac{y-1}{n} \\bmod n \\].
    In paillier, later we define \\( \\begin{align} \\tag{0.3} L(y) = \\frac{y-1}{n} \\end{align} \\)
    therefore
    \\[ L(y \\bmod n^2) \\equiv x \\bmod n \\]
    11. \n
\n

Paillier

    \n

  1. key generation
    KeyGen() -> (pk, sk)
    randomly select two big prime numbers \\(p, q\\). it shoud satisfy \\(gcd(pq, (p-1)(q-1)) =1 \\), \\(p\\) and \\(q\\) should have similar bit length. let \\( n = pq \\), \\(\\lambda = lcm(p-1, q-1)\\). randomly sample \\( g \\in Z_{n^2}^{\\ast}\\). to simplify, let \\( g = n+1\\). we have
    \\[ pk=(n,g) \\]
    \\[ sk = (\\lambda)\\]

    \n
    2. \n

  2. encryption
    Enc(pk, m) -> c
    randomly sample \\( r \\in Z_{n}^{\\ast}\\), then also have \\( r \\in Z_{n^2}^{\\ast}\\), cypher is calculated
    \\[ \\begin{align} \\tag{1.1} c = g^mr^n \\bmod n^2 \\end{align} \\]

    \n
    3. \n

  3. Decryption
    Dec(sk, c) -> m
    Let \\(L(x) = \\frac{x-1}{n} \\), we have message
    \\[ \\begin{align} \\tag{1.2} m = \\frac{L(c^{\\lambda} \\bmod n^2)}{L(g^{\\lambda} \\bmod n^2)} \\bmod n \\end{align}\\]

    \n
    4. \n

  4. proof of correctness
    based on Eq(1), we have \\[ \\begin{align} \\tag{1.3} c^{\\lambda} \\bmod n^2 = g^{m\\lambda}r^{n\\lambda} \\bmod n^2 \\end{align}\\]
    where \\( r^{n\\lambda} \\bmod n^2 \\equiv 1 \\bmod n^2\\), which is proved by Carmichael theorem later on. then Eq(3) becomes
    \\[ \\begin{align} \\tag{1.4} c^{\\lambda} \\bmod n^2 = g^{m\\lambda}\\bmod n^2 \\end{align}\\]
    since \\( g = n+1\\), we have
    \\[ \\begin{align} \\tag{1.5} c^{\\lambda} \\bmod n^2 = (1+n)^{m\\lambda}\\bmod n^2 \\end{align}\\]
    According to Eq(0.2), we have
    \\[ \\begin{align} \\tag{1.6} c^{\\lambda} \\bmod n^2 = 1 + nm\\lambda \\bmod n^2 \\end{align}\\]
    \\[ \\begin{align} \\tag{1.7} g^{\\lambda} \\bmod n^2 \\equiv (1+n)^{\\lambda} \\bmod n^2 = 1 +\\lambda n \\bmod n^2 \\end{align}\\]
    therefore, based on definition given by Eq(0.3) we have
    \\[ \\begin{align} \\tag{1.8} L(c^{\\lambda} \\bmod n^2) = \\frac{c^{\\lambda}-1}{n} \\bmod n^2 \\end{align} \\]
    Substitute Eq(1.6) into Eq(1.8), we have
    \\[ \\begin{align} \\tag{1.9} L(c^{\\lambda} \\bmod n^2) = m\\lambda \\bmod n^2 \\end{align} \\]
    Further, we have
    \\[ \\begin{align} \\tag{1.10} L(g^{\\lambda} \\bmod n^2) = \\frac{g^\\lambda -1}{n} \\end{align} \\]
    Sub Eq(1.7) into Eq(1.10), we have
    \\[ \\begin{align} \\tag{1.11} L(g^{\\lambda} \\bmod n^2) = \\frac{\\lambda n}{n} \\equiv \\lambda \\bmod n^2\\end{align} \\]
    At last, Eq(1.2) becomes （bu sub Eq1.9 and Eq1.11)
    \\[ \\begin{align} m = \\frac{L(c^{\\lambda} \\bmod n^2)}{L(g^{\\lambda} \\bmod n^2)} \\bmod n = \\frac{m \\lambda}{\\lambda} \\equiv m \\bmod n \\end{align}\\]
    proved!!!

    \n
    5. \n

  5. Carmichael theorem
    In number theory, a branch of mathematics, the Carmichael function \\(λ(n)\\) of a positive integer n is the smallest positive integer m such that \\(a^{m}\\equiv 1{\\pmod {n}}\\) (similar but different from Euler’s totient function). Carmichael’s λ function, the reduced totient function, and the least universal exponent function
    \"carmichael

    let \\( n = pq\\), where p and q are prime numbers; \\( \\phi(n)\\) is the Euler’s totient function. Let \\(\\lambda(n)\\) denotes carmichael function. We have \\(\\phi(n)=(p-1)(q-1)\\) and \\( \\lambda(n)=\\phi(n) = (p-1)(q-1)\\).

    \n
    6. \n
\n

Since \\( |Z_{n^2}^{\\ast}| = \\phi(n^2) = n \\phi(n)\\) (according to Eq(0.1)). Thereby, for any \\( w \\in Z_{n^2}^{\\ast}\\)
\\[ \\begin{align} \\tag{1.12} w^{n\\phi(n)} \\equiv w^{n\\lambda} \\equiv 1 \\bmod n^2 \\end{align}\\]

\n

\\[ \\begin{align} \\tag{1.13} w^{\\lambda} \\equiv 1 \\bmod n \\end{align}\\]
Eq(1.13) is just Carmichael’s function

\n

Based on Carmichael’s theorem
\\[ \\lambda(n^2) = lcm(\\lambda(q^2),\\lambda(p^2)) = lcm(\\phi(q^2),\\phi(p^2)) = lcm(q(q-1), p(p-1)) = pq(lcm(p-1, q-1)) = n\\lambda(n) \\]
therefore, we have

\n

\\[w^{\\lambda(n^2)} = w ^{n\\lambda} \\equiv 1 \\bmod n^2\\]

\n
    \n

  1. Addition homomorphic
    \"homomorphic

    \n
    2. \n

  2. Multiplication homomorphic
    \"homomorphic

    \n
    3. \n
\n

references

\n","site":{"data":{}},"excerpt":"","more":"\n\n\n

fundamentals

    \n
  1. fundamental theorem of arighmetic
    the fundamental theorem of arithmetic, also called the unique factorization theorem and prime factorization theorem, states that every integer greater than 1 can be represented uniquely as a product of prime numbers, up to the order of the factors wiki
    2. \n
  2. Euler’s totient function
    In number theory, Euler’s totient function counts the positive integers up to a given integer n that are relatively prime to n. It is written using the Greek letter phi as \\( \\phi (n) \\), and may also be called Euler’s phi function. In other words, it is the number of integers k in the range 1 ≤ k ≤ n for which the greatest common divisor gcd(n, k) is equal to 1. The integers k of this form are sometimes referred to as totatives of n. the collection of k is denoted by \\( Z_{n}^{\\ast } \\), and \\[ \\phi(n) = |Z_n^{\\ast }| \\]
    3. \n
  3. if p is prime, then \\( Z_p^{\\ast } = Z_p \\), \\( \\phi(p) = p-1 \\)
    4. \n
  4. if p is prime, for any integer r, then \\( \\begin{align} \\tag{0.1} \\phi(p^{r}) =p^{r-1}\\phi(p)=p^{r-1}(p-1)\\end{align} \\)
    5. \n
  5. Euler’s totient function is a multiplicative function, meaning that if two numbers m and n are relatively prime, then \\(\\phi(mn) = \\phi(m)\\phi(n)\\)
    6. \n
  6. Euler’s product formula, it states
    \\[ \\phi(n) = n \\prod_{p|n}^{}(1-\\frac{1}{p}) \\]
    where the product is over the distinct prime numbers dividing n.
    7. \n
  7. Euler’s theorem
    if \\(a\\) and \\(n\\) are coprime positive integers, and \\( \\phi(n)\\) is Euler’s totient function, then \\(a\\) raised to the power \\(\\phi(n)\\) is congruent to 1 modulo n; that is
    \\[a^{\\phi(n)} \\equiv 1 \\bmod n\\]
    8. \n
  8. according to 7, we have \\( a \\cdot a^{\\phi(n)-1} \\equiv 1 \\bmod n \\). then
    \\[ a^{-1} = a^{\\phi(n)-1} \\]
    9. \n
  9. Fermat’s little theorem
    Fermat’s little theorem states that if p is a prime number, then for any integer a, the number
    \\(a^{p}-a \\) is an integer multiple of p. In the notation of modular arithmetic, this is expressed as
    \\[ a^{p} \\equiv a \\bmod p\\]
    10. \n
  10. Binomial theorem
    it states
    \\[ y = (1+n)^{x} = \\sum_{k=0}^{x}\\tbinom{x}{k}n^{k} = 1 + nx + \\tbinom{x}{2}n^2 + …\\]
    observe that, the higher degree could be divided by \\(n^2\\). we have
    \\[ \\begin{align} \\tag{0.2} (1+n)^{x} \\equiv 1 + nx \\bmod n^2 \\end{align} \\]
    therefore, \\( y - 1 \\equiv nx \\bmod n^2 \\). then we have
    \\[ x \\equiv \\frac{y-1}{n} \\bmod n \\].
    In paillier, later we define \\( \\begin{align} \\tag{0.3} L(y) = \\frac{y-1}{n} \\end{align} \\)
    therefore
    \\[ L(y \\bmod n^2) \\equiv x \\bmod n \\]
    11. \n
\n

Paillier

    \n

  1. key generation
    KeyGen() -> (pk, sk)
    randomly select two big prime numbers \\(p, q\\). it shoud satisfy \\(gcd(pq, (p-1)(q-1)) =1 \\), \\(p\\) and \\(q\\) should have similar bit length. let \\( n = pq \\), \\(\\lambda = lcm(p-1, q-1)\\). randomly sample \\( g \\in Z_{n^2}^{\\ast}\\). to simplify, let \\( g = n+1\\). we have
    \\[ pk=(n,g) \\]
    \\[ sk = (\\lambda)\\]

    \n
    2. \n

  2. encryption
    Enc(pk, m) -> c
    randomly sample \\( r \\in Z_{n}^{\\ast}\\), then also have \\( r \\in Z_{n^2}^{\\ast}\\), cypher is calculated
    \\[ \\begin{align} \\tag{1.1} c = g^mr^n \\bmod n^2 \\end{align} \\]

    \n
    3. \n

  3. Decryption
    Dec(sk, c) -> m
    Let \\(L(x) = \\frac{x-1}{n} \\), we have message
    \\[ \\begin{align} \\tag{1.2} m = \\frac{L(c^{\\lambda} \\bmod n^2)}{L(g^{\\lambda} \\bmod n^2)} \\bmod n \\end{align}\\]

    \n
    4. \n

  4. proof of correctness
    based on Eq(1), we have \\[ \\begin{align} \\tag{1.3} c^{\\lambda} \\bmod n^2 = g^{m\\lambda}r^{n\\lambda} \\bmod n^2 \\end{align}\\]
    where \\( r^{n\\lambda} \\bmod n^2 \\equiv 1 \\bmod n^2\\), which is proved by Carmichael theorem later on. then Eq(3) becomes
    \\[ \\begin{align} \\tag{1.4} c^{\\lambda} \\bmod n^2 = g^{m\\lambda}\\bmod n^2 \\end{align}\\]
    since \\( g = n+1\\), we have
    \\[ \\begin{align} \\tag{1.5} c^{\\lambda} \\bmod n^2 = (1+n)^{m\\lambda}\\bmod n^2 \\end{align}\\]
    According to Eq(0.2), we have
    \\[ \\begin{align} \\tag{1.6} c^{\\lambda} \\bmod n^2 = 1 + nm\\lambda \\bmod n^2 \\end{align}\\]
    \\[ \\begin{align} \\tag{1.7} g^{\\lambda} \\bmod n^2 \\equiv (1+n)^{\\lambda} \\bmod n^2 = 1 +\\lambda n \\bmod n^2 \\end{align}\\]
    therefore, based on definition given by Eq(0.3) we have
    \\[ \\begin{align} \\tag{1.8} L(c^{\\lambda} \\bmod n^2) = \\frac{c^{\\lambda}-1}{n} \\bmod n^2 \\end{align} \\]
    Substitute Eq(1.6) into Eq(1.8), we have
    \\[ \\begin{align} \\tag{1.9} L(c^{\\lambda} \\bmod n^2) = m\\lambda \\bmod n^2 \\end{align} \\]
    Further, we have
    \\[ \\begin{align} \\tag{1.10} L(g^{\\lambda} \\bmod n^2) = \\frac{g^\\lambda -1}{n} \\end{align} \\]
    Sub Eq(1.7) into Eq(1.10), we have
    \\[ \\begin{align} \\tag{1.11} L(g^{\\lambda} \\bmod n^2) = \\frac{\\lambda n}{n} \\equiv \\lambda \\bmod n^2\\end{align} \\]
    At last, Eq(1.2) becomes （bu sub Eq1.9 and Eq1.11)
    \\[ \\begin{align} m = \\frac{L(c^{\\lambda} \\bmod n^2)}{L(g^{\\lambda} \\bmod n^2)} \\bmod n = \\frac{m \\lambda}{\\lambda} \\equiv m \\bmod n \\end{align}\\]
    proved!!!

    \n
    5. \n

  5. Carmichael theorem
    In number theory, a branch of mathematics, the Carmichael function \\(λ(n)\\) of a positive integer n is the smallest positive integer m such that \\(a^{m}\\equiv 1{\\pmod {n}}\\) (similar but different from Euler’s totient function). Carmichael’s λ function, the reduced totient function, and the least universal exponent function
    \"carmichael

    let \\( n = pq\\), where p and q are prime numbers; \\( \\phi(n)\\) is the Euler’s totient function. Let \\(\\lambda(n)\\) denotes carmichael function. We have \\(\\phi(n)=(p-1)(q-1)\\) and \\( \\lambda(n)=\\phi(n) = (p-1)(q-1)\\).

    \n
    6. \n
\n

Since \\( |Z_{n^2}^{\\ast}| = \\phi(n^2) = n \\phi(n)\\) (according to Eq(0.1)). Thereby, for any \\( w \\in Z_{n^2}^{\\ast}\\)
\\[ \\begin{align} \\tag{1.12} w^{n\\phi(n)} \\equiv w^{n\\lambda} \\equiv 1 \\bmod n^2 \\end{align}\\]

\n

\\[ \\begin{align} \\tag{1.13} w^{\\lambda} \\equiv 1 \\bmod n \\end{align}\\]
Eq(1.13) is just Carmichael’s function

\n

Based on Carmichael’s theorem
\\[ \\lambda(n^2) = lcm(\\lambda(q^2),\\lambda(p^2)) = lcm(\\phi(q^2),\\phi(p^2)) = lcm(q(q-1), p(p-1)) = pq(lcm(p-1, q-1)) = n\\lambda(n) \\]
therefore, we have

\n

\\[w^{\\lambda(n^2)} = w ^{n\\lambda} \\equiv 1 \\bmod n^2\\]

\n
    \n

  1. Addition homomorphic
    \"homomorphic

    \n
    2. \n

  2. Multiplication homomorphic
    \"homomorphic

    \n
    3. \n
\n

references

\n"},{"title":"two party ecdsa","date":"2023-02-07T06:29:26.000Z","_content":"\n\n\n## overview\nthis post is my reading summary of paper Yehuda Lindell 2017: Fast secure two-party ecdsa signing. the implementation could be found in tss-lib (golang), zengo's library (rust).\n\nUnlike other schemes like RSA, Schnorr signatures and more, it is particularly hard to construct efficient threshold signature protocols for ECDSA as there is an inverse computaion of \\\\( k \\\\).\n\nIn this paper, we consider the specific case of two parties (and thus no honest majority) and construct a protocol that is approximately two orders of magnitude faster than the previous best.\n\n## Comparing ECDSA signing to EC-Schnorr signing\nIn both cases, the public verification key is an elliptic curve point \\\\( Q \\\\) and the private signing key is \\\\( x \\\\) such that \\\\( Q = x \\cdot G \\\\), where \\\\( G \\\\) is the generator point of an EC group of order \\\\( q \\\\).\n![schnorr ecdsa comparison](/images/two_party_ecdsa/schnorr_ecdsa_comparison.png)\n\nObserve that Schnorr signing can be easily distributed since the private key \\\\( x \\\\) and the value k are both used in a linear equation. In contrast, in ECDSA signing, the equation for computing \\\\( s \\\\) includes \\\\( k^{-1} \\\\). Now, given shares \\\\(k_1\\\\), \\\\(k_2\\\\) such that \\\\(k_1 + k_2 = k \\bmod q\\\\) .It is very difficult to compute \\\\(k_1^{\\prime}\\\\), \\\\(k_2^{\\prime}\\\\) such that \\\\(k_1^{\\prime} + k_2^{\\prime} = k^{-1} \\bmod q\\\\)\n\n\ntwo-party protocols for ECDSA signing use multiplicative sharing of \\\\( x \\\\) and of \\\\( k \\\\). That is, the parties hold \\\\(x_1\\\\), \\\\(x_2\\\\) such that \\\\(x_1 \\cdot x_2 = x \\bmod q\\\\), and in each signing operation they generate \\\\(k_1\\\\), \\\\(k_2\\\\) such that \\\\(k_1 \\cdot k_2 = k \\bmod q\\\\). This enables them to easily compute \\\\(k^{-1}\\\\) since each party can locally compute \\\\(k_i^{\\prime} = k_i^{-1} \\bmod q\\\\), and then \\\\(k_1^{\\prime}\\\\), \\\\(k_2^{\\prime}\\\\) are multiplicative shares of \\\\(k^{-1}\\\\). The parties can then use additively homomorphic encryption – specifically Paillier encryption – in order to combine their equations. For example, \\\\(P_1\\\\) can compute \\\\(c_1 = Enc_{pk}(k_1^{-1} \\cdot H(m))\\\\) and \\\\(c_2 = Enc_{pk}(k_1^{-1} \\cdot x_1 \\cdot r)\\\\) . Then, using scar multiplication (denoted ⊙) and homomorphic addition (denoted ⊕), \\\\( P_2 \\\\) can compute \\\\( (k_2^{-1} ⊙ c_1 ) ⊕ [( k_2^{-1} \\cdot x_2) ⊙ c_2 ]\\\\), which will be an encryption of \n\n![paillier encryption](/images/two_party_ecdsa/paillier_enc.png)\n\nHowever, proving that each party worked correctly is extremely difficult. For example, the first party must prove that the Paillier encryption includes \\\\(k_1^{-1}\\\\) when the second party only has \\\\(R_1 = k_1 \\cdot G\\\\). it must prove that the Paillier encryptions are to values in the expected range, and more. This can be done, but it results in a protocol that is very expensive.\n\n## their results\n[WIP]\n\n## references\n- [original papger](https://eprint.iacr.org/2017/552.pdf)","source":"_posts/cryptography/two-party-ecdsa.md","raw":"---\ntitle: two party ecdsa\ndate: 2023-02-07 14:29:26\ntags: [cryptography,mpc,ecdsa]\n---\n\n\n\n## overview\nthis post is my reading summary of paper Yehuda Lindell 2017: Fast secure two-party ecdsa signing. the implementation could be found in tss-lib (golang), zengo's library (rust).\n\nUnlike other schemes like RSA, Schnorr signatures and more, it is particularly hard to construct efficient threshold signature protocols for ECDSA as there is an inverse computaion of \\\\( k \\\\).\n\nIn this paper, we consider the specific case of two parties (and thus no honest majority) and construct a protocol that is approximately two orders of magnitude faster than the previous best.\n\n## Comparing ECDSA signing to EC-Schnorr signing\nIn both cases, the public verification key is an elliptic curve point \\\\( Q \\\\) and the private signing key is \\\\( x \\\\) such that \\\\( Q = x \\cdot G \\\\), where \\\\( G \\\\) is the generator point of an EC group of order \\\\( q \\\\).\n![schnorr ecdsa comparison](/images/two_party_ecdsa/schnorr_ecdsa_comparison.png)\n\nObserve that Schnorr signing can be easily distributed since the private key \\\\( x \\\\) and the value k are both used in a linear equation. In contrast, in ECDSA signing, the equation for computing \\\\( s \\\\) includes \\\\( k^{-1} \\\\). Now, given shares \\\\(k_1\\\\), \\\\(k_2\\\\) such that \\\\(k_1 + k_2 = k \\bmod q\\\\) .It is very difficult to compute \\\\(k_1^{\\prime}\\\\), \\\\(k_2^{\\prime}\\\\) such that \\\\(k_1^{\\prime} + k_2^{\\prime} = k^{-1} \\bmod q\\\\)\n\n\ntwo-party protocols for ECDSA signing use multiplicative sharing of \\\\( x \\\\) and of \\\\( k \\\\). That is, the parties hold \\\\(x_1\\\\), \\\\(x_2\\\\) such that \\\\(x_1 \\cdot x_2 = x \\bmod q\\\\), and in each signing operation they generate \\\\(k_1\\\\), \\\\(k_2\\\\) such that \\\\(k_1 \\cdot k_2 = k \\bmod q\\\\). This enables them to easily compute \\\\(k^{-1}\\\\) since each party can locally compute \\\\(k_i^{\\prime} = k_i^{-1} \\bmod q\\\\), and then \\\\(k_1^{\\prime}\\\\), \\\\(k_2^{\\prime}\\\\) are multiplicative shares of \\\\(k^{-1}\\\\). The parties can then use additively homomorphic encryption – specifically Paillier encryption – in order to combine their equations. For example, \\\\(P_1\\\\) can compute \\\\(c_1 = Enc_{pk}(k_1^{-1} \\cdot H(m))\\\\) and \\\\(c_2 = Enc_{pk}(k_1^{-1} \\cdot x_1 \\cdot r)\\\\) . Then, using scar multiplication (denoted ⊙) and homomorphic addition (denoted ⊕), \\\\( P_2 \\\\) can compute \\\\( (k_2^{-1} ⊙ c_1 ) ⊕ [( k_2^{-1} \\cdot x_2) ⊙ c_2 ]\\\\), which will be an encryption of \n\n![paillier encryption](/images/two_party_ecdsa/paillier_enc.png)\n\nHowever, proving that each party worked correctly is extremely difficult. For example, the first party must prove that the Paillier encryption includes \\\\(k_1^{-1}\\\\) when the second party only has \\\\(R_1 = k_1 \\cdot G\\\\). it must prove that the Paillier encryptions are to values in the expected range, and more. This can be done, but it results in a protocol that is very expensive.\n\n## their results\n[WIP]\n\n## references\n- [original papger](https://eprint.iacr.org/2017/552.pdf)","slug":"cryptography/two-party-ecdsa","published":1,"updated":"2023-11-05T04:21:17.034Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8dq000pqwsjfar5aa4q","content":"\n\n\n

overview

this post is my reading summary of paper Yehuda Lindell 2017: Fast secure two-party ecdsa signing. the implementation could be found in tss-lib (golang), zengo’s library (rust).

\n

Unlike other schemes like RSA, Schnorr signatures and more, it is particularly hard to construct efficient threshold signature protocols for ECDSA as there is an inverse computaion of \\( k \\).

\n

In this paper, we consider the specific case of two parties (and thus no honest majority) and construct a protocol that is approximately two orders of magnitude faster than the previous best.

\n

Comparing ECDSA signing to EC-Schnorr signing

In both cases, the public verification key is an elliptic curve point \\( Q \\) and the private signing key is \\( x \\) such that \\( Q = x \\cdot G \\), where \\( G \\) is the generator point of an EC group of order \\( q \\).
\"schnorr

\n

Observe that Schnorr signing can be easily distributed since the private key \\( x \\) and the value k are both used in a linear equation. In contrast, in ECDSA signing, the equation for computing \\( s \\) includes \\( k^{-1} \\). Now, given shares \\(k_1\\), \\(k_2\\) such that \\(k_1 + k_2 = k \\bmod q\\) .It is very difficult to compute \\(k_1^{\\prime}\\), \\(k_2^{\\prime}\\) such that \\(k_1^{\\prime} + k_2^{\\prime} = k^{-1} \\bmod q\\)

\n

two-party protocols for ECDSA signing use multiplicative sharing of \\( x \\) and of \\( k \\). That is, the parties hold \\(x_1\\), \\(x_2\\) such that \\(x_1 \\cdot x_2 = x \\bmod q\\), and in each signing operation they generate \\(k_1\\), \\(k_2\\) such that \\(k_1 \\cdot k_2 = k \\bmod q\\). This enables them to easily compute \\(k^{-1}\\) since each party can locally compute \\(k_i^{\\prime} = k_i^{-1} \\bmod q\\), and then \\(k_1^{\\prime}\\), \\(k_2^{\\prime}\\) are multiplicative shares of \\(k^{-1}\\). The parties can then use additively homomorphic encryption – specifically Paillier encryption – in order to combine their equations. For example, \\(P_1\\) can compute \\(c_1 = Enc_{pk}(k_1^{-1} \\cdot H(m))\\) and \\(c_2 = Enc_{pk}(k_1^{-1} \\cdot x_1 \\cdot r)\\) . Then, using scar multiplication (denoted ⊙) and homomorphic addition (denoted ⊕), \\( P_2 \\) can compute \\( (k_2^{-1} ⊙ c_1 ) ⊕ [( k_2^{-1} \\cdot x_2) ⊙ c_2 ]\\), which will be an encryption of

\n

\"paillier

\n

However, proving that each party worked correctly is extremely difficult. For example, the first party must prove that the Paillier encryption includes \\(k_1^{-1}\\) when the second party only has \\(R_1 = k_1 \\cdot G\\). it must prove that the Paillier encryptions are to values in the expected range, and more. This can be done, but it results in a protocol that is very expensive.

\n

their results

[WIP]

\n

references

\n","site":{"data":{}},"excerpt":"","more":"\n\n\n

overview

this post is my reading summary of paper Yehuda Lindell 2017: Fast secure two-party ecdsa signing. the implementation could be found in tss-lib (golang), zengo’s library (rust).

\n

Unlike other schemes like RSA, Schnorr signatures and more, it is particularly hard to construct efficient threshold signature protocols for ECDSA as there is an inverse computaion of \\( k \\).

\n

In this paper, we consider the specific case of two parties (and thus no honest majority) and construct a protocol that is approximately two orders of magnitude faster than the previous best.

\n

Comparing ECDSA signing to EC-Schnorr signing

In both cases, the public verification key is an elliptic curve point \\( Q \\) and the private signing key is \\( x \\) such that \\( Q = x \\cdot G \\), where \\( G \\) is the generator point of an EC group of order \\( q \\).
\"schnorr

\n

Observe that Schnorr signing can be easily distributed since the private key \\( x \\) and the value k are both used in a linear equation. In contrast, in ECDSA signing, the equation for computing \\( s \\) includes \\( k^{-1} \\). Now, given shares \\(k_1\\), \\(k_2\\) such that \\(k_1 + k_2 = k \\bmod q\\) .It is very difficult to compute \\(k_1^{\\prime}\\), \\(k_2^{\\prime}\\) such that \\(k_1^{\\prime} + k_2^{\\prime} = k^{-1} \\bmod q\\)

\n

two-party protocols for ECDSA signing use multiplicative sharing of \\( x \\) and of \\( k \\). That is, the parties hold \\(x_1\\), \\(x_2\\) such that \\(x_1 \\cdot x_2 = x \\bmod q\\), and in each signing operation they generate \\(k_1\\), \\(k_2\\) such that \\(k_1 \\cdot k_2 = k \\bmod q\\). This enables them to easily compute \\(k^{-1}\\) since each party can locally compute \\(k_i^{\\prime} = k_i^{-1} \\bmod q\\), and then \\(k_1^{\\prime}\\), \\(k_2^{\\prime}\\) are multiplicative shares of \\(k^{-1}\\). The parties can then use additively homomorphic encryption – specifically Paillier encryption – in order to combine their equations. For example, \\(P_1\\) can compute \\(c_1 = Enc_{pk}(k_1^{-1} \\cdot H(m))\\) and \\(c_2 = Enc_{pk}(k_1^{-1} \\cdot x_1 \\cdot r)\\) . Then, using scar multiplication (denoted ⊙) and homomorphic addition (denoted ⊕), \\( P_2 \\) can compute \\( (k_2^{-1} ⊙ c_1 ) ⊕ [( k_2^{-1} \\cdot x_2) ⊙ c_2 ]\\), which will be an encryption of

\n

\"paillier

\n

However, proving that each party worked correctly is extremely difficult. For example, the first party must prove that the Paillier encryption includes \\(k_1^{-1}\\) when the second party only has \\(R_1 = k_1 \\cdot G\\). it must prove that the Paillier encryptions are to values in the expected range, and more. This can be done, but it results in a protocol that is very expensive.

\n

their results

[WIP]

\n

references

\n"},{"title":"go reflect","date":"2023-03-02T02:44:50.000Z","_content":"\n## introduction\nReflection is the ability of a program to examine its own structure, particularly through types. \n\n## type and interfaces\nGo is statically typed. Every variable has a static type, Exactly one type known and fixed at compile time. If we declare\n```go\ntype MyInt int\n\nvar i int\nvar j MyInt\n```\n**The variables i and j have distinct static types and, although they have the same underlying type, they cannot be assigned to one another without a conversion.**\n\nOne important category of type is interface types, which represent fixed sets of methods. An interface variable can store any concrete (non-interface) value as long as that value implements the interface’s methods. An extremely important example of an interface type is the empty interface:\n```go\ninterface{}\n```\nor its equivalent alias,\n```go\nany\n```\nIt represents the empty set of methods and is satisfied by any value at all, since every value has zero or more methods.\na variable of interface type always has the same static type, and even though at run time the value stored in the interface variable may change type, that value will always satisfy the interface.\n\n## the representation of an interface\nA variable of interface type stores a pair: the concrete value assigned to the variable, and that value’s type descriptor.\nFor instance, after\n```golang\nvar r io.Reader\ntty, err := os.OpenFile(\"/dev/tty\", os.O_RDWR, 0)\nif err != nil {\n return nil, err\n}\nr = tty\n```\nr contains, schematically, the (value, type) pair, (tty, *os.File). Notice that the type *os.File implements methods other than Read; even though the interface value provides access only to the Read method, the value inside carries all the type information about that value. That’s why we can do things like this:\n```go\nvar w io.Writer\nw = r.(io.Writer)\n```\nThe expression in this assignment is a type assertion; what it asserts is that the item inside r also implements io.Writer, and so we can assign it to w. After the assignment, w will contain the pair (tty, *os.File). That’s the same pair as was held in r. \nOne important detail is that the pair inside an interface variable always has the form (value, concrete type) and cannot have the form (value, interface type). Interfaces do not hold interface values.\n\n## the first law of reflection\n1. Reflection goes from interface value to reflection object\nAt the basic level, reflection is just a mechanism to examine the type and value pair stored inside an interface variable. `reflect.TypeOf` and `reflect.ValueOf`, retrieve `reflect.Type` and `reflect.Value` pieces out of an interface value.\n```go\nvar x float64 = 3.4\nfmt.Println(\"type:\", reflect.TypeOf(x))\n```\n```\ntype: float64\n```\n```go\nvar x float64 = 3.4\nv := reflect.ValueOf(x)\nfmt.Println(\"type:\", v.Type())\nfmt.Println(\"kind is float64:\", v.Kind() == reflect.Float64)\nfmt.Println(\"value:\", v.Float())\n```\n```\ntype: float64\nkind is float64: true\nvalue: 3.4\n```\nThere are also methods like SetInt and SetFloat. **to keep the API simple, the “getter” and “setter” methods of Value operate on the largest type that can hold the value**: int64 for all the signed integers, for instance. \n```go\nvar x uint8 = 'x'\nv := reflect.ValueOf(x)\nfmt.Println(\"type:\", v.Type()) // uint8.\nfmt.Println(\"kind is uint8: \", v.Kind() == reflect.Uint8) // true.\nx = uint8(v.Uint()) // v.Uint returns a uint64.\n```\n\n2. Reflection goes from reflection object to interface value.\nGiven a reflect.Value we can recover an interface value using the Interface method;\n```go\n// Interface returns v's current value as an interface{}.\n// It is equivalent to:\n//\n//\tvar i interface{} = (v's underlying value)\nfunc (v Value) Interface() interface{}\n```\n```go\ny := v.Interface().(float64) // y will have type float64.\nfmt.Println(y)\n```\n\n3. To modify a reflection object, the value must be settable.\nThe CanSet method of Value reports the settability of a Value; in our case,\n```go\nvar x float64 = 3.4\nv := reflect.ValueOf(x)\nfmt.Println(\"settability of v:\", v.CanSet())\n```\n```\nsettability of v: false\n```\nwe pass a copy of x to reflect.ValueOf, so the interface value created as the argument to reflect.ValueOf is a copy of x, not x itself. Thus, if the statement `v.SetFloat(7.1)` were allowed to succeed, it would not update x, even though v looks like it was created from x. Instead, it would update the copy of x stored inside the reflection value and x itself would be unaffected. That would be confusing and useless, so it is illegal, and settability is the property used to avoid this issue. If we want to modify x by reflection, we must give the reflection library a pointer to the value we want to modify.\nLet’s do that. \n```go\nvar x float64 = 3.4\np := reflect.ValueOf(&x) // Note: take the address of x.\nfmt.Println(\"type of p:\", p.Type())\nfmt.Println(\"settability of p:\", p.CanSet())\n```\nThe reflection object p isn’t settable, but it’s not p we want to set, it’s (in effect) *p. To get to what p points to, we call the Elem method of Value, which indirects through the pointer, and save the result in a reflection Value called v:\n```go\nv := p.Elem()\nfmt.Println(\"settability of v:\", v.CanSet())\n```\n```\nsettability of v: true\n```\n\n## structs\n\n```go\ntype T struct {\n A int\n B string\n}\nt := T{23, \"skidoo\"}\ns := reflect.ValueOf(&t).Elem()\ntypeOfT := s.Type()\nfor i := 0; i < s.NumField(); i++ {\n f := s.Field(i)\n fmt.Printf(\"%d: %s %s = %v\\n\", i,\n typeOfT.Field(i).Name, f.Type(), f.Interface())\n}\n```\n```\n0: A int = 23\n1: B string = skidoo\n```\nThere’s one more point about settability introduced in passing here: the field names of T are upper case (exported) because only exported fields of a struct are settable.\nBecause s contains a settable reflection object, we can modify the fields of the structure.\n```\ns.Field(0).SetInt(77)\ns.Field(1).SetString(\"Sunset Strip\")\nfmt.Println(\"t is now\", t)\n```\nIf we modified the program so that s was created from t, not &t, the calls to SetInt and SetString would fail as the fields of t would not be settable.\n\n## references\n- [official blog](https://go.dev/blog/laws-of-reflection)\n- [go data structure: interface](https://research.swtch.com/interfaces)","source":"_posts/golang/go-reflect.md","raw":"---\ntitle: go reflect\ndate: 2023-03-02 10:44:50\ntags: [golang]\n---\n\n## introduction\nReflection is the ability of a program to examine its own structure, particularly through types. \n\n## type and interfaces\nGo is statically typed. Every variable has a static type, Exactly one type known and fixed at compile time. If we declare\n```go\ntype MyInt int\n\nvar i int\nvar j MyInt\n```\n**The variables i and j have distinct static types and, although they have the same underlying type, they cannot be assigned to one another without a conversion.**\n\nOne important category of type is interface types, which represent fixed sets of methods. An interface variable can store any concrete (non-interface) value as long as that value implements the interface’s methods. An extremely important example of an interface type is the empty interface:\n```go\ninterface{}\n```\nor its equivalent alias,\n```go\nany\n```\nIt represents the empty set of methods and is satisfied by any value at all, since every value has zero or more methods.\na variable of interface type always has the same static type, and even though at run time the value stored in the interface variable may change type, that value will always satisfy the interface.\n\n## the representation of an interface\nA variable of interface type stores a pair: the concrete value assigned to the variable, and that value’s type descriptor.\nFor instance, after\n```golang\nvar r io.Reader\ntty, err := os.OpenFile(\"/dev/tty\", os.O_RDWR, 0)\nif err != nil {\n return nil, err\n}\nr = tty\n```\nr contains, schematically, the (value, type) pair, (tty, *os.File). Notice that the type *os.File implements methods other than Read; even though the interface value provides access only to the Read method, the value inside carries all the type information about that value. That’s why we can do things like this:\n```go\nvar w io.Writer\nw = r.(io.Writer)\n```\nThe expression in this assignment is a type assertion; what it asserts is that the item inside r also implements io.Writer, and so we can assign it to w. After the assignment, w will contain the pair (tty, *os.File). That’s the same pair as was held in r. \nOne important detail is that the pair inside an interface variable always has the form (value, concrete type) and cannot have the form (value, interface type). Interfaces do not hold interface values.\n\n## the first law of reflection\n1. Reflection goes from interface value to reflection object\nAt the basic level, reflection is just a mechanism to examine the type and value pair stored inside an interface variable. `reflect.TypeOf` and `reflect.ValueOf`, retrieve `reflect.Type` and `reflect.Value` pieces out of an interface value.\n```go\nvar x float64 = 3.4\nfmt.Println(\"type:\", reflect.TypeOf(x))\n```\n```\ntype: float64\n```\n```go\nvar x float64 = 3.4\nv := reflect.ValueOf(x)\nfmt.Println(\"type:\", v.Type())\nfmt.Println(\"kind is float64:\", v.Kind() == reflect.Float64)\nfmt.Println(\"value:\", v.Float())\n```\n```\ntype: float64\nkind is float64: true\nvalue: 3.4\n```\nThere are also methods like SetInt and SetFloat. **to keep the API simple, the “getter” and “setter” methods of Value operate on the largest type that can hold the value**: int64 for all the signed integers, for instance. \n```go\nvar x uint8 = 'x'\nv := reflect.ValueOf(x)\nfmt.Println(\"type:\", v.Type()) // uint8.\nfmt.Println(\"kind is uint8: \", v.Kind() == reflect.Uint8) // true.\nx = uint8(v.Uint()) // v.Uint returns a uint64.\n```\n\n2. Reflection goes from reflection object to interface value.\nGiven a reflect.Value we can recover an interface value using the Interface method;\n```go\n// Interface returns v's current value as an interface{}.\n// It is equivalent to:\n//\n//\tvar i interface{} = (v's underlying value)\nfunc (v Value) Interface() interface{}\n```\n```go\ny := v.Interface().(float64) // y will have type float64.\nfmt.Println(y)\n```\n\n3. To modify a reflection object, the value must be settable.\nThe CanSet method of Value reports the settability of a Value; in our case,\n```go\nvar x float64 = 3.4\nv := reflect.ValueOf(x)\nfmt.Println(\"settability of v:\", v.CanSet())\n```\n```\nsettability of v: false\n```\nwe pass a copy of x to reflect.ValueOf, so the interface value created as the argument to reflect.ValueOf is a copy of x, not x itself. Thus, if the statement `v.SetFloat(7.1)` were allowed to succeed, it would not update x, even though v looks like it was created from x. Instead, it would update the copy of x stored inside the reflection value and x itself would be unaffected. That would be confusing and useless, so it is illegal, and settability is the property used to avoid this issue. If we want to modify x by reflection, we must give the reflection library a pointer to the value we want to modify.\nLet’s do that. \n```go\nvar x float64 = 3.4\np := reflect.ValueOf(&x) // Note: take the address of x.\nfmt.Println(\"type of p:\", p.Type())\nfmt.Println(\"settability of p:\", p.CanSet())\n```\nThe reflection object p isn’t settable, but it’s not p we want to set, it’s (in effect) *p. To get to what p points to, we call the Elem method of Value, which indirects through the pointer, and save the result in a reflection Value called v:\n```go\nv := p.Elem()\nfmt.Println(\"settability of v:\", v.CanSet())\n```\n```\nsettability of v: true\n```\n\n## structs\n\n```go\ntype T struct {\n A int\n B string\n}\nt := T{23, \"skidoo\"}\ns := reflect.ValueOf(&t).Elem()\ntypeOfT := s.Type()\nfor i := 0; i < s.NumField(); i++ {\n f := s.Field(i)\n fmt.Printf(\"%d: %s %s = %v\\n\", i,\n typeOfT.Field(i).Name, f.Type(), f.Interface())\n}\n```\n```\n0: A int = 23\n1: B string = skidoo\n```\nThere’s one more point about settability introduced in passing here: the field names of T are upper case (exported) because only exported fields of a struct are settable.\nBecause s contains a settable reflection object, we can modify the fields of the structure.\n```\ns.Field(0).SetInt(77)\ns.Field(1).SetString(\"Sunset Strip\")\nfmt.Println(\"t is now\", t)\n```\nIf we modified the program so that s was created from t, not &t, the calls to SetInt and SetString would fail as the fields of t would not be settable.\n\n## references\n- [official blog](https://go.dev/blog/laws-of-reflection)\n- [go data structure: interface](https://research.swtch.com/interfaces)","slug":"golang/go-reflect","published":1,"updated":"2023-11-05T04:21:13.733Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8dq000rqwsj5r3jepey","content":"

introduction

Reflection is the ability of a program to examine its own structure, particularly through types.

\n

type and interfaces

Go is statically typed. Every variable has a static type, Exactly one type known and fixed at compile time. If we declare

\n
1
2
3
4
type MyInt int

var i int
var j MyInt
\n

The variables i and j have distinct static types and, although they have the same underlying type, they cannot be assigned to one another without a conversion.

\n

One important category of type is interface types, which represent fixed sets of methods. An interface variable can store any concrete (non-interface) value as long as that value implements the interface’s methods. An extremely important example of an interface type is the empty interface:

\n
1
interface{}
\n

or its equivalent alias,

\n
1
any
\n

It represents the empty set of methods and is satisfied by any value at all, since every value has zero or more methods.
a variable of interface type always has the same static type, and even though at run time the value stored in the interface variable may change type, that value will always satisfy the interface.

\n

the representation of an interface

A variable of interface type stores a pair: the concrete value assigned to the variable, and that value’s type descriptor.
For instance, after

\n
1
2
3
4
5
6
var r io.Reader
tty, err := os.OpenFile("/dev/tty", os.O_RDWR, 0)
if err != nil {
    return nil, err
}
r = tty
\n

r contains, schematically, the (value, type) pair, (tty, *os.File). Notice that the type *os.File implements methods other than Read; even though the interface value provides access only to the Read method, the value inside carries all the type information about that value. That’s why we can do things like this:

\n
1
2
var w io.Writer
w = r.(io.Writer)
\n

The expression in this assignment is a type assertion; what it asserts is that the item inside r also implements io.Writer, and so we can assign it to w. After the assignment, w will contain the pair (tty, *os.File). That’s the same pair as was held in r.
One important detail is that the pair inside an interface variable always has the form (value, concrete type) and cannot have the form (value, interface type). Interfaces do not hold interface values.

\n

the first law of reflection

    \n

  1. Reflection goes from interface value to reflection object
    At the basic level, reflection is just a mechanism to examine the type and value pair stored inside an interface variable. reflect.TypeOf and reflect.ValueOf, retrieve reflect.Type and reflect.Value pieces out of an interface value.

    \n
    1
    2
    var x float64 = 3.4
    fmt.Println("type:", reflect.TypeOf(x))
    \n
    1
    type: float64
    \n
    1
    2
    3
    4
    5
    var x float64 = 3.4
    v := reflect.ValueOf(x)
    fmt.Println("type:", v.Type())
    fmt.Println("kind is float64:", v.Kind() == reflect.Float64)
    fmt.Println("value:", v.Float())
    \n
    1
    2
    3
    type: float64
    kind is float64: true
    value: 3.4
    \n

    There are also methods like SetInt and SetFloat. to keep the API simple, the “getter” and “setter” methods of Value operate on the largest type that can hold the value: int64 for all the signed integers, for instance.

    \n
    1
    2
    3
    4
    5
    var x uint8 = 'x'
    v := reflect.ValueOf(x)
    fmt.Println("type:", v.Type())                            // uint8.
    fmt.Println("kind is uint8: ", v.Kind() == reflect.Uint8) // true.
    x = uint8(v.Uint())                                       // v.Uint returns a uint64.
    \n
    2. \n

  2. Reflection goes from reflection object to interface value.
    Given a reflect.Value we can recover an interface value using the Interface method;

    \n
    1
    2
    3
    4
    5
    // Interface returns v's current value as an interface{}.
    // It is equivalent to:
    //
    //\tvar i interface{} = (v's underlying value)
    func (v Value) Interface() interface{}
    \n
    1
    2
    y := v.Interface().(float64) // y will have type float64.
    fmt.Println(y)
    \n
    3. \n

  3. To modify a reflection object, the value must be settable.
    The CanSet method of Value reports the settability of a Value; in our case,

    \n
    1
    2
    3
    var x float64 = 3.4
    v := reflect.ValueOf(x)
    fmt.Println("settability of v:", v.CanSet())
    \n
    1
    settability of v: false
    \n

    we pass a copy of x to reflect.ValueOf, so the interface value created as the argument to reflect.ValueOf is a copy of x, not x itself. Thus, if the statement v.SetFloat(7.1) were allowed to succeed, it would not update x, even though v looks like it was created from x. Instead, it would update the copy of x stored inside the reflection value and x itself would be unaffected. That would be confusing and useless, so it is illegal, and settability is the property used to avoid this issue. If we want to modify x by reflection, we must give the reflection library a pointer to the value we want to modify.
    Let’s do that.

    \n
    1
    2
    3
    4
    var x float64 = 3.4
    p := reflect.ValueOf(&x) // Note: take the address of x.
    fmt.Println("type of p:", p.Type())
    fmt.Println("settability of p:", p.CanSet())
    \n

    The reflection object p isn’t settable, but it’s not p we want to set, it’s (in effect) *p. To get to what p points to, we call the Elem method of Value, which indirects through the pointer, and save the result in a reflection Value called v:

    \n
    1
    2
    v := p.Elem()
    fmt.Println("settability of v:", v.CanSet())
    \n
    1
    settability of v: true
    4. \n
\n

structs

1
2
3
4
5
6
7
8
9
10
11
12
type T struct {
    A int
    B string
}
t := T{23, "skidoo"}
s := reflect.ValueOf(&t).Elem()
typeOfT := s.Type()
for i := 0; i < s.NumField(); i++ {
    f := s.Field(i)
    fmt.Printf("%d: %s %s = %v\\n", i,
        typeOfT.Field(i).Name, f.Type(), f.Interface())
}
\n
1
2
0: A int = 23
1: B string = skidoo
\n

There’s one more point about settability introduced in passing here: the field names of T are upper case (exported) because only exported fields of a struct are settable.
Because s contains a settable reflection object, we can modify the fields of the structure.

\n
1
2
3
s.Field(0).SetInt(77)
s.Field(1).SetString("Sunset Strip")
fmt.Println("t is now", t)
\n

If we modified the program so that s was created from t, not &t, the calls to SetInt and SetString would fail as the fields of t would not be settable.

\n

references

\n","site":{"data":{}},"excerpt":"","more":"

introduction

Reflection is the ability of a program to examine its own structure, particularly through types.

\n

type and interfaces

Go is statically typed. Every variable has a static type, Exactly one type known and fixed at compile time. If we declare

\n
1
2
3
4
type MyInt int

var i int
var j MyInt
\n

The variables i and j have distinct static types and, although they have the same underlying type, they cannot be assigned to one another without a conversion.

\n

One important category of type is interface types, which represent fixed sets of methods. An interface variable can store any concrete (non-interface) value as long as that value implements the interface’s methods. An extremely important example of an interface type is the empty interface:

\n
1
interface{}
\n

or its equivalent alias,

\n
1
any
\n

It represents the empty set of methods and is satisfied by any value at all, since every value has zero or more methods.
a variable of interface type always has the same static type, and even though at run time the value stored in the interface variable may change type, that value will always satisfy the interface.

\n

the representation of an interface

A variable of interface type stores a pair: the concrete value assigned to the variable, and that value’s type descriptor.
For instance, after

\n
1
2
3
4
5
6
var r io.Reader
tty, err := os.OpenFile("/dev/tty", os.O_RDWR, 0)
if err != nil {
    return nil, err
}
r = tty
\n

r contains, schematically, the (value, type) pair, (tty, *os.File). Notice that the type *os.File implements methods other than Read; even though the interface value provides access only to the Read method, the value inside carries all the type information about that value. That’s why we can do things like this:

\n
1
2
var w io.Writer
w = r.(io.Writer)
\n

The expression in this assignment is a type assertion; what it asserts is that the item inside r also implements io.Writer, and so we can assign it to w. After the assignment, w will contain the pair (tty, *os.File). That’s the same pair as was held in r.
One important detail is that the pair inside an interface variable always has the form (value, concrete type) and cannot have the form (value, interface type). Interfaces do not hold interface values.

\n

the first law of reflection

    \n

  1. Reflection goes from interface value to reflection object
    At the basic level, reflection is just a mechanism to examine the type and value pair stored inside an interface variable. reflect.TypeOf and reflect.ValueOf, retrieve reflect.Type and reflect.Value pieces out of an interface value.

    \n
    1
    2
    var x float64 = 3.4
    fmt.Println("type:", reflect.TypeOf(x))
    \n
    1
    type: float64
    \n
    1
    2
    3
    4
    5
    var x float64 = 3.4
    v := reflect.ValueOf(x)
    fmt.Println("type:", v.Type())
    fmt.Println("kind is float64:", v.Kind() == reflect.Float64)
    fmt.Println("value:", v.Float())
    \n
    1
    2
    3
    type: float64
    kind is float64: true
    value: 3.4
    \n

    There are also methods like SetInt and SetFloat. to keep the API simple, the “getter” and “setter” methods of Value operate on the largest type that can hold the value: int64 for all the signed integers, for instance.

    \n
    1
    2
    3
    4
    5
    var x uint8 = 'x'
    v := reflect.ValueOf(x)
    fmt.Println("type:", v.Type())                            // uint8.
    fmt.Println("kind is uint8: ", v.Kind() == reflect.Uint8) // true.
    x = uint8(v.Uint())                                       // v.Uint returns a uint64.
    \n
    2. \n

  2. Reflection goes from reflection object to interface value.
    Given a reflect.Value we can recover an interface value using the Interface method;

    \n
    1
    2
    3
    4
    5
    // Interface returns v's current value as an interface{}.
    // It is equivalent to:
    //
    //\tvar i interface{} = (v's underlying value)
    func (v Value) Interface() interface{}
    \n
    1
    2
    y := v.Interface().(float64) // y will have type float64.
    fmt.Println(y)
    \n
    3. \n

  3. To modify a reflection object, the value must be settable.
    The CanSet method of Value reports the settability of a Value; in our case,

    \n
    1
    2
    3
    var x float64 = 3.4
    v := reflect.ValueOf(x)
    fmt.Println("settability of v:", v.CanSet())
    \n
    1
    settability of v: false
    \n

    we pass a copy of x to reflect.ValueOf, so the interface value created as the argument to reflect.ValueOf is a copy of x, not x itself. Thus, if the statement v.SetFloat(7.1) were allowed to succeed, it would not update x, even though v looks like it was created from x. Instead, it would update the copy of x stored inside the reflection value and x itself would be unaffected. That would be confusing and useless, so it is illegal, and settability is the property used to avoid this issue. If we want to modify x by reflection, we must give the reflection library a pointer to the value we want to modify.
    Let’s do that.

    \n
    1
    2
    3
    4
    var x float64 = 3.4
    p := reflect.ValueOf(&x) // Note: take the address of x.
    fmt.Println("type of p:", p.Type())
    fmt.Println("settability of p:", p.CanSet())
    \n

    The reflection object p isn’t settable, but it’s not p we want to set, it’s (in effect) *p. To get to what p points to, we call the Elem method of Value, which indirects through the pointer, and save the result in a reflection Value called v:

    \n
    1
    2
    v := p.Elem()
    fmt.Println("settability of v:", v.CanSet())
    \n
    1
    settability of v: true
    4. \n
\n

structs

1
2
3
4
5
6
7
8
9
10
11
12
type T struct {
    A int
    B string
}
t := T{23, "skidoo"}
s := reflect.ValueOf(&t).Elem()
typeOfT := s.Type()
for i := 0; i < s.NumField(); i++ {
    f := s.Field(i)
    fmt.Printf("%d: %s %s = %v\\n", i,
        typeOfT.Field(i).Name, f.Type(), f.Interface())
}
\n
1
2
0: A int = 23
1: B string = skidoo
\n

There’s one more point about settability introduced in passing here: the field names of T are upper case (exported) because only exported fields of a struct are settable.
Because s contains a settable reflection object, we can modify the fields of the structure.

\n
1
2
3
s.Field(0).SetInt(77)
s.Field(1).SetString("Sunset Strip")
fmt.Println("t is now", t)
\n

If we modified the program so that s was created from t, not &t, the calls to SetInt and SetString would fail as the fields of t would not be settable.

\n

references

\n"},{"title":"go similar concepts comparison","date":"2023-03-05T02:44:50.000Z","_content":"\n## struct{} & struct{}{}\n`struct` is a keyword in Go. It is used to define struct types, which is a sequence of named elements.\n\nFor example:\n```go\ntype Person struct {\n Name string\n Age int\n}\n```\nThe `struct{}` is a struct type with zero elements. It is often used when no information is to be stored. It has the benefit of being 0-sized, so usually no memory is required to store a value of type `struct{}`.\n\n`struct{}{}` on the other hand is a [composite literal](https://go.dev/ref/spec#Composite_literals), it constructs a value of type `struct{}`. A composite literal constructs values for types such as `structs`, `arrays`, `maps` and `slices`. Its syntax is the type followed by the elements in braces. Since the \"empty\" struct (struct{}) has no fields, the elements list is also empty:\n\n struct{} {}\n| ^ | ^\n type empty element list\nAs an example let's create a \"set\" in Go. Go does not have a builtin set data structure, but it has a builtin map. We can use a map as a set, as a map can only have at most one entry with a given key. And since we want to only store keys (elements) in the map, we may choose the map value type to be struct{}.\n\nA map with string elements:\n```go\nvar set map[string]struct{}\n// Initialize the set\nset = make(map[string]struct{})\n\n// Add some values to the set:\nset[\"red\"] = struct{}{}\nset[\"blue\"] = struct{}{}\n\n// Check if a value is in the map:\n_, ok := set[\"red\"]\nfmt.Println(\"Is red in the map?\", ok)\n_, ok = set[\"green\"]\nfmt.Println(\"Is green in the map?\", ok)\n```","source":"_posts/golang/go-similar-concepts-comparison.md","raw":"---\ntitle: go similar concepts comparison\ndate: 2023-03-05 10:44:50\ntags: [golang]\n---\n\n## struct{} & struct{}{}\n`struct` is a keyword in Go. It is used to define struct types, which is a sequence of named elements.\n\nFor example:\n```go\ntype Person struct {\n Name string\n Age int\n}\n```\nThe `struct{}` is a struct type with zero elements. It is often used when no information is to be stored. It has the benefit of being 0-sized, so usually no memory is required to store a value of type `struct{}`.\n\n`struct{}{}` on the other hand is a [composite literal](https://go.dev/ref/spec#Composite_literals), it constructs a value of type `struct{}`. A composite literal constructs values for types such as `structs`, `arrays`, `maps` and `slices`. Its syntax is the type followed by the elements in braces. Since the \"empty\" struct (struct{}) has no fields, the elements list is also empty:\n\n struct{} {}\n| ^ | ^\n type empty element list\nAs an example let's create a \"set\" in Go. Go does not have a builtin set data structure, but it has a builtin map. We can use a map as a set, as a map can only have at most one entry with a given key. And since we want to only store keys (elements) in the map, we may choose the map value type to be struct{}.\n\nA map with string elements:\n```go\nvar set map[string]struct{}\n// Initialize the set\nset = make(map[string]struct{})\n\n// Add some values to the set:\nset[\"red\"] = struct{}{}\nset[\"blue\"] = struct{}{}\n\n// Check if a value is in the map:\n_, ok := set[\"red\"]\nfmt.Println(\"Is red in the map?\", ok)\n_, ok = set[\"green\"]\nfmt.Println(\"Is green in the map?\", ok)\n```","slug":"golang/go-similar-concepts-comparison","published":1,"updated":"2023-11-05T04:21:13.733Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8dq000uqwsj46jo8rlt","content":"

struct{} & struct{}{}

struct is a keyword in Go. It is used to define struct types, which is a sequence of named elements.

\n

For example:

\n
1
2
3
4
type Person struct {
    Name string
    Age  int
}
\n

The struct{} is a struct type with zero elements. It is often used when no information is to be stored. It has the benefit of being 0-sized, so usually no memory is required to store a value of type struct{}.

\n

struct{}{} on the other hand is a composite literal, it constructs a value of type struct{}. A composite literal constructs values for types such as structs, arrays, maps and slices. Its syntax is the type followed by the elements in braces. Since the “empty” struct (struct{}) has no fields, the elements list is also empty:

\n

struct{} {}
| ^ | ^
type empty element list
As an example let’s create a “set” in Go. Go does not have a builtin set data structure, but it has a builtin map. We can use a map as a set, as a map can only have at most one entry with a given key. And since we want to only store keys (elements) in the map, we may choose the map value type to be struct{}.

\n

A map with string elements:

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
var set map[string]struct{}
// Initialize the set
set = make(map[string]struct{})

// Add some values to the set:
set["red"] = struct{}{}
set["blue"] = struct{}{}

// Check if a value is in the map:
_, ok := set["red"]
fmt.Println("Is red in the map?", ok)
_, ok = set["green"]
fmt.Println("Is green in the map?", ok)
","site":{"data":{}},"excerpt":"","more":"

struct{} & struct{}{}

struct is a keyword in Go. It is used to define struct types, which is a sequence of named elements.

\n

For example:

\n
1
2
3
4
type Person struct {
    Name string
    Age  int
}
\n

The struct{} is a struct type with zero elements. It is often used when no information is to be stored. It has the benefit of being 0-sized, so usually no memory is required to store a value of type struct{}.

\n

struct{}{} on the other hand is a composite literal, it constructs a value of type struct{}. A composite literal constructs values for types such as structs, arrays, maps and slices. Its syntax is the type followed by the elements in braces. Since the “empty” struct (struct{}) has no fields, the elements list is also empty:

\n

struct{} {}
| ^ | ^
type empty element list
As an example let’s create a “set” in Go. Go does not have a builtin set data structure, but it has a builtin map. We can use a map as a set, as a map can only have at most one entry with a given key. And since we want to only store keys (elements) in the map, we may choose the map value type to be struct{}.

\n

A map with string elements:

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
var set map[string]struct{}
// Initialize the set
set = make(map[string]struct{})

// Add some values to the set:
set["red"] = struct{}{}
set["blue"] = struct{}{}

// Check if a value is in the map:
_, ok := set["red"]
fmt.Println("Is red in the map?", ok)
_, ok = set["green"]
fmt.Println("Is green in the map?", ok)
"},{"title":"rust resource","date":"2022-09-27T14:04:38.000Z","_content":"\n## learning resources\n- [the book](https://doc.rust-lang.org/book/)\n- [cargo book](https://doc.rust-lang.org/cargo/index.html)\n- [the rust performance book](https://nnethercote.github.io/perf-book/title-page.html)\n- [rust design patterns](https://rust-unofficial.github.io/patterns/intro.html)\n- [the rust RFC book](https://rust-lang.github.io/rfcs/)\n- [the rustdoc book](https://doc.rust-lang.org/rustdoc/what-is-rustdoc.html)\n- [rustnomicon](https://doc.rust-lang.org/nomicon/intro.html)\n- [the rust reference](https://doc.rust-lang.org/reference/introduction.html)\n- [rust by example](https://doc.rust-lang.org/rust-by-example/index.html)\n- [async programming in rust](https://rust-lang.github.io/async-book/01_getting_started/01_chapter.html)\n- [rust compiler development guide](https://rustc-dev-guide.rust-lang.org/about-this-guide.html)\n- [the little book of rust macros](https://veykril.github.io/tlborm/)\n\n## video resources\n- [youtube channel](https://www.youtube.com/@jonhoo)\n\n## books\n- [atomics and locks](https://marabos.nl/atomics/)\n- [data structure & algorithm](https://github.com/QMHTMY/RustBook/blob/main/books/rust-book-zh-cn-shieber.pdf)","source":"_posts/rust/rust-01-resource.md","raw":"---\ntitle: rust resource\ndate: 2022-09-27 22:04:38\ntags: [rust]\n---\n\n## learning resources\n- [the book](https://doc.rust-lang.org/book/)\n- [cargo book](https://doc.rust-lang.org/cargo/index.html)\n- [the rust performance book](https://nnethercote.github.io/perf-book/title-page.html)\n- [rust design patterns](https://rust-unofficial.github.io/patterns/intro.html)\n- [the rust RFC book](https://rust-lang.github.io/rfcs/)\n- [the rustdoc book](https://doc.rust-lang.org/rustdoc/what-is-rustdoc.html)\n- [rustnomicon](https://doc.rust-lang.org/nomicon/intro.html)\n- [the rust reference](https://doc.rust-lang.org/reference/introduction.html)\n- [rust by example](https://doc.rust-lang.org/rust-by-example/index.html)\n- [async programming in rust](https://rust-lang.github.io/async-book/01_getting_started/01_chapter.html)\n- [rust compiler development guide](https://rustc-dev-guide.rust-lang.org/about-this-guide.html)\n- [the little book of rust macros](https://veykril.github.io/tlborm/)\n\n## video resources\n- [youtube channel](https://www.youtube.com/@jonhoo)\n\n## books\n- [atomics and locks](https://marabos.nl/atomics/)\n- [data structure & algorithm](https://github.com/QMHTMY/RustBook/blob/main/books/rust-book-zh-cn-shieber.pdf)","slug":"rust/rust-01-resource","published":1,"updated":"2023-11-05T04:21:13.734Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8dr000wqwsjaarwgrmn","content":"

learning resources

\n

video resources

\n

books

\n","site":{"data":{}},"excerpt":"","more":"

learning resources

\n

video resources

\n

books

\n"},{"title":"rust ownership","date":"2022-10-11T09:09:28.000Z","_content":"\n## ownership rule\n- Each value in Rust has an owner.\n- There can only be one owner at a time.\n- When the owner goes out of scope, the value will be dropped.\n\n## variable scope\n```rust\nfn main() {\n { // s is not valid here, it’s not yet declared\n let s = \"hello\"; // s is valid from this point forward\n // do stuff with s\n } // this scope is now over, and s is no longer valid\n}\n```\n\n## Move\n### stack-only data: Copy trait\nsuch as primitive type\n```rust\nfn main() {\n let x = 5;\n let y = x;\n}\n```\nbind the value 5 to x; then make a copy of the value in x and bind it to y.\n\n### for heap variable\n```rust\nfn main() {\n let s1 = String::from(\"hello\");\n let s2 = s1;\n}\n```\n![move-string](/images/rust/ownership/move-string.png)\nptr, len, capacity is stored in stack, while string value is stored in heap \\\nWhen we assign s1 to s2, the String data is copied, meaning we copy the pointer, the length, and the capacity that are on the stack. We do not copy the data on the heap\n![move-string-2](/images/rust/ownership/move-string-2.png)\nsimilar to shallow copy\n\n## Clone\n```rust\nfn main() {\n let s1 = String::from(\"hello\");\n let s2 = s1.clone();\n\n println!(\"s1 = {}, s2 = {}\", s1, s2);\n}\n```\nIf we do want to deeply copy the heap data of the String, not just the stack data, we can use a common method called clone.\n\n### ownership and function\n- Passing a value to a function will result in a move or copy of ownership\n- difference between \"stack\" and \"heap\" variables: stack variables will be copied, and heap variables will be moved. When a variable containing heap data leaves the scope, its value will be cleared by the drop function, unless the ownership of the data is moved to another variable\n```rust\nfn main() {\n let s = String::from(\"hello\"); // s comes into scope\n takes_ownership(s); // s's value moves into the function...\n // ... and so is no longer valid here\n\n let x = 5; // x comes into scope\n\n makes_copy(x); // x would move into the function,\n // but i32 is Copy, so it's okay to still\n // use x afterward\n\n} // Here, x goes out of scope, then s. But because s's value was moved, nothing\n // special happens.\n\nfn takes_ownership(some_string: String) { // some_string comes into scope\n println!(\"{}\", some_string);\n} // Here, some_string goes out of scope and `drop` is called. The backing\n // memory is freed.\n\nfn makes_copy(some_integer: i32) { // some_integer comes into scope\n println!(\"{}\", some_integer);\n} // Here, some_integer goes out of scope. Nothing special happens.\n\n```\n\n### return values and scope\n```rust\nfn main() {\n let s1 = gives_ownership(); // gives_ownership moves its return\n // value into s1\n\n let s2 = String::from(\"hello\"); // s2 comes into scope\n\n let s3 = takes_and_gives_back(s2); // s2 is moved into\n // takes_and_gives_back, which also\n // moves its return value into s3\n} // Here, s3 goes out of scope and is dropped. s2 was moved, so nothing\n // happens. s1 goes out of scope and is dropped.\n\nfn gives_ownership() -> String { // gives_ownership will move its\n // return value into the function\n // that calls it\n\n let some_string = String::from(\"yours\"); // some_string comes into scope\n\n some_string // some_string is returned and\n // moves out to the calling\n // function\n}\n\n// This function takes a String and returns one\nfn takes_and_gives_back(a_string: String) -> String { // a_string comes into\n // scope\n\n a_string // a_string is returned and moves out to the calling function\n}\n```\n> What if we want to let a function use a value but not take ownership? \n> that's reference\n \n\n## reference & borrow\n- & means reference (borrow but not own)， default immutable\n- &mut a mutable reference， only one mutable reference allowed in same scope (avoid data racing)\n- Multiple mutable references can be created non-simultaneously by creating a new scope\n- **Cannot have mutable and immutable references at the same time**\n```rust\nfn main() {\n let mut s = String::from(\"hello\");\n {\n let s1 = &mut s;\n } // r1 goes out of scope here, so we can make a new reference with no problems.\n let s2 = &mut s;\n}\n\nfn main() {\n let mut s = String::from(\"hello\");\n\n let r1 = &s; // no problem\n let r2 = &s; // no problem\n println!(\"{} and {}\", r1, r2);\n // variables r1 and r2 will not be used after this point\n\n let r3 = &mut s; // no problem\n println!(\"{}\", r3);\n\n println!{\"{}\",r1} // got problem with above mutable borrow\n}\n```\n\n### reference as function arguments\n```rust\nfn main() {\n let s1 = String::from(\"hello\");\n\n let len = calculate_length(&s1);\n\n println!(\"The length of '{}' is {}.\", s1, len);\n}\n\nfn calculate_length(s: &String) -> usize { // s is a reference to a String\n s.len() \n} // Here, s goes out of scope. But because it does not have ownership of what\n // it refers to, nothing happens.\n```\nwe pass &s1 into calculate_length and, in its definition, we take &String rather than String. These ampersands represent references, and they allow you to refer to some value without taking ownership of it. Because it does not own it, the value it points to will not be dropped when the reference stops being used. \\\n When functions have references as parameters instead of the actual values, we won't need to return the values in order to give back ownership, because we never had ownership. \\\n We call the action of creating a reference borrowing. \n\n### mutable references\n```rust\nfn main() {\n let mut s = String::from(\"hello\");\n\n change(&mut s);\n}\n\nfn change(some_string: &mut String) {\n some_string.push_str(\", world\");\n}\n```\n\n### dangling references\n- A pointer refers to an address in memory, but the memory may have been freed and allocated for use by someone else\n- rust，The compiler can guarantee that there will never be dangling references\n```rust\nfn main() {\n let r = dangle();\n}\nfn dangle() -> &string { // dangle returns a reference to a String\n let s = String::from(\"hello\"); // s is a new String\n &s // we return a reference to the String, s\n} // Here, s goes out of scope, and is dropped. Its memory goes away.\n// Danger\n```\nThe solution here is to return the String directly:\n```rust\nfn main() {\n let string = no_dangle();\n}\n\nfn no_dangle() -> String {\n let s = String::from(\"hello\");\n s\n}\n```\nThis works without any problems. Ownership is moved out, and nothing is deallocated.\n\n\n## slice\n- Slices let you reference a contiguous sequence of elements in a collection rather than the whole collection. A slice is a kind of reference, so it does not have ownership.\n\n### string slice\n```rust\nfn main() {\n let mut s = String::from(\"Hello world\");\n\n let hello = &s[0..5]; \n let world = &s[6..11];\n}\n```\nRather than a reference to the entire String, hello is a reference to a portion of the String, \\\nWith Rust's `..` range syntax, if you want to start at index zero, you can drop the value before the two periods \\\nBy the same token, if your slice includes the last byte of the String, you can drop the trailing number. \n> Note: String slice range indices must occur at **valid UTF-8 character boundaries**. If you attempt to create a string slice in the middle of a multibyte character, your program will exit with an error.\n\n```rust\nfn first_word(s :&String) -> &str {\n let bytes = s.as_bytes();\n for(i, &item) in bytes.iter().enumerate() {\n if item == b' ' {\n return &s[..i];\n }\n }\n &s[..]\n}\n```\n\n### String Literals Are Slices\n```rust\nfn main() {\nlet s = \"Hello, world!\";\n}\n```\nThe type of s here is &str: it's a slice pointing to that specific point of the binary. \n\n### String Slices as Parameters\n- Pass &str as a parameter, you can receive parameters of type &String and &str at the same time\n```rust\nfn first_word(s: &String) -> &str\n```\nequivalent to\n```rust\nfn first_word(s: &str) -> &str\n```\n\n### other slices\narray slice\n```rust\nfn main() {\n let a = [1, 2, 3, 4, 5];\n let slice = &a[1..3];\n assert_eq!(slice, &[2, 3]);\n}\n```","source":"_posts/rust/rust-03-ownership.md","raw":"---\ntitle: rust ownership\ndate: 2022-10-11 17:09:28\ntags: [rust]\n---\n\n## ownership rule\n- Each value in Rust has an owner.\n- There can only be one owner at a time.\n- When the owner goes out of scope, the value will be dropped.\n\n## variable scope\n```rust\nfn main() {\n { // s is not valid here, it’s not yet declared\n let s = \"hello\"; // s is valid from this point forward\n // do stuff with s\n } // this scope is now over, and s is no longer valid\n}\n```\n\n## Move\n### stack-only data: Copy trait\nsuch as primitive type\n```rust\nfn main() {\n let x = 5;\n let y = x;\n}\n```\nbind the value 5 to x; then make a copy of the value in x and bind it to y.\n\n### for heap variable\n```rust\nfn main() {\n let s1 = String::from(\"hello\");\n let s2 = s1;\n}\n```\n![move-string](/images/rust/ownership/move-string.png)\nptr, len, capacity is stored in stack, while string value is stored in heap \\\nWhen we assign s1 to s2, the String data is copied, meaning we copy the pointer, the length, and the capacity that are on the stack. We do not copy the data on the heap\n![move-string-2](/images/rust/ownership/move-string-2.png)\nsimilar to shallow copy\n\n## Clone\n```rust\nfn main() {\n let s1 = String::from(\"hello\");\n let s2 = s1.clone();\n\n println!(\"s1 = {}, s2 = {}\", s1, s2);\n}\n```\nIf we do want to deeply copy the heap data of the String, not just the stack data, we can use a common method called clone.\n\n### ownership and function\n- Passing a value to a function will result in a move or copy of ownership\n- difference between \"stack\" and \"heap\" variables: stack variables will be copied, and heap variables will be moved. When a variable containing heap data leaves the scope, its value will be cleared by the drop function, unless the ownership of the data is moved to another variable\n```rust\nfn main() {\n let s = String::from(\"hello\"); // s comes into scope\n takes_ownership(s); // s's value moves into the function...\n // ... and so is no longer valid here\n\n let x = 5; // x comes into scope\n\n makes_copy(x); // x would move into the function,\n // but i32 is Copy, so it's okay to still\n // use x afterward\n\n} // Here, x goes out of scope, then s. But because s's value was moved, nothing\n // special happens.\n\nfn takes_ownership(some_string: String) { // some_string comes into scope\n println!(\"{}\", some_string);\n} // Here, some_string goes out of scope and `drop` is called. The backing\n // memory is freed.\n\nfn makes_copy(some_integer: i32) { // some_integer comes into scope\n println!(\"{}\", some_integer);\n} // Here, some_integer goes out of scope. Nothing special happens.\n\n```\n\n### return values and scope\n```rust\nfn main() {\n let s1 = gives_ownership(); // gives_ownership moves its return\n // value into s1\n\n let s2 = String::from(\"hello\"); // s2 comes into scope\n\n let s3 = takes_and_gives_back(s2); // s2 is moved into\n // takes_and_gives_back, which also\n // moves its return value into s3\n} // Here, s3 goes out of scope and is dropped. s2 was moved, so nothing\n // happens. s1 goes out of scope and is dropped.\n\nfn gives_ownership() -> String { // gives_ownership will move its\n // return value into the function\n // that calls it\n\n let some_string = String::from(\"yours\"); // some_string comes into scope\n\n some_string // some_string is returned and\n // moves out to the calling\n // function\n}\n\n// This function takes a String and returns one\nfn takes_and_gives_back(a_string: String) -> String { // a_string comes into\n // scope\n\n a_string // a_string is returned and moves out to the calling function\n}\n```\n> What if we want to let a function use a value but not take ownership? \n> that's reference\n \n\n## reference & borrow\n- & means reference (borrow but not own)， default immutable\n- &mut a mutable reference， only one mutable reference allowed in same scope (avoid data racing)\n- Multiple mutable references can be created non-simultaneously by creating a new scope\n- **Cannot have mutable and immutable references at the same time**\n```rust\nfn main() {\n let mut s = String::from(\"hello\");\n {\n let s1 = &mut s;\n } // r1 goes out of scope here, so we can make a new reference with no problems.\n let s2 = &mut s;\n}\n\nfn main() {\n let mut s = String::from(\"hello\");\n\n let r1 = &s; // no problem\n let r2 = &s; // no problem\n println!(\"{} and {}\", r1, r2);\n // variables r1 and r2 will not be used after this point\n\n let r3 = &mut s; // no problem\n println!(\"{}\", r3);\n\n println!{\"{}\",r1} // got problem with above mutable borrow\n}\n```\n\n### reference as function arguments\n```rust\nfn main() {\n let s1 = String::from(\"hello\");\n\n let len = calculate_length(&s1);\n\n println!(\"The length of '{}' is {}.\", s1, len);\n}\n\nfn calculate_length(s: &String) -> usize { // s is a reference to a String\n s.len() \n} // Here, s goes out of scope. But because it does not have ownership of what\n // it refers to, nothing happens.\n```\nwe pass &s1 into calculate_length and, in its definition, we take &String rather than String. These ampersands represent references, and they allow you to refer to some value without taking ownership of it. Because it does not own it, the value it points to will not be dropped when the reference stops being used. \\\n When functions have references as parameters instead of the actual values, we won't need to return the values in order to give back ownership, because we never had ownership. \\\n We call the action of creating a reference borrowing. \n\n### mutable references\n```rust\nfn main() {\n let mut s = String::from(\"hello\");\n\n change(&mut s);\n}\n\nfn change(some_string: &mut String) {\n some_string.push_str(\", world\");\n}\n```\n\n### dangling references\n- A pointer refers to an address in memory, but the memory may have been freed and allocated for use by someone else\n- rust，The compiler can guarantee that there will never be dangling references\n```rust\nfn main() {\n let r = dangle();\n}\nfn dangle() -> &string { // dangle returns a reference to a String\n let s = String::from(\"hello\"); // s is a new String\n &s // we return a reference to the String, s\n} // Here, s goes out of scope, and is dropped. Its memory goes away.\n// Danger\n```\nThe solution here is to return the String directly:\n```rust\nfn main() {\n let string = no_dangle();\n}\n\nfn no_dangle() -> String {\n let s = String::from(\"hello\");\n s\n}\n```\nThis works without any problems. Ownership is moved out, and nothing is deallocated.\n\n\n## slice\n- Slices let you reference a contiguous sequence of elements in a collection rather than the whole collection. A slice is a kind of reference, so it does not have ownership.\n\n### string slice\n```rust\nfn main() {\n let mut s = String::from(\"Hello world\");\n\n let hello = &s[0..5]; \n let world = &s[6..11];\n}\n```\nRather than a reference to the entire String, hello is a reference to a portion of the String, \\\nWith Rust's `..` range syntax, if you want to start at index zero, you can drop the value before the two periods \\\nBy the same token, if your slice includes the last byte of the String, you can drop the trailing number. \n> Note: String slice range indices must occur at **valid UTF-8 character boundaries**. If you attempt to create a string slice in the middle of a multibyte character, your program will exit with an error.\n\n```rust\nfn first_word(s :&String) -> &str {\n let bytes = s.as_bytes();\n for(i, &item) in bytes.iter().enumerate() {\n if item == b' ' {\n return &s[..i];\n }\n }\n &s[..]\n}\n```\n\n### String Literals Are Slices\n```rust\nfn main() {\nlet s = \"Hello, world!\";\n}\n```\nThe type of s here is &str: it's a slice pointing to that specific point of the binary. \n\n### String Slices as Parameters\n- Pass &str as a parameter, you can receive parameters of type &String and &str at the same time\n```rust\nfn first_word(s: &String) -> &str\n```\nequivalent to\n```rust\nfn first_word(s: &str) -> &str\n```\n\n### other slices\narray slice\n```rust\nfn main() {\n let a = [1, 2, 3, 4, 5];\n let slice = &a[1..3];\n assert_eq!(slice, &[2, 3]);\n}\n```","slug":"rust/rust-03-ownership","published":1,"updated":"2023-11-05T04:21:13.734Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8dr000yqwsj41fu3ebh","content":"

ownership rule

    \n
  • Each value in Rust has an owner.
    • \n
  • There can only be one owner at a time.
    • \n
  • When the owner goes out of scope, the value will be dropped.
    • \n
\n

variable scope

1
2
3
4
5
6
fn main() {
    {                      // s is not valid here, it’s not yet declared
        let s = "hello";   // s is valid from this point forward
        // do stuff with s
    }                      // this scope is now over, and s is no longer valid
}
\n\n

Move

stack-only data: Copy trait

such as primitive type

\n
1
2
3
4
fn main() {
    let x = 5;
    let y = x;
}
\n

bind the value 5 to x; then make a copy of the value in x and bind it to y.

\n

for heap variable

1
2
3
4
fn main() {
    let s1 = String::from("hello");
    let s2 = s1;
}
\n

\"move-string\"
ptr, len, capacity is stored in stack, while string value is stored in heap
When we assign s1 to s2, the String data is copied, meaning we copy the pointer, the length, and the capacity that are on the stack. We do not copy the data on the heap
\"move-string-2\"
similar to shallow copy

\n

Clone

1
2
3
4
5
6
fn main() {
    let s1 = String::from("hello");
    let s2 = s1.clone();

    println!("s1 = {}, s2 = {}", s1, s2);
}
\n

If we do want to deeply copy the heap data of the String, not just the stack data, we can use a common method called clone.

\n

ownership and function

    \n
  • Passing a value to a function will result in a move or copy of ownership
    • \n
  • difference between “stack” and “heap” variables: stack variables will be copied, and heap variables will be moved. When a variable containing heap data leaves the scope, its value will be cleared by the drop function, unless the ownership of the data is moved to another variable
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    fn main() {
        let s = String::from("hello");  // s comes into scope
        takes_ownership(s);             // s's value moves into the function...
                                        // ... and so is no longer valid here

        let x = 5;                      // x comes into scope

        makes_copy(x);                  // x would move into the function,
                                        // but i32 is Copy, so it's okay to still
                                        // use x afterward

    } // Here, x goes out of scope, then s. But because s's value was moved, nothing
      // special happens.

    fn takes_ownership(some_string: String) { // some_string comes into scope
        println!("{}", some_string);
    } // Here, some_string goes out of scope and `drop` is called. The backing
      // memory is freed.

    fn makes_copy(some_integer: i32) { // some_integer comes into scope
        println!("{}", some_integer);
    } // Here, some_integer goes out of scope. Nothing special happens.

    • \n
\n

return values and scope

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
fn main() {
    let s1 = gives_ownership();         // gives_ownership moves its return
                                        // value into s1

    let s2 = String::from("hello");     // s2 comes into scope

    let s3 = takes_and_gives_back(s2);  // s2 is moved into
                                        // takes_and_gives_back, which also
                                        // moves its return value into s3
} // Here, s3 goes out of scope and is dropped. s2 was moved, so nothing
  // happens. s1 goes out of scope and is dropped.

fn gives_ownership() -> String {             // gives_ownership will move its
                                             // return value into the function
                                             // that calls it

    let some_string = String::from("yours"); // some_string comes into scope

    some_string                              // some_string is returned and
                                             // moves out to the calling
                                             // function
}

// This function takes a String and returns one
fn takes_and_gives_back(a_string: String) -> String { // a_string comes into
                                                      // scope

    a_string  // a_string is returned and moves out to the calling function
}
\n
\n

What if we want to let a function use a value but not take ownership?
that’s reference

\n
\n

reference & borrow

    \n
  • & means reference (borrow but not own)， default immutable
    • \n
  • &mut a mutable reference， only one mutable reference allowed in same scope (avoid data racing)
    • \n
  • Multiple mutable references can be created non-simultaneously by creating a new scope
    • \n
  • Cannot have mutable and immutable references at the same time
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    fn main() {
        let mut s = String::from("hello");
        {
            let s1 = &mut s;
        } // r1 goes out of scope here, so we can make a new reference with no problems.
        let s2 = &mut s;
    }

    fn main() {
        let mut s = String::from("hello");

        let r1 = &s; // no problem
        let r2 = &s; // no problem
        println!("{} and {}", r1, r2);
        // variables r1 and r2 will not be used after this point

        let r3 = &mut s; // no problem
        println!("{}", r3);

        println!{"{}",r1} // got problem with above mutable borrow
    }
    • \n
\n

reference as function arguments

1
2
3
4
5
6
7
8
9
10
11
12
fn main() {
    let s1 = String::from("hello");

    let len = calculate_length(&s1);

    println!("The length of '{}' is {}.", s1, len);
}

fn calculate_length(s: &String) -> usize { // s is a reference to a String
    s.len() 
}  // Here, s goes out of scope. But because it does not have ownership of what
   // it refers to, nothing happens.
\n

we pass &s1 into calculate_length and, in its definition, we take &String rather than String. These ampersands represent references, and they allow you to refer to some value without taking ownership of it. Because it does not own it, the value it points to will not be dropped when the reference stops being used.
When functions have references as parameters instead of the actual values, we won’t need to return the values in order to give back ownership, because we never had ownership.
We call the action of creating a reference borrowing.

\n

mutable references

1
2
3
4
5
6
7
8
9
fn main() {
    let mut s = String::from("hello");

    change(&mut s);
}

fn change(some_string: &mut String) {
    some_string.push_str(", world");
}
\n\n

dangling references

    \n
  • A pointer refers to an address in memory, but the memory may have been freed and allocated for use by someone else
    • \n
  • rust，The compiler can guarantee that there will never be dangling references
    1
    2
    3
    4
    5
    6
    7
    8
    fn main() {
        let r = dangle();
    }
    fn dangle() -> &string {  // dangle returns a reference to a String
        let s = String::from("hello");  // s is a new String
        &s // we return a reference to the String, s
    } // Here, s goes out of scope, and is dropped. Its memory goes away.
    // Danger
    \nThe solution here is to return the String directly:
    1
    2
    3
    4
    5
    6
    7
    8
    fn main() {
        let string = no_dangle();
    }

    fn no_dangle() -> String {
        let s = String::from("hello");
        s
    }
    \nThis works without any problems. Ownership is moved out, and nothing is deallocated.
    • \n
\n

slice

    \n
  • Slices let you reference a contiguous sequence of elements in a collection rather than the whole collection. A slice is a kind of reference, so it does not have ownership.
    • \n
\n

string slice

1
2
3
4
5
6
fn main() {
    let mut s = String::from("Hello world");

    let hello = &s[0..5]; 
    let world = &s[6..11];
}
\n

Rather than a reference to the entire String, hello is a reference to a portion of the String,
With Rust’s .. range syntax, if you want to start at index zero, you can drop the value before the two periods
By the same token, if your slice includes the last byte of the String, you can drop the trailing number.

\n
\n

Note: String slice range indices must occur at valid UTF-8 character boundaries. If you attempt to create a string slice in the middle of a multibyte character, your program will exit with an error.

\n
\n
1
2
3
4
5
6
7
8
9
fn first_word(s :&String) -> &str {
    let bytes = s.as_bytes();
    for(i, &item) in bytes.iter().enumerate() {
        if item == b' ' {
            return &s[..i];
        }
    }
    &s[..]
}
\n\n

String Literals Are Slices

1
2
3
fn main() {
let s = "Hello, world!";
}
\n

The type of s here is &str: it’s a slice pointing to that specific point of the binary.

\n

String Slices as Parameters

    \n
  • Pass &str as a parameter, you can receive parameters of type &String and &str at the same time
    1
    fn first_word(s: &String) -> &str
    \nequivalent to
    1
    fn first_word(s: &str) -> &str
    • \n
\n

other slices

array slice

\n
1
2
3
4
5
fn main() {
  let a = [1, 2, 3, 4, 5];
  let slice = &a[1..3];
  assert_eq!(slice, &[2, 3]);
}
","site":{"data":{}},"excerpt":"","more":"

ownership rule

    \n
  • Each value in Rust has an owner.
    • \n
  • There can only be one owner at a time.
    • \n
  • When the owner goes out of scope, the value will be dropped.
    • \n
\n

variable scope

1
2
3
4
5
6
fn main() {
    {                      // s is not valid here, it’s not yet declared
        let s = "hello";   // s is valid from this point forward
        // do stuff with s
    }                      // this scope is now over, and s is no longer valid
}
\n\n

Move

stack-only data: Copy trait

such as primitive type

\n
1
2
3
4
fn main() {
    let x = 5;
    let y = x;
}
\n

bind the value 5 to x; then make a copy of the value in x and bind it to y.

\n

for heap variable

1
2
3
4
fn main() {
    let s1 = String::from("hello");
    let s2 = s1;
}
\n

\"move-string\"
ptr, len, capacity is stored in stack, while string value is stored in heap
When we assign s1 to s2, the String data is copied, meaning we copy the pointer, the length, and the capacity that are on the stack. We do not copy the data on the heap
\"move-string-2\"
similar to shallow copy

\n

Clone

1
2
3
4
5
6
fn main() {
    let s1 = String::from("hello");
    let s2 = s1.clone();

    println!("s1 = {}, s2 = {}", s1, s2);
}
\n

If we do want to deeply copy the heap data of the String, not just the stack data, we can use a common method called clone.

\n

ownership and function

    \n
  • Passing a value to a function will result in a move or copy of ownership
    • \n
  • difference between “stack” and “heap” variables: stack variables will be copied, and heap variables will be moved. When a variable containing heap data leaves the scope, its value will be cleared by the drop function, unless the ownership of the data is moved to another variable
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    fn main() {
        let s = String::from("hello");  // s comes into scope
        takes_ownership(s);             // s's value moves into the function...
                                        // ... and so is no longer valid here

        let x = 5;                      // x comes into scope

        makes_copy(x);                  // x would move into the function,
                                        // but i32 is Copy, so it's okay to still
                                        // use x afterward

    } // Here, x goes out of scope, then s. But because s's value was moved, nothing
      // special happens.

    fn takes_ownership(some_string: String) { // some_string comes into scope
        println!("{}", some_string);
    } // Here, some_string goes out of scope and `drop` is called. The backing
      // memory is freed.

    fn makes_copy(some_integer: i32) { // some_integer comes into scope
        println!("{}", some_integer);
    } // Here, some_integer goes out of scope. Nothing special happens.

    • \n
\n

return values and scope

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
fn main() {
    let s1 = gives_ownership();         // gives_ownership moves its return
                                        // value into s1

    let s2 = String::from("hello");     // s2 comes into scope

    let s3 = takes_and_gives_back(s2);  // s2 is moved into
                                        // takes_and_gives_back, which also
                                        // moves its return value into s3
} // Here, s3 goes out of scope and is dropped. s2 was moved, so nothing
  // happens. s1 goes out of scope and is dropped.

fn gives_ownership() -> String {             // gives_ownership will move its
                                             // return value into the function
                                             // that calls it

    let some_string = String::from("yours"); // some_string comes into scope

    some_string                              // some_string is returned and
                                             // moves out to the calling
                                             // function
}

// This function takes a String and returns one
fn takes_and_gives_back(a_string: String) -> String { // a_string comes into
                                                      // scope

    a_string  // a_string is returned and moves out to the calling function
}
\n
\n

What if we want to let a function use a value but not take ownership?
that’s reference

\n
\n

reference & borrow

    \n
  • & means reference (borrow but not own)， default immutable
    • \n
  • &mut a mutable reference， only one mutable reference allowed in same scope (avoid data racing)
    • \n
  • Multiple mutable references can be created non-simultaneously by creating a new scope
    • \n
  • Cannot have mutable and immutable references at the same time
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    fn main() {
        let mut s = String::from("hello");
        {
            let s1 = &mut s;
        } // r1 goes out of scope here, so we can make a new reference with no problems.
        let s2 = &mut s;
    }

    fn main() {
        let mut s = String::from("hello");

        let r1 = &s; // no problem
        let r2 = &s; // no problem
        println!("{} and {}", r1, r2);
        // variables r1 and r2 will not be used after this point

        let r3 = &mut s; // no problem
        println!("{}", r3);

        println!{"{}",r1} // got problem with above mutable borrow
    }
    • \n
\n

reference as function arguments

1
2
3
4
5
6
7
8
9
10
11
12
fn main() {
    let s1 = String::from("hello");

    let len = calculate_length(&s1);

    println!("The length of '{}' is {}.", s1, len);
}

fn calculate_length(s: &String) -> usize { // s is a reference to a String
    s.len() 
}  // Here, s goes out of scope. But because it does not have ownership of what
   // it refers to, nothing happens.
\n

we pass &s1 into calculate_length and, in its definition, we take &String rather than String. These ampersands represent references, and they allow you to refer to some value without taking ownership of it. Because it does not own it, the value it points to will not be dropped when the reference stops being used.
When functions have references as parameters instead of the actual values, we won’t need to return the values in order to give back ownership, because we never had ownership.
We call the action of creating a reference borrowing.

\n

mutable references

1
2
3
4
5
6
7
8
9
fn main() {
    let mut s = String::from("hello");

    change(&mut s);
}

fn change(some_string: &mut String) {
    some_string.push_str(", world");
}
\n\n

dangling references

    \n
  • A pointer refers to an address in memory, but the memory may have been freed and allocated for use by someone else
    • \n
  • rust，The compiler can guarantee that there will never be dangling references
    1
    2
    3
    4
    5
    6
    7
    8
    fn main() {
        let r = dangle();
    }
    fn dangle() -> &string {  // dangle returns a reference to a String
        let s = String::from("hello");  // s is a new String
        &s // we return a reference to the String, s
    } // Here, s goes out of scope, and is dropped. Its memory goes away.
    // Danger
    \nThe solution here is to return the String directly:
    1
    2
    3
    4
    5
    6
    7
    8
    fn main() {
        let string = no_dangle();
    }

    fn no_dangle() -> String {
        let s = String::from("hello");
        s
    }
    \nThis works without any problems. Ownership is moved out, and nothing is deallocated.
    • \n
\n

slice

    \n
  • Slices let you reference a contiguous sequence of elements in a collection rather than the whole collection. A slice is a kind of reference, so it does not have ownership.
    • \n
\n

string slice

1
2
3
4
5
6
fn main() {
    let mut s = String::from("Hello world");

    let hello = &s[0..5]; 
    let world = &s[6..11];
}
\n

Rather than a reference to the entire String, hello is a reference to a portion of the String,
With Rust’s .. range syntax, if you want to start at index zero, you can drop the value before the two periods
By the same token, if your slice includes the last byte of the String, you can drop the trailing number.

\n
\n

Note: String slice range indices must occur at valid UTF-8 character boundaries. If you attempt to create a string slice in the middle of a multibyte character, your program will exit with an error.

\n
\n
1
2
3
4
5
6
7
8
9
fn first_word(s :&String) -> &str {
    let bytes = s.as_bytes();
    for(i, &item) in bytes.iter().enumerate() {
        if item == b' ' {
            return &s[..i];
        }
    }
    &s[..]
}
\n\n

String Literals Are Slices

1
2
3
fn main() {
let s = "Hello, world!";
}
\n

The type of s here is &str: it’s a slice pointing to that specific point of the binary.

\n

String Slices as Parameters

    \n
  • Pass &str as a parameter, you can receive parameters of type &String and &str at the same time
    1
    fn first_word(s: &String) -> &str
    \nequivalent to
    1
    fn first_word(s: &str) -> &str
    • \n
\n

other slices

array slice

\n
1
2
3
4
5
fn main() {
  let a = [1, 2, 3, 4, 5];
  let slice = &a[1..3];
  assert_eq!(slice, &[2, 3]);
}
"},{"title":"rust basics","date":"2022-10-04T07:55:04.000Z","_content":"\n## frequently used cmd\n```\nrustc [filename].rs\ncargo new [project_name]\ncargo build [--release]\ncargo run [--release]\ncargo check # check whether compile success, no executible output\n```\n\n## data type\n\n### integer\n- i8,i16,i32,i64,i128,isize,u8,u16,u32,u64,u128,usize，etc\n- isize, usize indicates that the type is determined by the architecture of the computer. For example, on a 32 bit target, this is 4 bytes and on a 64 bit target, this is 8 bytes.\n- 0x: hex，0o Octal，0b binary，starting with b: byte （u8 only）\n\n| Number Literals | Example |\n| ----------- | ----------- |\n| Decimal | 98_222 |\n| Hex | 0xff |\n| Octal | 0o77 |\n| Binary | 0b1111_0000 |\n| Byte(u8 only) | b'A' |\n\n### Tuple\n- The length of Tuple is fixed, and the length cannot be changed once declared\n```rust\nfn main() {\n // tuple could be declared as mut\n let mut tuple_1 = (\"Hello\", 39, \"Years\");\n let tuple_2:(i32, &str ) = (1983, \"since.\");\n tuple_1.0 = \"Hi\";\n println!(\"{} {} {}\", tuple_1.0, tuple_1.1, tuple_1.2);\n // destructure\n let (a,b) = tuple_2;\n println!(\"{} {}\", a, b);\n}\n```\n\n### array\n- arrays in Rust have a fixed length.\n- Vector is similar to an array, it is provided by the standard library, and its length can be changed\n\n```rust\nfn main() {\n\n let arr_test:[u8; 3] = [1,2,3];\n println!(\"Number is {},{},{}\", arr_test[0],arr_test[1],arr_test[2]);\n\n let arr_test = [\"I\",\"love\",\"you\"];\n println!(\"You said : {} {} {}\", arr_test[0],arr_test[1],arr_test[2]);\n\n let arr_test = [1;3]; \n println!(\"Call Num : {}&{}&{}\", arr_test[0],arr_test[1],arr_test[2]);\n}\n```\n\n\n\n### String\n- Basic data types are stored on the stack, but the String type is stored on the heap\n```rust\nlet s = String::from(\"hello\");\n```\n- push_str(): append a str slice a string\n- push(): appends a single character to a String\n```rust\nfn main() { \n let mut data = String::from(\"andy\");\n data.push_str(\" is stronger\");\n data.push('!');\n}\n```\n- + operator, chaining strings. the left side of the + operator is the ownership of the string, and the right side is the string slice\n- String is actually a wrapper for Vec, so the length can be measured by the len() method, but note that Len() is not length of character, but byte len\n- String iteration\n```rust\nfn main() { \n let mut data = String::from(\"andy\");\n data.push_str(\" is stronger\");\n data.push('!');\n\n for i in data.bytes() {\n ///\n }\n\n for i in data.chars() {\n ///\n }\n}\n```\n\n### Vector\n- Vector is like any other struct. When Vector leaves the scope, the variable value is cleaned up, and all its elements are also cleaned up.\n```rust\nfn main() {\n let vec: Vec = Vec::new();\n let vec2: Vec = vec![3,4，5] // create vector by macro\n for i in vec2 {\n println!(\"Vector value is : {}\", i);\n }\n}\n```\n\n### HashMap\n- HashMap is not preloaded, so it needs to be included `use std::collections::HashMap`\n```rust\nuse std::collections::HashMap;\nfn main() {\n let keys = vec![\"andy\".to_string(), \"cliff\".to_string()] ;\n let ages = vec![38, 26];\n let map :HashMap<_,_> = keys.iter().zip(ages.iter()).collect();\n println!(\"{:?}\", map); /// print {\"andy\": 38, \"cliff\": 26}\n}\n```\n#### HashMap ownership\n- For types that implement the Copy trait (such as i32), the value will be copied into the HashMap\n- For values with ownership, such as (String), the value will be moved and ownership will be given to HashMap\n- If a reference to a value is inserted into the HashMap, the value itself does not move\n\n#### HashMap iteration\n```rust\nuse std::collections::HashMap;\n\nfn main() { \n let name = \"andy\".to_string();\n let age = 36;\n let mut map = HashMap::new();\n map.insert(name, age);\n map.insert(String::from(\"cliff\"), 26);\n println!(\"{:?}\", &map);\n for (k, v) in map {\n println!(\"{} age {}\", k, v);\n } /// cliff age 26\n /// andy age 36\n}\n```\n#### update\n```rust\nuse std::collections::HashMap;\n\nfn main() { \n let name = \"andy\".to_string();\n let age = 36;\n let mut map = HashMap::new();\n map.insert(name, age);\n map.insert(String::from(\"cliff\"), 26);\n\n let result = map.entry(\"bob\".to_string());\n println!(\"{:?}\", result); /// Entry(VacantEntry(\"bob\"))\n\n let result = map.entry(\"andy\".to_string());\n println!(\"{:?}\", result); /// Entry(OccupiedEntry { key: \"andy\", value: 36, .. })\n\n map.entry(\"bob\".to_string()).or_insert(28);\n map.entry(\"cliff\".to_string()).or_insert(0);\n}\n```\n\n## control flow\n- if\n```rust\nfn main() {\n let condition = 1;\n let x = if condition == 1 { \"A\" } else { \"B\" };\n println!(\"Result x = {}\" , x) ;\n}\n```\n- loop\n```rust\nfn main() {\n let mut condition = 0;\n\n let result = 'outer: loop { // 'outer is label\n 'inner: loop {\n condition += 1;\n if 3 == condition {\n break 'outer 3 * condition; // break outer loop\n }\n }\n };\n println!(\"Loop result is : {}\", result); /// Loop result is : 9\n}\n\n```\n- rot\n```rust\nfn main() {\n let arr = [3,2,3];\n for num in arr.iter() {\n println!(\"For value is {}\", num);\n }\n}\n```\n\n## Range iterator\n- Range\n```rust\nfn main() {\n for number in (1..=3) {\n println!(\"Number A is {}\", number ); /// 1,2,3\n }\n \n for number in (1..=3).rev() { /// rev means reverse,\n println!(\"Number B is {}\", number ); /// 3,2,1\n }\n}\n\n```\n\n## struct\n- If struct is declared mutable then all fields in the instance are mutable\n### tuple struct\n```rust\nstruct Color(i32,i32,i32);\nlet black = Color(0,0,0);\n```\n### Unit-Like struct\n```rust\nstruct Man {};\n```\n### struct method\n```rust\n\nfn main() {\n let rec = Rectangle {\n width: 30,\n height: 50,\n };\n\n let result = rec.area(); \n println!(\"rectangle：{:?}，area is：{}\", rec, result);\n}\n\n\n#[derive(Debug)]\nstruct Rectangle {\n width: u32,\n height: u32,\n}\n\nimpl Rectangle {\n fn area(&self) -> u32{\n self.width * self.height\n }\n}\n\n```\n### associative func（similar to static method）\n- You can define a function that does not take `self` as the first parameter in the impl block. This form is called an associated function, and the calling method is similar to `String::from()`\n```rust\nimpl Rectangle {\n fn create_square(width: u32) -> Rectangle {\n Rectangle {\n width,\n height: width,\n }\n }\n}\n```\n \n## enum\n```rust\nenum Ip {\n V4,\n V6,\n}\n\nenum IpAddr {\n V4(String),\n V6(String),\n}\n``` \n\n## match\n- match must exhaust all possibilities\n- If there are too many matchings, you can also use \"_\" for wildcarding, but note that \"_\" must be placed at the end\n```rust\nenum Color {\n Red,\n Yellow,\n Blue,\n}\nenum ColorWithVal {\n Red(u8,u8,u8),\n Yellow(u8,u8,u8),\n Blue(u8,u8,u8),\n}\nfn main(){\n let colour = Color::Blue;\n match colour {\n Color::Red => {\n println!(\"Red colour.\");\n },\n _ => {\n println!(\"Other colour.\");\n }\n }\n\n let colour = ColorWithVal::Red(222,111,22);\n match colour {\n ColorWithVal::Red(r,g,b) => {\n println!(\"Red colour. {},{},{}\", r,g,b);\n },\n _ => {\n println!(\"Other colour.\");\n }\n }\n}\n```\n\n## if let\n```rust\nfn main(){\n let colour = Color::Red(Some(222),Some(222),Some(222));\n\n if let Color::Red(r,g,b) = colour {\n println!(\"Red colour. {:?},{:?},{:?}\", r,g,b);\n } else {\n println!(\"Other colour.\");\n }\n}\n```\n\n## Result\n- Recoverable err via Result, non-recoverable via panic!\n- upon panic!, the program will expand an error message, unwind, clean up the call stack (Stack) and finally exit the program\n- You can set panic = 'abort' in Cargo.toml to terminate the cleaning of the call stack\n```rust\n[profile.release]\npanic='abort'\n```\n- RUST_BACKTRACE = 1 prints detailed error messages in the stack\n```rust\nuse std::fs::File;\nfn main() { \n let fp = File::open(\"hello.txt\");\n let file = match fp {\n Ok(file)=> {\n file\n },\n Err(error) => panic!(\"file not found {:?} \", error),\n };\n}\n```\n\n```rust\nuse std::{fs::File, io::ErrorKind};\nfn main() { \n let fp = File::open(\"hello.txt\");\n let file = match fp {\n Ok(file)=> {\n file\n },\n Err(error) => {\n match error.kind() {\n ErrorKind::NotFound => {\n match File::create(\"hello.txt\") {\n Ok(file) => {\n file\n },\n Err(err) => {\n panic!(\"file create error:{:?}\", &err);\n },\n }\n },\n oe => panic!(\"other error {:?}\", oe),\n }\n } ,\n };\n}\n```\n```rust\nuse std::{fs::File, io::ErrorKind};\nfn main() { \n let file = File::open(\"hello.txt\").unwrap_or_else(|err| {\n if err.kind() == ErrorKind::NotFound {\n File::create(\"hello.txt\").unwrap_or_else(|err|{\n panic!(\"error：{:?}\", err);\n })\n }else{\n panic!(\"other error：{:?}\", err);\n }\n });\n}\n```\n### unwrap && expect\n- If do not want to deal with Err, can use unwarp() method. If result is Ok(val), return val. If Err, then call the panic! macro.\n- expect can specify what the error message is, which is easier to debug\n\n### The question mark operator, ?\nWhen writing code that calls many functions that return the Result type, the error handling can be tedious. The question mark operator, ?, hides some of the boilerplate of propagating errors up the call stack.\n```rust\nlet mut file = File::create(\"my_best_friends.txt\")?;\n```\n\n## generic\n```rust\n#[derive(Debug)]\nstruct Point {\n x : T,\n y : U,\n}\nimpl Point {\n fn mixup(self, other: Point) -> Point {\n Point{x: self.x , y: other.y, }\n }\n}\n```\n\n## trait\n### definition\n```rust\npub trait Summary {\n fn summarize(&self) -> String {\n \"... more\".to_string() /// default \n }\n}\npub struct Tweet {\n user_name :String,\n replay_count :u32,\n like_count :u32,\n}\nimpl Tweet {\n fn like(&mut self) {\n self.like_count += 1;\n }\n}\n\nimpl Summary for Tweet {\n fn summarize(&self) -> String {\n format!(\"{} like count :{} , replay count :{}\", &self.user_name, &self.replay_count, &self.like_count)\n }\n}\n```\n\n### trait as arguments\n```rust\nfn notify_msg (info: T) {\n println!(\"summary : {}\", info.summarize() );\n}\nfn notify_msg (info: impl Summary + Display) {\n println!(\"summary : {}\", info.summarize() );\n}\nfn notify_msg (info: T) \nwhere \n T: Summary + Display,\n{\n println!(\"summary : {}\", info.summarize() );\n println!(\"display implement info : {}\", info);\n}\n```\n### trait as return\n- impl Trait can only return the same type, if it returns a different type, even if the Trait is implemented, an error will be reported\n\n## references\n- [the rust programming language](https://doc.rust-lang.org/book/)\n- [mooc course](https://time.geekbang.org/course/intro/100060601?tab=catalog)\n- [bilibili tutorial](https://www.bilibili.com/video/BV1hp4y1k7SV?p=50&spm_id_from=pageDriver)\n- [jianshu notes](https://www.jianshu.com/p/30d917790298)","source":"_posts/rust/rust-02-basics.md","raw":"---\ntitle: rust basics\ndate: 2022-10-04 15:55:04\ntags: [rust]\n---\n\n## frequently used cmd\n```\nrustc [filename].rs\ncargo new [project_name]\ncargo build [--release]\ncargo run [--release]\ncargo check # check whether compile success, no executible output\n```\n\n## data type\n\n### integer\n- i8,i16,i32,i64,i128,isize,u8,u16,u32,u64,u128,usize，etc\n- isize, usize indicates that the type is determined by the architecture of the computer. For example, on a 32 bit target, this is 4 bytes and on a 64 bit target, this is 8 bytes.\n- 0x: hex，0o Octal，0b binary，starting with b: byte （u8 only）\n\n| Number Literals | Example |\n| ----------- | ----------- |\n| Decimal | 98_222 |\n| Hex | 0xff |\n| Octal | 0o77 |\n| Binary | 0b1111_0000 |\n| Byte(u8 only) | b'A' |\n\n### Tuple\n- The length of Tuple is fixed, and the length cannot be changed once declared\n```rust\nfn main() {\n // tuple could be declared as mut\n let mut tuple_1 = (\"Hello\", 39, \"Years\");\n let tuple_2:(i32, &str ) = (1983, \"since.\");\n tuple_1.0 = \"Hi\";\n println!(\"{} {} {}\", tuple_1.0, tuple_1.1, tuple_1.2);\n // destructure\n let (a,b) = tuple_2;\n println!(\"{} {}\", a, b);\n}\n```\n\n### array\n- arrays in Rust have a fixed length.\n- Vector is similar to an array, it is provided by the standard library, and its length can be changed\n\n```rust\nfn main() {\n\n let arr_test:[u8; 3] = [1,2,3];\n println!(\"Number is {},{},{}\", arr_test[0],arr_test[1],arr_test[2]);\n\n let arr_test = [\"I\",\"love\",\"you\"];\n println!(\"You said : {} {} {}\", arr_test[0],arr_test[1],arr_test[2]);\n\n let arr_test = [1;3]; \n println!(\"Call Num : {}&{}&{}\", arr_test[0],arr_test[1],arr_test[2]);\n}\n```\n\n\n\n### String\n- Basic data types are stored on the stack, but the String type is stored on the heap\n```rust\nlet s = String::from(\"hello\");\n```\n- push_str(): append a str slice a string\n- push(): appends a single character to a String\n```rust\nfn main() { \n let mut data = String::from(\"andy\");\n data.push_str(\" is stronger\");\n data.push('!');\n}\n```\n- + operator, chaining strings. the left side of the + operator is the ownership of the string, and the right side is the string slice\n- String is actually a wrapper for Vec, so the length can be measured by the len() method, but note that Len() is not length of character, but byte len\n- String iteration\n```rust\nfn main() { \n let mut data = String::from(\"andy\");\n data.push_str(\" is stronger\");\n data.push('!');\n\n for i in data.bytes() {\n ///\n }\n\n for i in data.chars() {\n ///\n }\n}\n```\n\n### Vector\n- Vector is like any other struct. When Vector leaves the scope, the variable value is cleaned up, and all its elements are also cleaned up.\n```rust\nfn main() {\n let vec: Vec = Vec::new();\n let vec2: Vec = vec![3,4，5] // create vector by macro\n for i in vec2 {\n println!(\"Vector value is : {}\", i);\n }\n}\n```\n\n### HashMap\n- HashMap is not preloaded, so it needs to be included `use std::collections::HashMap`\n```rust\nuse std::collections::HashMap;\nfn main() {\n let keys = vec![\"andy\".to_string(), \"cliff\".to_string()] ;\n let ages = vec![38, 26];\n let map :HashMap<_,_> = keys.iter().zip(ages.iter()).collect();\n println!(\"{:?}\", map); /// print {\"andy\": 38, \"cliff\": 26}\n}\n```\n#### HashMap ownership\n- For types that implement the Copy trait (such as i32), the value will be copied into the HashMap\n- For values with ownership, such as (String), the value will be moved and ownership will be given to HashMap\n- If a reference to a value is inserted into the HashMap, the value itself does not move\n\n#### HashMap iteration\n```rust\nuse std::collections::HashMap;\n\nfn main() { \n let name = \"andy\".to_string();\n let age = 36;\n let mut map = HashMap::new();\n map.insert(name, age);\n map.insert(String::from(\"cliff\"), 26);\n println!(\"{:?}\", &map);\n for (k, v) in map {\n println!(\"{} age {}\", k, v);\n } /// cliff age 26\n /// andy age 36\n}\n```\n#### update\n```rust\nuse std::collections::HashMap;\n\nfn main() { \n let name = \"andy\".to_string();\n let age = 36;\n let mut map = HashMap::new();\n map.insert(name, age);\n map.insert(String::from(\"cliff\"), 26);\n\n let result = map.entry(\"bob\".to_string());\n println!(\"{:?}\", result); /// Entry(VacantEntry(\"bob\"))\n\n let result = map.entry(\"andy\".to_string());\n println!(\"{:?}\", result); /// Entry(OccupiedEntry { key: \"andy\", value: 36, .. })\n\n map.entry(\"bob\".to_string()).or_insert(28);\n map.entry(\"cliff\".to_string()).or_insert(0);\n}\n```\n\n## control flow\n- if\n```rust\nfn main() {\n let condition = 1;\n let x = if condition == 1 { \"A\" } else { \"B\" };\n println!(\"Result x = {}\" , x) ;\n}\n```\n- loop\n```rust\nfn main() {\n let mut condition = 0;\n\n let result = 'outer: loop { // 'outer is label\n 'inner: loop {\n condition += 1;\n if 3 == condition {\n break 'outer 3 * condition; // break outer loop\n }\n }\n };\n println!(\"Loop result is : {}\", result); /// Loop result is : 9\n}\n\n```\n- rot\n```rust\nfn main() {\n let arr = [3,2,3];\n for num in arr.iter() {\n println!(\"For value is {}\", num);\n }\n}\n```\n\n## Range iterator\n- Range\n```rust\nfn main() {\n for number in (1..=3) {\n println!(\"Number A is {}\", number ); /// 1,2,3\n }\n \n for number in (1..=3).rev() { /// rev means reverse,\n println!(\"Number B is {}\", number ); /// 3,2,1\n }\n}\n\n```\n\n## struct\n- If struct is declared mutable then all fields in the instance are mutable\n### tuple struct\n```rust\nstruct Color(i32,i32,i32);\nlet black = Color(0,0,0);\n```\n### Unit-Like struct\n```rust\nstruct Man {};\n```\n### struct method\n```rust\n\nfn main() {\n let rec = Rectangle {\n width: 30,\n height: 50,\n };\n\n let result = rec.area(); \n println!(\"rectangle：{:?}，area is：{}\", rec, result);\n}\n\n\n#[derive(Debug)]\nstruct Rectangle {\n width: u32,\n height: u32,\n}\n\nimpl Rectangle {\n fn area(&self) -> u32{\n self.width * self.height\n }\n}\n\n```\n### associative func（similar to static method）\n- You can define a function that does not take `self` as the first parameter in the impl block. This form is called an associated function, and the calling method is similar to `String::from()`\n```rust\nimpl Rectangle {\n fn create_square(width: u32) -> Rectangle {\n Rectangle {\n width,\n height: width,\n }\n }\n}\n```\n \n## enum\n```rust\nenum Ip {\n V4,\n V6,\n}\n\nenum IpAddr {\n V4(String),\n V6(String),\n}\n``` \n\n## match\n- match must exhaust all possibilities\n- If there are too many matchings, you can also use \"_\" for wildcarding, but note that \"_\" must be placed at the end\n```rust\nenum Color {\n Red,\n Yellow,\n Blue,\n}\nenum ColorWithVal {\n Red(u8,u8,u8),\n Yellow(u8,u8,u8),\n Blue(u8,u8,u8),\n}\nfn main(){\n let colour = Color::Blue;\n match colour {\n Color::Red => {\n println!(\"Red colour.\");\n },\n _ => {\n println!(\"Other colour.\");\n }\n }\n\n let colour = ColorWithVal::Red(222,111,22);\n match colour {\n ColorWithVal::Red(r,g,b) => {\n println!(\"Red colour. {},{},{}\", r,g,b);\n },\n _ => {\n println!(\"Other colour.\");\n }\n }\n}\n```\n\n## if let\n```rust\nfn main(){\n let colour = Color::Red(Some(222),Some(222),Some(222));\n\n if let Color::Red(r,g,b) = colour {\n println!(\"Red colour. {:?},{:?},{:?}\", r,g,b);\n } else {\n println!(\"Other colour.\");\n }\n}\n```\n\n## Result\n- Recoverable err via Result, non-recoverable via panic!\n- upon panic!, the program will expand an error message, unwind, clean up the call stack (Stack) and finally exit the program\n- You can set panic = 'abort' in Cargo.toml to terminate the cleaning of the call stack\n```rust\n[profile.release]\npanic='abort'\n```\n- RUST_BACKTRACE = 1 prints detailed error messages in the stack\n```rust\nuse std::fs::File;\nfn main() { \n let fp = File::open(\"hello.txt\");\n let file = match fp {\n Ok(file)=> {\n file\n },\n Err(error) => panic!(\"file not found {:?} \", error),\n };\n}\n```\n\n```rust\nuse std::{fs::File, io::ErrorKind};\nfn main() { \n let fp = File::open(\"hello.txt\");\n let file = match fp {\n Ok(file)=> {\n file\n },\n Err(error) => {\n match error.kind() {\n ErrorKind::NotFound => {\n match File::create(\"hello.txt\") {\n Ok(file) => {\n file\n },\n Err(err) => {\n panic!(\"file create error:{:?}\", &err);\n },\n }\n },\n oe => panic!(\"other error {:?}\", oe),\n }\n } ,\n };\n}\n```\n```rust\nuse std::{fs::File, io::ErrorKind};\nfn main() { \n let file = File::open(\"hello.txt\").unwrap_or_else(|err| {\n if err.kind() == ErrorKind::NotFound {\n File::create(\"hello.txt\").unwrap_or_else(|err|{\n panic!(\"error：{:?}\", err);\n })\n }else{\n panic!(\"other error：{:?}\", err);\n }\n });\n}\n```\n### unwrap && expect\n- If do not want to deal with Err, can use unwarp() method. If result is Ok(val), return val. If Err, then call the panic! macro.\n- expect can specify what the error message is, which is easier to debug\n\n### The question mark operator, ?\nWhen writing code that calls many functions that return the Result type, the error handling can be tedious. The question mark operator, ?, hides some of the boilerplate of propagating errors up the call stack.\n```rust\nlet mut file = File::create(\"my_best_friends.txt\")?;\n```\n\n## generic\n```rust\n#[derive(Debug)]\nstruct Point {\n x : T,\n y : U,\n}\nimpl Point {\n fn mixup(self, other: Point) -> Point {\n Point{x: self.x , y: other.y, }\n }\n}\n```\n\n## trait\n### definition\n```rust\npub trait Summary {\n fn summarize(&self) -> String {\n \"... more\".to_string() /// default \n }\n}\npub struct Tweet {\n user_name :String,\n replay_count :u32,\n like_count :u32,\n}\nimpl Tweet {\n fn like(&mut self) {\n self.like_count += 1;\n }\n}\n\nimpl Summary for Tweet {\n fn summarize(&self) -> String {\n format!(\"{} like count :{} , replay count :{}\", &self.user_name, &self.replay_count, &self.like_count)\n }\n}\n```\n\n### trait as arguments\n```rust\nfn notify_msg (info: T) {\n println!(\"summary : {}\", info.summarize() );\n}\nfn notify_msg (info: impl Summary + Display) {\n println!(\"summary : {}\", info.summarize() );\n}\nfn notify_msg (info: T) \nwhere \n T: Summary + Display,\n{\n println!(\"summary : {}\", info.summarize() );\n println!(\"display implement info : {}\", info);\n}\n```\n### trait as return\n- impl Trait can only return the same type, if it returns a different type, even if the Trait is implemented, an error will be reported\n\n## references\n- [the rust programming language](https://doc.rust-lang.org/book/)\n- [mooc course](https://time.geekbang.org/course/intro/100060601?tab=catalog)\n- [bilibili tutorial](https://www.bilibili.com/video/BV1hp4y1k7SV?p=50&spm_id_from=pageDriver)\n- [jianshu notes](https://www.jianshu.com/p/30d917790298)","slug":"rust/rust-02-basics","published":1,"updated":"2023-11-05T04:21:13.734Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8dr0010qwsj4oml40f0","content":"

frequently used cmd

1
2
3
4
5
rustc [filename].rs
cargo new [project_name]
cargo build [--release]
cargo run [--release]
cargo check # check whether compile success, no executible output
\n\n

data type

integer

    \n
  • i8,i16,i32,i64,i128,isize,u8,u16,u32,u64,u128,usize，etc
    • \n
  • isize, usize indicates that the type is determined by the architecture of the computer. For example, on a 32 bit target, this is 4 bytes and on a 64 bit target, this is 8 bytes.
    • \n
  • 0x: hex，0o Octal，0b binary，starting with b: byte （u8 only）
    • \n
\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
Number LiteralsExample
Decimal98_222
Hex0xff
Octal0o77
Binary0b1111_0000
Byte(u8 only)b’A’
\n

Tuple

    \n
  • The length of Tuple is fixed, and the length cannot be changed once declared
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    fn main() {
        // tuple could be declared as mut
        let mut tuple_1 = ("Hello", 39, "Years");
        let tuple_2:(i32, &str ) = (1983, "since.");
        tuple_1.0 = "Hi";
        println!("{} {} {}", tuple_1.0, tuple_1.1, tuple_1.2);
        // destructure
        let (a,b) = tuple_2;
        println!("{} {}", a, b);
    }
    • \n
\n

array

    \n
  • arrays in Rust have a fixed length.
    • \n
  • Vector is similar to an array, it is provided by the standard library, and its length can be changed
    • \n
\n
1
2
3
4
5
6
7
8
9
10
11
fn main() {

    let arr_test:[u8; 3] = [1,2,3];
    println!("Number is {},{},{}", arr_test[0],arr_test[1],arr_test[2]);

    let arr_test = ["I","love","you"];
    println!("You said : {} {} {}", arr_test[0],arr_test[1],arr_test[2]);

    let arr_test = [1;3]; 
    println!("Call Num : {}&{}&{}", arr_test[0],arr_test[1],arr_test[2]);
}
\n\n\n\n

String

    \n
  • Basic data types are stored on the stack, but the String type is stored on the heap
    1
    let s = String::from("hello");
    • \n
  • push_str(): append a str slice a string
    • \n
  • push(): appends a single character to a String
    1
    2
    3
    4
    5
    fn main() { 
        let mut data = String::from("andy");
        data.push_str(" is stronger");
        data.push('!');
    }
    • \n
  • + operator, chaining strings. the left side of the + operator is the ownership of the string, and the right side is the string slice
    • \n
  • String is actually a wrapper for Vec, so the length can be measured by the len() method, but note that Len() is not length of character, but byte len
    • \n
  • String iteration
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    fn main() { 
        let mut data = String::from("andy");
        data.push_str(" is stronger");
        data.push('!');

        for i in data.bytes() {
            ///
        }

        for i in data.chars() {
            ///
        }
    }
    • \n
\n

Vector

    \n
  • Vector is like any other struct. When Vector leaves the scope, the variable value is cleaned up, and all its elements are also cleaned up.
    1
    2
    3
    4
    5
    6
    7
    fn main() {
        let vec: Vec<u16> = Vec::new();
        let vec2: Vec<i32> = vec![3,4，5] // create vector by macro
        for i in vec2 {
            println!("Vector value is : {}", i);
        }
    }
    • \n
\n

HashMap

    \n
  • HashMap is not preloaded, so it needs to be included use std::collections::HashMap
    1
    2
    3
    4
    5
    6
    7
    use std::collections::HashMap;
    fn main() {
        let keys = vec!["andy".to_string(), "cliff".to_string()] ;
        let ages = vec![38, 26];
        let map :HashMap<_,_> = keys.iter().zip(ages.iter()).collect();
        println!("{:?}", map); /// print {"andy": 38, "cliff": 26}
    }
    • \n
\n

HashMap ownership

    \n
  • For types that implement the Copy trait (such as i32), the value will be copied into the HashMap
    • \n
  • For values with ownership, such as (String), the value will be moved and ownership will be given to HashMap
    • \n
  • If a reference to a value is inserted into the HashMap, the value itself does not move
    • \n
\n

HashMap iteration

1
2
3
4
5
6
7
8
9
10
11
12
13
14
use std::collections::HashMap;

fn main() { 
    let name = "andy".to_string();
    let age = 36;
    let mut map = HashMap::new();
    map.insert(name, age);
    map.insert(String::from("cliff"), 26);
    println!("{:?}", &map);
    for (k, v) in map {
        println!("{} age {}", k, v);
    } /// cliff age 26
      /// andy age 36
}
\n

update

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
use std::collections::HashMap;

fn main() { 
    let name = "andy".to_string();
    let age = 36;
    let mut map = HashMap::new();
    map.insert(name, age);
    map.insert(String::from("cliff"), 26);

    let result = map.entry("bob".to_string());
    println!("{:?}", result); /// Entry(VacantEntry("bob"))

    let result = map.entry("andy".to_string());
    println!("{:?}", result); /// Entry(OccupiedEntry { key: "andy", value: 36, .. })

    map.entry("bob".to_string()).or_insert(28);
    map.entry("cliff".to_string()).or_insert(0);
}
\n\n

control flow

    \n
  • if
    1
    2
    3
    4
    5
    fn main() {
        let condition = 1;
        let x = if condition == 1 { "A" } else { "B" };
        println!("Result x = {}" , x) ;
    }
    • \n
  • loop
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    fn main() {
        let mut condition = 0;

        let result = 'outer: loop {  // 'outer is label
            'inner: loop {
                condition += 1;
                if 3 == condition {
                    break 'outer 3 * condition; // break outer loop
                }
            }
        };
        println!("Loop result is : {}", result); /// Loop result is : 9
    }

    • \n
  • rot
    1
    2
    3
    4
    5
    6
    fn main() {
        let arr = [3,2,3];
        for num in arr.iter() {
            println!("For value is {}", num);
        }
    }
    • \n
\n

Range iterator

    \n
  • Range
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    fn main() {
         for number in (1..=3) {
            println!("Number A is {}", number ); /// 1,2,3
        }
     
        for number in (1..=3).rev() { /// rev means reverse,
            println!("Number B is {}", number ); /// 3,2,1
        }
    }

    • \n
\n

struct

    \n
  • If struct is declared mutable then all fields in the instance are mutable
    • \n
\n

tuple struct

1
2
struct Color(i32,i32,i32);
let black = Color(0,0,0);
\n

Unit-Like struct

1
struct Man {};
\n

struct method

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

fn main() {
    let rec = Rectangle {
        width: 30,
        height: 50,
    };

    let result = rec.area(); 
    println!("rectangle：{:?}，area is：{}", rec, result);
}


#[derive(Debug)]
struct Rectangle {
    width: u32,
    height: u32,
}

impl Rectangle {
    fn area(&self) -> u32{
        self.width * self.height
    }
}

\n

associative func（similar to static method）

    \n
  • You can define a function that does not take self as the first parameter in the impl block. This form is called an associated function, and the calling method is similar to String::from()
    1
    2
    3
    4
    5
    6
    7
    8
    impl Rectangle {
        fn create_square(width: u32) -> Rectangle {
            Rectangle {
                width,
                height: width,
            }
        }
    }
    • \n
\n

enum

1
2
3
4
5
6
7
8
9
enum Ip {
    V4,
    V6,
}

enum IpAddr {
    V4(String),
    V6(String),
}
\n\n

match

    \n
  • match must exhaust all possibilities
    • \n
  • If there are too many matchings, you can also use ““ for wildcarding, but note that ““ must be placed at the end
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28
    29
    30
    31
    enum Color {
        Red,
        Yellow,
        Blue,
    }
    enum ColorWithVal {
        Red(u8,u8,u8),
        Yellow(u8,u8,u8),
        Blue(u8,u8,u8),
    }
    fn main(){
        let colour = Color::Blue;
        match colour {
            Color::Red => {
                println!("Red colour.");
            },
            _ => {
                println!("Other colour.");
            }
        }

        let colour = ColorWithVal::Red(222,111,22);
        match colour {
            ColorWithVal::Red(r,g,b) => {
                println!("Red colour. {},{},{}", r,g,b);
            },
            _ => {
                println!("Other colour.");
            }
        }
    }
    • \n
\n

if let

1
2
3
4
5
6
7
8
9
fn main(){
    let colour = Color::Red(Some(222),Some(222),Some(222));

    if let Color::Red(r,g,b) = colour {
        println!("Red colour. {:?},{:?},{:?}", r,g,b);
    } else {
        println!("Other colour.");
    }
}
\n\n

Result<T,E>

    \n
  • Recoverable err via Result<T,E>, non-recoverable via panic!
    • \n
  • upon panic!, the program will expand an error message, unwind, clean up the call stack (Stack) and finally exit the program
    • \n
  • You can set panic = ‘abort’ in Cargo.toml to terminate the cleaning of the call stack
    1
    2
    [profile.release]
    panic='abort'
    • \n
  • RUST_BACKTRACE = 1 prints detailed error messages in the stack
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    use std::fs::File;
    fn main() { 
        let fp = File::open("hello.txt");
        let file = match fp {
            Ok(file)=> {
                file
            },
            Err(error) => panic!("file not found {:?} ", error),
        };
    }
    • \n
\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
use std::{fs::File, io::ErrorKind};
fn main() { 
    let fp = File::open("hello.txt");
    let file = match fp {
        Ok(file)=> {
            file
        },
        Err(error) => {
            match error.kind() {
                ErrorKind::NotFound => {
                    match File::create("hello.txt") {
                        Ok(file) => {
                            file
                        },
                        Err(err) => {
                            panic!("file create error:{:?}", &err);
                        },
                    }
                },
                oe => panic!("other error {:?}", oe),
            }
        } ,
    };
}
\n
1
2
3
4
5
6
7
8
9
10
11
12
use std::{fs::File, io::ErrorKind};
fn main() { 
    let file = File::open("hello.txt").unwrap_or_else(|err| {
        if err.kind() == ErrorKind::NotFound {
            File::create("hello.txt").unwrap_or_else(|err|{
                panic!("error：{:?}", err);
            })
        }else{
            panic!("other error：{:?}", err);
        }
    });
}
\n

unwrap && expect

    \n
  • If do not want to deal with Err, can use unwarp() method. If result is Ok(val), return val. If Err, then call the panic! macro.
    • \n
  • expect can specify what the error message is, which is easier to debug
    • \n
\n

The question mark operator, ?

When writing code that calls many functions that return the Result type, the error handling can be tedious. The question mark operator, ?, hides some of the boilerplate of propagating errors up the call stack.

\n
1
let mut file = File::create("my_best_friends.txt")?;
\n\n

generic

1
2
3
4
5
6
7
8
9
10
#[derive(Debug)]
struct Point<T, U>  {
   x : T,
   y : U,
}
impl <T, U> Point<T, U> {
    fn mixup<V, W>(self, other: Point<V, W>) -> Point<T, W> {
        Point{x: self.x , y: other.y, }
    }
}
\n\n

trait

definition

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
pub trait Summary {
    fn summarize(&self) -> String {
         "... more".to_string() /// default 
    }
}
pub struct Tweet {
    user_name :String,
    replay_count :u32,
    like_count :u32,
}
impl Tweet {
    fn like(&mut self) {
        self.like_count += 1;
    }
}

impl Summary for Tweet {
    fn summarize(&self) -> String {
        format!("{} like count :{} , replay count :{}", &self.user_name, &self.replay_count, &self.like_count)
    }
}
\n\n

trait as arguments

1
2
3
4
5
6
7
8
9
10
11
12
13
fn notify_msg <T:Summary> (info: T) {
    println!("summary : {}", info.summarize() );
}
fn notify_msg (info: impl Summary + Display) {
    println!("summary : {}", info.summarize() );
}
fn notify_msg <T> (info: T) 
where 
    T: Summary + Display,
{
    println!("summary : {}", info.summarize() );
    println!("display implement info : {}", info);
}
\n

trait as return

    \n
  • impl Trait can only return the same type, if it returns a different type, even if the Trait is implemented, an error will be reported
    • \n
\n

references

\n","site":{"data":{}},"excerpt":"","more":"

frequently used cmd

1
2
3
4
5
rustc [filename].rs
cargo new [project_name]
cargo build [--release]
cargo run [--release]
cargo check # check whether compile success, no executible output
\n\n

data type

integer

    \n
  • i8,i16,i32,i64,i128,isize,u8,u16,u32,u64,u128,usize，etc
    • \n
  • isize, usize indicates that the type is determined by the architecture of the computer. For example, on a 32 bit target, this is 4 bytes and on a 64 bit target, this is 8 bytes.
    • \n
  • 0x: hex，0o Octal，0b binary，starting with b: byte （u8 only）
    • \n
\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
Number LiteralsExample
Decimal98_222
Hex0xff
Octal0o77
Binary0b1111_0000
Byte(u8 only)b’A’
\n

Tuple

    \n
  • The length of Tuple is fixed, and the length cannot be changed once declared
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    fn main() {
        // tuple could be declared as mut
        let mut tuple_1 = ("Hello", 39, "Years");
        let tuple_2:(i32, &str ) = (1983, "since.");
        tuple_1.0 = "Hi";
        println!("{} {} {}", tuple_1.0, tuple_1.1, tuple_1.2);
        // destructure
        let (a,b) = tuple_2;
        println!("{} {}", a, b);
    }
    • \n
\n

array

    \n
  • arrays in Rust have a fixed length.
    • \n
  • Vector is similar to an array, it is provided by the standard library, and its length can be changed
    • \n
\n
1
2
3
4
5
6
7
8
9
10
11
fn main() {

    let arr_test:[u8; 3] = [1,2,3];
    println!("Number is {},{},{}", arr_test[0],arr_test[1],arr_test[2]);

    let arr_test = ["I","love","you"];
    println!("You said : {} {} {}", arr_test[0],arr_test[1],arr_test[2]);

    let arr_test = [1;3]; 
    println!("Call Num : {}&{}&{}", arr_test[0],arr_test[1],arr_test[2]);
}
\n\n\n\n

String

    \n
  • Basic data types are stored on the stack, but the String type is stored on the heap
    1
    let s = String::from("hello");
    • \n
  • push_str(): append a str slice a string
    • \n
  • push(): appends a single character to a String
    1
    2
    3
    4
    5
    fn main() { 
        let mut data = String::from("andy");
        data.push_str(" is stronger");
        data.push('!');
    }
    • \n
  • + operator, chaining strings. the left side of the + operator is the ownership of the string, and the right side is the string slice
    • \n
  • String is actually a wrapper for Vec, so the length can be measured by the len() method, but note that Len() is not length of character, but byte len
    • \n
  • String iteration
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    fn main() { 
        let mut data = String::from("andy");
        data.push_str(" is stronger");
        data.push('!');

        for i in data.bytes() {
            ///
        }

        for i in data.chars() {
            ///
        }
    }
    • \n
\n

Vector

    \n
  • Vector is like any other struct. When Vector leaves the scope, the variable value is cleaned up, and all its elements are also cleaned up.
    1
    2
    3
    4
    5
    6
    7
    fn main() {
        let vec: Vec<u16> = Vec::new();
        let vec2: Vec<i32> = vec![3,4，5] // create vector by macro
        for i in vec2 {
            println!("Vector value is : {}", i);
        }
    }
    • \n
\n

HashMap

    \n
  • HashMap is not preloaded, so it needs to be included use std::collections::HashMap
    1
    2
    3
    4
    5
    6
    7
    use std::collections::HashMap;
    fn main() {
        let keys = vec!["andy".to_string(), "cliff".to_string()] ;
        let ages = vec![38, 26];
        let map :HashMap<_,_> = keys.iter().zip(ages.iter()).collect();
        println!("{:?}", map); /// print {"andy": 38, "cliff": 26}
    }
    • \n
\n

HashMap ownership

    \n
  • For types that implement the Copy trait (such as i32), the value will be copied into the HashMap
    • \n
  • For values with ownership, such as (String), the value will be moved and ownership will be given to HashMap
    • \n
  • If a reference to a value is inserted into the HashMap, the value itself does not move
    • \n
\n

HashMap iteration

1
2
3
4
5
6
7
8
9
10
11
12
13
14
use std::collections::HashMap;

fn main() { 
    let name = "andy".to_string();
    let age = 36;
    let mut map = HashMap::new();
    map.insert(name, age);
    map.insert(String::from("cliff"), 26);
    println!("{:?}", &map);
    for (k, v) in map {
        println!("{} age {}", k, v);
    } /// cliff age 26
      /// andy age 36
}
\n

update

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
use std::collections::HashMap;

fn main() { 
    let name = "andy".to_string();
    let age = 36;
    let mut map = HashMap::new();
    map.insert(name, age);
    map.insert(String::from("cliff"), 26);

    let result = map.entry("bob".to_string());
    println!("{:?}", result); /// Entry(VacantEntry("bob"))

    let result = map.entry("andy".to_string());
    println!("{:?}", result); /// Entry(OccupiedEntry { key: "andy", value: 36, .. })

    map.entry("bob".to_string()).or_insert(28);
    map.entry("cliff".to_string()).or_insert(0);
}
\n\n

control flow

    \n
  • if
    1
    2
    3
    4
    5
    fn main() {
        let condition = 1;
        let x = if condition == 1 { "A" } else { "B" };
        println!("Result x = {}" , x) ;
    }
    • \n
  • loop
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    fn main() {
        let mut condition = 0;

        let result = 'outer: loop {  // 'outer is label
            'inner: loop {
                condition += 1;
                if 3 == condition {
                    break 'outer 3 * condition; // break outer loop
                }
            }
        };
        println!("Loop result is : {}", result); /// Loop result is : 9
    }

    • \n
  • rot
    1
    2
    3
    4
    5
    6
    fn main() {
        let arr = [3,2,3];
        for num in arr.iter() {
            println!("For value is {}", num);
        }
    }
    • \n
\n

Range iterator

    \n
  • Range
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    fn main() {
         for number in (1..=3) {
            println!("Number A is {}", number ); /// 1,2,3
        }
     
        for number in (1..=3).rev() { /// rev means reverse,
            println!("Number B is {}", number ); /// 3,2,1
        }
    }

    • \n
\n

struct

    \n
  • If struct is declared mutable then all fields in the instance are mutable
    • \n
\n

tuple struct

1
2
struct Color(i32,i32,i32);
let black = Color(0,0,0);
\n

Unit-Like struct

1
struct Man {};
\n

struct method

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

fn main() {
    let rec = Rectangle {
        width: 30,
        height: 50,
    };

    let result = rec.area(); 
    println!("rectangle：{:?}，area is：{}", rec, result);
}


#[derive(Debug)]
struct Rectangle {
    width: u32,
    height: u32,
}

impl Rectangle {
    fn area(&self) -> u32{
        self.width * self.height
    }
}

\n

associative func（similar to static method）

    \n
  • You can define a function that does not take self as the first parameter in the impl block. This form is called an associated function, and the calling method is similar to String::from()
    1
    2
    3
    4
    5
    6
    7
    8
    impl Rectangle {
        fn create_square(width: u32) -> Rectangle {
            Rectangle {
                width,
                height: width,
            }
        }
    }
    • \n
\n

enum

1
2
3
4
5
6
7
8
9
enum Ip {
    V4,
    V6,
}

enum IpAddr {
    V4(String),
    V6(String),
}
\n\n

match

    \n
  • match must exhaust all possibilities
    • \n
  • If there are too many matchings, you can also use ““ for wildcarding, but note that ““ must be placed at the end
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28
    29
    30
    31
    enum Color {
        Red,
        Yellow,
        Blue,
    }
    enum ColorWithVal {
        Red(u8,u8,u8),
        Yellow(u8,u8,u8),
        Blue(u8,u8,u8),
    }
    fn main(){
        let colour = Color::Blue;
        match colour {
            Color::Red => {
                println!("Red colour.");
            },
            _ => {
                println!("Other colour.");
            }
        }

        let colour = ColorWithVal::Red(222,111,22);
        match colour {
            ColorWithVal::Red(r,g,b) => {
                println!("Red colour. {},{},{}", r,g,b);
            },
            _ => {
                println!("Other colour.");
            }
        }
    }
    • \n
\n

if let

1
2
3
4
5
6
7
8
9
fn main(){
    let colour = Color::Red(Some(222),Some(222),Some(222));

    if let Color::Red(r,g,b) = colour {
        println!("Red colour. {:?},{:?},{:?}", r,g,b);
    } else {
        println!("Other colour.");
    }
}
\n\n

Result<T,E>

    \n
  • Recoverable err via Result<T,E>, non-recoverable via panic!
    • \n
  • upon panic!, the program will expand an error message, unwind, clean up the call stack (Stack) and finally exit the program
    • \n
  • You can set panic = ‘abort’ in Cargo.toml to terminate the cleaning of the call stack
    1
    2
    [profile.release]
    panic='abort'
    • \n
  • RUST_BACKTRACE = 1 prints detailed error messages in the stack
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    use std::fs::File;
    fn main() { 
        let fp = File::open("hello.txt");
        let file = match fp {
            Ok(file)=> {
                file
            },
            Err(error) => panic!("file not found {:?} ", error),
        };
    }
    • \n
\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
use std::{fs::File, io::ErrorKind};
fn main() { 
    let fp = File::open("hello.txt");
    let file = match fp {
        Ok(file)=> {
            file
        },
        Err(error) => {
            match error.kind() {
                ErrorKind::NotFound => {
                    match File::create("hello.txt") {
                        Ok(file) => {
                            file
                        },
                        Err(err) => {
                            panic!("file create error:{:?}", &err);
                        },
                    }
                },
                oe => panic!("other error {:?}", oe),
            }
        } ,
    };
}
\n
1
2
3
4
5
6
7
8
9
10
11
12
use std::{fs::File, io::ErrorKind};
fn main() { 
    let file = File::open("hello.txt").unwrap_or_else(|err| {
        if err.kind() == ErrorKind::NotFound {
            File::create("hello.txt").unwrap_or_else(|err|{
                panic!("error：{:?}", err);
            })
        }else{
            panic!("other error：{:?}", err);
        }
    });
}
\n

unwrap && expect

    \n
  • If do not want to deal with Err, can use unwarp() method. If result is Ok(val), return val. If Err, then call the panic! macro.
    • \n
  • expect can specify what the error message is, which is easier to debug
    • \n
\n

The question mark operator, ?

When writing code that calls many functions that return the Result type, the error handling can be tedious. The question mark operator, ?, hides some of the boilerplate of propagating errors up the call stack.

\n
1
let mut file = File::create("my_best_friends.txt")?;
\n\n

generic

1
2
3
4
5
6
7
8
9
10
#[derive(Debug)]
struct Point<T, U>  {
   x : T,
   y : U,
}
impl <T, U> Point<T, U> {
    fn mixup<V, W>(self, other: Point<V, W>) -> Point<T, W> {
        Point{x: self.x , y: other.y, }
    }
}
\n\n

trait

definition

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
pub trait Summary {
    fn summarize(&self) -> String {
         "... more".to_string() /// default 
    }
}
pub struct Tweet {
    user_name :String,
    replay_count :u32,
    like_count :u32,
}
impl Tweet {
    fn like(&mut self) {
        self.like_count += 1;
    }
}

impl Summary for Tweet {
    fn summarize(&self) -> String {
        format!("{} like count :{} , replay count :{}", &self.user_name, &self.replay_count, &self.like_count)
    }
}
\n\n

trait as arguments

1
2
3
4
5
6
7
8
9
10
11
12
13
fn notify_msg <T:Summary> (info: T) {
    println!("summary : {}", info.summarize() );
}
fn notify_msg (info: impl Summary + Display) {
    println!("summary : {}", info.summarize() );
}
fn notify_msg <T> (info: T) 
where 
    T: Summary + Display,
{
    println!("summary : {}", info.summarize() );
    println!("display implement info : {}", info);
}
\n

trait as return

    \n
  • impl Trait can only return the same type, if it returns a different type, even if the Trait is implemented, an error will be reported
    • \n
\n

references

\n"},{"title":"rust lifetime","date":"2022-10-18T13:33:26.000Z","_content":"\n## lifetime\n- Every reference in Rust has its own lifecycle\n- most of the time, Rust's lifetime is implicit and can be inferred\n- There are two types of life cycle: input life cycle and output life cycle\n- 'static is a special life cycle annotation\n### Example of lifetime out of scope\n```rust\nfn main() {\n let mut x;\n {\n let y = String::from(\"hello\");\n // x = y; // this is allowed\n x = &y; // not allowed. borrowed value (y) does not live long enough\n }\n println!(\"Str:{}\", x);\n}\n```\n\n### lifetime checker\nRust compiler's borrow checker to determine whether a borrow is legal\n```rust\n// If it returns a reference value, no matter how simple your function is written, it will always report an error `missing lifetime specifier.`\nfn longest(x:&str, y:&str) -> &str { /// this function's return type contains a borrowed value, but the signature does not say whether it is borrowed from `x` or `y`\n if x.len() > y.len() {\n x\n }else{\n y\n }\n}\n```\nlet's find out why it is such case\n```rust\nfn main() {\n // variable to hold the result value\n let long_str; \n\n let x = \"abc\".to_string();\n {\n let y = \"bbccd\".to_string();\n long_str = longest(x.as_str(), y.as_str());\n }\n // if x.len() > y.len() then it is OK，the long_str variable will hole x; if not, long_str supposed tohold y, however, y has a smaller scope than x, long_str will hold to a dropped value\n println!(\"Longest str: {}\", long_str);\n}\n\n```\nHence, we need lifetime annotation `'`\n```rust\nfn longest<'a>(x:&'a str, y:&'a str) -> &'a str {\n if x.len() > y.len() {\n x\n }else{\n y\n }\n}\n```\n\n### deeper understanding\n- When returning a reference value from a function, the lifetime of the return type needs to match the lifetime of one of the parameters\n\n### Struct lifetime annotation\n```rust\nfn main() {\n let info = String::from(\"File not found.\");\n // 存放结果值的变量\n let exc = ImportantExcepiton {\n part: info.as_str()\n };\n\n println!(\"{:?}\", exc);\n}\n#[derive(Debug)]\nstruct ImportantExcepiton<'a> {\n part: &'a str,\n}\n```\n- lifetime of the field `part` must be longer than struct\n\n### Lifetime Elision\nIn order to make common patterns more ergonomic, Rust allows lifetimes to be elided in function signatures.\nElision rules are as follows:\n- Each elided lifetime in input position becomes a distinct lifetime parameter.\n- If there is exactly one input lifetime position (elided or not), that lifetime is assigned to all elided output lifetimes.\n- If there are multiple input lifetime positions, but one of them is &self or &mut self, the lifetime of self is assigned to all elided output lifetimes.\n- Otherwise, it is an error to elide an output lifetime.\n\n### struct lifetime annotation\n```rust\n\nfn main() {\n let info = String::from(\"File not found.\");\n let exc = ImportantExcepiton {\n part: info.as_str()\n };\n\n println!(\"{:?}\", exc);\n}\n#[derive(Debug)]\nstruct ImportantExcepiton <'a>{\n part: &'a str,\n}\n\n// the first 'a is decraration, the second is usage\nimpl<'a> ImportantExcepiton <'a> {\n // in return value, 'a is omitted according to Lifetime Elision rule\n fn callname(&self ) -> &str{\n self.part\n }\n}\n```\n\n### 'static\n'static is a special lifetime that takes up the duration of the entire program, for example all string literals have a 'static lifetime","source":"_posts/rust/rust-04-lifetime.md","raw":"---\ntitle: rust lifetime\ndate: 2022-10-18 21:33:26\ntags: [rust]\n---\n\n## lifetime\n- Every reference in Rust has its own lifecycle\n- most of the time, Rust's lifetime is implicit and can be inferred\n- There are two types of life cycle: input life cycle and output life cycle\n- 'static is a special life cycle annotation\n### Example of lifetime out of scope\n```rust\nfn main() {\n let mut x;\n {\n let y = String::from(\"hello\");\n // x = y; // this is allowed\n x = &y; // not allowed. borrowed value (y) does not live long enough\n }\n println!(\"Str:{}\", x);\n}\n```\n\n### lifetime checker\nRust compiler's borrow checker to determine whether a borrow is legal\n```rust\n// If it returns a reference value, no matter how simple your function is written, it will always report an error `missing lifetime specifier.`\nfn longest(x:&str, y:&str) -> &str { /// this function's return type contains a borrowed value, but the signature does not say whether it is borrowed from `x` or `y`\n if x.len() > y.len() {\n x\n }else{\n y\n }\n}\n```\nlet's find out why it is such case\n```rust\nfn main() {\n // variable to hold the result value\n let long_str; \n\n let x = \"abc\".to_string();\n {\n let y = \"bbccd\".to_string();\n long_str = longest(x.as_str(), y.as_str());\n }\n // if x.len() > y.len() then it is OK，the long_str variable will hole x; if not, long_str supposed tohold y, however, y has a smaller scope than x, long_str will hold to a dropped value\n println!(\"Longest str: {}\", long_str);\n}\n\n```\nHence, we need lifetime annotation `'`\n```rust\nfn longest<'a>(x:&'a str, y:&'a str) -> &'a str {\n if x.len() > y.len() {\n x\n }else{\n y\n }\n}\n```\n\n### deeper understanding\n- When returning a reference value from a function, the lifetime of the return type needs to match the lifetime of one of the parameters\n\n### Struct lifetime annotation\n```rust\nfn main() {\n let info = String::from(\"File not found.\");\n // 存放结果值的变量\n let exc = ImportantExcepiton {\n part: info.as_str()\n };\n\n println!(\"{:?}\", exc);\n}\n#[derive(Debug)]\nstruct ImportantExcepiton<'a> {\n part: &'a str,\n}\n```\n- lifetime of the field `part` must be longer than struct\n\n### Lifetime Elision\nIn order to make common patterns more ergonomic, Rust allows lifetimes to be elided in function signatures.\nElision rules are as follows:\n- Each elided lifetime in input position becomes a distinct lifetime parameter.\n- If there is exactly one input lifetime position (elided or not), that lifetime is assigned to all elided output lifetimes.\n- If there are multiple input lifetime positions, but one of them is &self or &mut self, the lifetime of self is assigned to all elided output lifetimes.\n- Otherwise, it is an error to elide an output lifetime.\n\n### struct lifetime annotation\n```rust\n\nfn main() {\n let info = String::from(\"File not found.\");\n let exc = ImportantExcepiton {\n part: info.as_str()\n };\n\n println!(\"{:?}\", exc);\n}\n#[derive(Debug)]\nstruct ImportantExcepiton <'a>{\n part: &'a str,\n}\n\n// the first 'a is decraration, the second is usage\nimpl<'a> ImportantExcepiton <'a> {\n // in return value, 'a is omitted according to Lifetime Elision rule\n fn callname(&self ) -> &str{\n self.part\n }\n}\n```\n\n### 'static\n'static is a special lifetime that takes up the duration of the entire program, for example all string literals have a 'static lifetime","slug":"rust/rust-04-lifetime","published":1,"updated":"2023-11-05T04:21:13.734Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8ds0012qwsj7v2de3x4","content":"

lifetime

    \n
  • Every reference in Rust has its own lifecycle
    • \n
  • most of the time, Rust’s lifetime is implicit and can be inferred
    • \n
  • There are two types of life cycle: input life cycle and output life cycle
    • \n
  • ‘static is a special life cycle annotation
    • \n
\n

Example of lifetime out of scope

1
2
3
4
5
6
7
8
9
fn main() {
    let mut x;
    {
        let y = String::from("hello");
        // x = y; // this is allowed
        x = &y; // not allowed. borrowed value (y) does not live long enough
    }
    println!("Str:{}", x);
}
\n\n

lifetime checker

Rust compiler’s borrow checker to determine whether a borrow is legal

\n
1
2
3
4
5
6
7
8
// If it returns a reference value, no matter how simple your function is written, it will always report an error `missing lifetime specifier.`
fn longest(x:&str, y:&str) -> &str { /// this function's return type contains a borrowed value, but the signature does not say whether it is borrowed from `x` or `y`
    if x.len() > y.len() {
        x
    }else{
        y
    }
}
\n

let’s find out why it is such case

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
fn main() {
    // variable to hold the result value
    let long_str; 

    let x = "abc".to_string();
    {
        let y = "bbccd".to_string();
        long_str = longest(x.as_str(), y.as_str());
    }
    // if x.len() > y.len() then it is OK，the long_str variable will hole x; if not, long_str supposed tohold y, however, y has a smaller scope than x, long_str will hold to a dropped value
    println!("Longest str: {}", long_str);
}

\n

Hence, we need lifetime annotation '

\n
1
2
3
4
5
6
7
fn longest<'a>(x:&'a str, y:&'a str) -> &'a str {
    if x.len() > y.len() {
        x
    }else{
        y
    }
}
\n\n

deeper understanding

    \n
  • When returning a reference value from a function, the lifetime of the return type needs to match the lifetime of one of the parameters
    • \n
\n

Struct lifetime annotation

1
2
3
4
5
6
7
8
9
10
11
12
13
fn main() {
    let info = String::from("File not found.");
    // 存放结果值的变量
    let exc = ImportantExcepiton {
        part: info.as_str()
    };

    println!("{:?}", exc);
}
#[derive(Debug)]
struct ImportantExcepiton<'a> {
    part: &'a str,
}
\n
    \n
  • lifetime of the field part must be longer than struct
    • \n
\n

Lifetime Elision

In order to make common patterns more ergonomic, Rust allows lifetimes to be elided in function signatures.
Elision rules are as follows:

\n
    \n
  • Each elided lifetime in input position becomes a distinct lifetime parameter.
    • \n
  • If there is exactly one input lifetime position (elided or not), that lifetime is assigned to all elided output lifetimes.
    • \n
  • If there are multiple input lifetime positions, but one of them is &self or &mut self, the lifetime of self is assigned to all elided output lifetimes.
    • \n
  • Otherwise, it is an error to elide an output lifetime.
    • \n
\n

struct lifetime annotation

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21

fn main() {
    let info = String::from("File not found.");
    let exc = ImportantExcepiton {
        part: info.as_str()
    };

    println!("{:?}", exc);
}
#[derive(Debug)]
struct ImportantExcepiton <'a>{
    part: &'a str,
}

// the first 'a is decraration, the second is usage
impl<'a> ImportantExcepiton <'a> {
    // in return value, 'a is omitted according to Lifetime Elision rule
    fn callname(&self ) -> &str{
        self.part
    }
}
\n\n

‘static

‘static is a special lifetime that takes up the duration of the entire program, for example all string literals have a ‘static lifetime

\n","site":{"data":{}},"excerpt":"","more":"

lifetime

    \n
  • Every reference in Rust has its own lifecycle
    • \n
  • most of the time, Rust’s lifetime is implicit and can be inferred
    • \n
  • There are two types of life cycle: input life cycle and output life cycle
    • \n
  • ‘static is a special life cycle annotation
    • \n
\n

Example of lifetime out of scope

1
2
3
4
5
6
7
8
9
fn main() {
    let mut x;
    {
        let y = String::from("hello");
        // x = y; // this is allowed
        x = &y; // not allowed. borrowed value (y) does not live long enough
    }
    println!("Str:{}", x);
}
\n\n

lifetime checker

Rust compiler’s borrow checker to determine whether a borrow is legal

\n
1
2
3
4
5
6
7
8
// If it returns a reference value, no matter how simple your function is written, it will always report an error `missing lifetime specifier.`
fn longest(x:&str, y:&str) -> &str { /// this function's return type contains a borrowed value, but the signature does not say whether it is borrowed from `x` or `y`
    if x.len() > y.len() {
        x
    }else{
        y
    }
}
\n

let’s find out why it is such case

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
fn main() {
    // variable to hold the result value
    let long_str; 

    let x = "abc".to_string();
    {
        let y = "bbccd".to_string();
        long_str = longest(x.as_str(), y.as_str());
    }
    // if x.len() > y.len() then it is OK，the long_str variable will hole x; if not, long_str supposed tohold y, however, y has a smaller scope than x, long_str will hold to a dropped value
    println!("Longest str: {}", long_str);
}

\n

Hence, we need lifetime annotation '

\n
1
2
3
4
5
6
7
fn longest<'a>(x:&'a str, y:&'a str) -> &'a str {
    if x.len() > y.len() {
        x
    }else{
        y
    }
}
\n\n

deeper understanding

    \n
  • When returning a reference value from a function, the lifetime of the return type needs to match the lifetime of one of the parameters
    • \n
\n

Struct lifetime annotation

1
2
3
4
5
6
7
8
9
10
11
12
13
fn main() {
    let info = String::from("File not found.");
    // 存放结果值的变量
    let exc = ImportantExcepiton {
        part: info.as_str()
    };

    println!("{:?}", exc);
}
#[derive(Debug)]
struct ImportantExcepiton<'a> {
    part: &'a str,
}
\n
    \n
  • lifetime of the field part must be longer than struct
    • \n
\n

Lifetime Elision

In order to make common patterns more ergonomic, Rust allows lifetimes to be elided in function signatures.
Elision rules are as follows:

\n
    \n
  • Each elided lifetime in input position becomes a distinct lifetime parameter.
    • \n
  • If there is exactly one input lifetime position (elided or not), that lifetime is assigned to all elided output lifetimes.
    • \n
  • If there are multiple input lifetime positions, but one of them is &self or &mut self, the lifetime of self is assigned to all elided output lifetimes.
    • \n
  • Otherwise, it is an error to elide an output lifetime.
    • \n
\n

struct lifetime annotation

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21

fn main() {
    let info = String::from("File not found.");
    let exc = ImportantExcepiton {
        part: info.as_str()
    };

    println!("{:?}", exc);
}
#[derive(Debug)]
struct ImportantExcepiton <'a>{
    part: &'a str,
}

// the first 'a is decraration, the second is usage
impl<'a> ImportantExcepiton <'a> {
    // in return value, 'a is omitted according to Lifetime Elision rule
    fn callname(&self ) -> &str{
        self.part
    }
}
\n\n

‘static

‘static is a special lifetime that takes up the duration of the entire program, for example all string literals have a ‘static lifetime

\n"},{"title":"rust smart pointer","date":"2022-10-30T03:00:38.000Z","_content":"\n## Overview\nThe most common kind of pointer in Rust is a reference (borrow but not own)\nSmart pointers, on the other hand, are data structures that not only act like a pointer but also have additional metadata and capabilities.\nreferences are pointers that only borrow data; in contrast, in many cases, smart pointers own the data they point to.\n\n### smart pointers example\n- String\n- Vec\nBoth these types count as smart pointers because they own some memory and allow you to manipulate it. They also have metadata (such as their capacity) and extra capabilities or guarantees (such as with String ensuring its data will always be valid UTF-8).\n\n### Deref & Drop\nSmart pointers are usually implemented using structs. The characteristic that distinguishes a smart pointer from an ordinary struct is that smart pointers implement the `Deref` and `Drop` traits.\n- The Deref trait allows an instance of the smart pointer struct to behave like a reference so you can write code that works with either references or smart pointers. \n- The Drop trait allows you to customize the code that is run when an instance of the smart pointer goes out of scope.\n\n### the most common smart pointers in the standard library:\n- `Box` for allocating values on the heap\n- `Rc`, a reference counting type that enables multiple ownership\n- `Ref` and `RefMut`, accessed through `RefCell`, a type that enforces the borrowing rules at runtime instead of compile time\n\n## Box\n### when to use\n- When you have a type whose size can’t be known at compile time and you want to use a value of that type in a context that requires an exact size. (such as cons)\n- When you have a large amount of data and you want to transfer ownership but ensure the data won’t be copied when you do so\n- When you want to own a value and you care only that it’s a type that implements a particular trait rather than being of a specific type\n\n\n### enabling recursive types with Box\nAt compile time, Rust needs to know how much space a type takes up\nOne type whose size can’t be known at compile time is a recursive type (cons list), use Box, which only contains a memory address\n\n```rust\nenum List {\n Cons(i32, List),\n Nil,\n}\n\nuse crate::List::{Cons, Nil};\nfn main() {\n let list = Cons(1, Cons(2, Cons(3, Nil))); // not allowed, infinite size\n}\n```\n\nuse Box\n```rust\nenum List {\n Cons(i32, Box),\n Nil,\n}\n\nuse crate::List::{Cons, Nil};\n\nfn main() {\n let list = Cons(1, Box::new(Cons(2, Box::new(Cons(3, Box::new(Nil))))));\n}\n```\n\n## Treating Smart Pointers Like Regular References with the Deref Trait\nImplementing the Deref trait allows you to customize the behavior of the dereference operator, `*`\nBy implementing Deref in such a way that a smart pointer can be treated like a regular reference, you can write code that operates on references and use that code with smart pointers too.\n\n### Defining Our Own Smart Pointer\n```rust\nuse std::ops::Deref;\n\nstruct MyBox(T); // The MyBox type is a tuple struct with one element of type T\n\nimpl MyBox {\n fn new(x: T) -> MyBox {\n MyBox(x)\n }\n}\n\nimpl Deref for MyBox {\n type Target = T;\n\n fn deref(&self) -> &Self::Target {\n &self.0\n }\n}\n\nfn main() {\n let x = 5;\n let y = MyBox::new(x);\n\n assert_eq!(5, x);\n assert_eq!(5, *y);\n}\n```\n- We fill in the body of the deref method with &self.0 so deref returns a reference to the value we want to access with the * operator. \n- behind the scenes Rust actually ran this code: *(y.deref()). Rust substitutes the * operator with a call to the deref method\n- The reason the deref method returns a reference to a value, and that the plain dereference outside the parentheses in *(y.deref()) is still necessary, is the ownership system. If the deref method returned the value directly instead of a reference to the value, the value would be moved out of self. We don’t want to take ownership of the inner value inside MyBox in this case or in most cases where we use the dereference operator.\n\n### Implicit Deref Coercions with Functions and Methods\nDeref coercion is a convenience that Rust performs on arguments to functions and methods. \nDeref coercion works only on types that implement the Deref trait. Deref coercion converts a reference to such a type into a reference to another type. For example, deref coercion can convert &String to &str because String implements the Deref trait such that it returns &str\nA sequence of calls to the deref method converts the type we provided into the type the parameter needs.\n\n```rust\nfn hello(name: &str) {\n println!(\"Hello, {}!\", name);\n}\n\nfn main() {\n let m = MyBox::new(String::from(\"Rust\"));\n hello(\"rust\"); // ok\n hello(&m); // also ok\n hello(&(*m)[..]); // without deref coercion. The (*m) dereferences the MyBox into a String. Then the & and [..] take a string slice of the String that is equal to the whole string to match the signature of hello\n}\n```\nHere we’re calling the hello function with the argument &m, which is a reference to a MyBox value. Because we implemented the Deref trait on MyBox, Rust can turn &MyBox into &String by calling deref. The standard library provides an implementation of Deref on String that returns a string slice. Rust calls deref again to turn the &String into &str, which matches the hello function’s definition.\n\n### How Deref Coercion Interacts with Mutability\nSimilar to how you use the Deref trait to override the * operator on immutable references, you can use the DerefMut trait to override the * operator on mutable references.\n\nRust does deref coercion when it finds types and trait implementations in three cases:\n- From &T to &U when T: Deref\n- From &mut T to &mut U when T: DerefMut\n- From &mut T to &U when T: Deref\n\n## Running Code on Cleanup with the Drop Trait\nDrop, which lets you customize what happens when a value is about to go out of scope. \n\n```rust\nstruct CustomSmartPointer {\n data: String,\n}\n\nimpl Drop for CustomSmartPointer {\n fn drop(&mut self) {\n println!(\"Dropping CustomSmartPointer with data `{}`!\", self.data);\n }\n}\n\nfn main() {\n let c = CustomSmartPointer {\n data: String::from(\"my stuff\"),\n };\n c.drop(); // not allowed\n let d = CustomSmartPointer {\n data: String::from(\"other stuff\"),\n };\n println!(\"CustomSmartPointers created.\");\n}\n\n```\n### Dropping a Value Early with std::mem::drop\nstd::mem::drop is in prelude, can use directly\n```rust\nstruct CustomSmartPointer {\n data: String,\n}\n\nimpl Drop for CustomSmartPointer {\n fn drop(&mut self) {\n println!(\"Dropping CustomSmartPointer with data `{}`!\", self.data);\n }\n}\n\nfn main() {\n let c = CustomSmartPointer {\n data: String::from(\"some data\"),\n };\n println!(\"CustomSmartPointer created.\");\n drop(c); // ok\n println!(\"CustomSmartPointer dropped before the end of main.\");\n} // c goes out of scope, will occur double drop\n\n```\n\n\n## Rc: the Reference Counted Smart Pointer\nIn the majority of cases, ownership is clear: you know exactly which variable owns a given value. However, there are cases when a single value might have multiple owners. \ntype keeps track of the number of references to a value to determine whether or not the value is still in use. If there are zero references to a value, the value can be cleaned up without any references becoming invalid.\n\nWe use the `Rc` type when we want to allocate some data on the heap for multiple parts of our program to read and we can’t determine at compile time which part will finish using the data last.\n\nRc is only for use in single-threaded scenarios\n\n### using Rc to share data\n\n![rc](/images/rust/pointers/rc.png)\nimplement use box will not work, as below\n```rust\nenum List {\n Cons(i32, Box),\n Nil,\n}\n\nuse crate::List::{Cons, Nil};\n\nfn main() {\n let a = Cons(5, Box::new(Cons(10, Box::new(Nil))));\n let b = Cons(3, Box::new(a)); // value moved here\n let c = Cons(4, Box::new(a)); // value used here after move\n}\n```\nuse Rc\n```rust\nenum List {\n Cons(i32, Rc),\n Nil,\n}\n\nuse crate::List::{Cons, Nil};\nuse std::rc::Rc;\n\nfn main() {\n let a = Rc::new(Cons(5, Rc::new(Cons(10, Rc::new(Nil)))));\n let b = Cons(3, Rc::clone(&a)); // shalow copy only reference, not data\n let c = Cons(4, Rc::clone(&a));\n}\n```\n\n\nWe could have called a.clone() rather than Rc::clone(&a), but Rust’s convention is to use Rc::clone in this case. The implementation of Rc::clone doesn’t make a deep copy of all the data like most types’ implementations of clone do. The call to Rc::clone only increments the reference count, which doesn’t take much time.\n\n### Cloning an Rc Increases the Reference Count\n```rust\nenum List {\n Cons(i32, Rc),\n Nil,\n}\n\nuse crate::List::{Cons, Nil};\nuse std::rc::Rc;\n\nfn main() {\n let a = Rc::new(Cons(5, Rc::new(Cons(10, Rc::new(Nil)))));\n println!(\"count after creating a = {}\", Rc::strong_count(&a)); // 1\n let b = Cons(3, Rc::clone(&a));\n println!(\"count after creating b = {}\", Rc::strong_count(&a)); // 2\n {\n let c = Cons(4, Rc::clone(&a));\n println!(\"count after creating c = {}\", Rc::strong_count(&a)); // 3\n }\n println!(\"count after c goes out of scope = {}\", Rc::strong_count(&a)); // 2\n}\n```\nWhat we can’t see in this example is that when b and then a go out of scope at the end of main, the count is then 0, and the Rc is cleaned up completely at that point.\n\n## RefCell and interior mutability pattern\nInterior mutability is a design pattern in Rust that allows you to mutate data even when there are immutable references to that data; normally, this action is disallowed by the borrowing rules.\nTo mutate data, the pattern uses unsafe code inside a data structure to bend Rust’s usual rules that govern mutation and borrowing.\nWe can use types that use the interior mutability pattern when we can ensure that the borrowing rules will be followed at runtime, even though the compiler can’t guarantee that. \nThe unsafe code involved is then wrapped in a safe API, and the outer type is still immutable.\n\n### Enforcing Borrowing Rules at Runtime with RefCell\nUnlike Rc, the RefCell type represents single ownership over the data it holds.\n\nrecall borrowing rules\n- At any given time, you can have either (but not both of) one mutable reference or any number of immutable references.\n- References must always be valid.\nWith references and Box, the borrowing rules’ invariants are enforced at compile time. With RefCell, these invariants are enforced at runtime. \nBecause RefCell allows mutable borrows checked at runtime, you can mutate the value inside the RefCell even when the RefCell is immutable.\n\n### Interior Mutability: A Mutable Borrow to an Immutable Value\n```rust\nfn main() {\n let x = 5;\n let y = &mut x; // not allowed. cannot borrow `x` as mutable, as it is not declared as mutable\n}\n```\n\n### A Use Case for Interior Mutability: Mock Objects\n```rust\npub trait Messenger {\n fn send(&self, msg: &str);\n}\n\npub struct LimitTracker<'a, T: Messenger> {\n messenger: &'a T,\n value: usize,\n max: usize,\n}\n\nimpl<'a, T> LimitTracker<'a, T>\nwhere\n T: Messenger,\n{\n pub fn new(messenger: &T, max: usize) -> LimitTracker {\n LimitTracker {\n messenger,\n value: 0,\n max,\n }\n }\n\n pub fn set_value(&mut self, value: usize) {\n self.value = value;\n\n let percentage_of_max = self.value as f64 / self.max as f64;\n\n if percentage_of_max >= 1.0 {\n self.messenger.send(\"Error: You are over your quota!\");\n } else if percentage_of_max >= 0.9 {\n self.messenger\n .send(\"Urgent warning: You've used up over 90% of your quota!\");\n } else if percentage_of_max >= 0.75 {\n self.messenger\n .send(\"Warning: You've used up over 75% of your quota!\");\n }\n }\n}\n\n```\na problematic usage\n```rust\npub trait Messenger {\n fn send(&self, msg: &str);\n}\n\npub struct LimitTracker<'a, T: Messenger> {\n messenger: &'a T,\n value: usize,\n max: usize,\n}\n\nimpl<'a, T> LimitTracker<'a, T>\nwhere\n T: Messenger,\n{\n pub fn new(messenger: &T, max: usize) -> LimitTracker {\n LimitTracker {\n messenger,\n value: 0,\n max,\n }\n }\n\n pub fn set_value(&mut self, value: usize) {\n self.value = value;\n\n let percentage_of_max = self.value as f64 / self.max as f64;\n\n if percentage_of_max >= 1.0 {\n self.messenger.send(\"Error: You are over your quota!\");\n } else if percentage_of_max >= 0.9 {\n self.messenger\n .send(\"Urgent warning: You've used up over 90% of your quota!\");\n } else if percentage_of_max >= 0.75 {\n self.messenger\n .send(\"Warning: You've used up over 75% of your quota!\");\n }\n }\n}\n\n#[cfg(test)]\nmod tests {\n use super::*;\n\n struct MockMessenger {\n sent_messages: Vec,\n }\n\n impl MockMessenger {\n fn new() -> MockMessenger {\n MockMessenger {\n sent_messages: vec![],\n }\n }\n }\n\n impl Messenger for MockMessenger {\n fn send(&self, message: &str) {\n self.sent_messages.push(String::from(message)); // not allowed. cannot borrow `self.sent_messages` as mutable, as it is behind a `&` reference\n }\n }\n\n #[test]\n fn it_sends_an_over_75_percent_warning_message() {\n let mock_messenger = MockMessenger::new();\n let mut limit_tracker = LimitTracker::new(&mock_messenger, 100);\n\n limit_tracker.set_value(80);\n\n assert_eq!(mock_messenger.sent_messages.len(), 1);\n }\n}\n\n```\nWe can’t modify the MockMessenger to keep track of the messages, because the send method takes an immutable reference to self. We also can’t take the suggestion from the error text to use &mut self instead, because then the signature of send wouldn’t match the signature in the Messenger trait definition\n\nThis is a situation in which interior mutability can help! \n```rust\npub trait Messenger {\n fn send(&self, msg: &str);\n}\n\npub struct LimitTracker<'a, T: Messenger> {\n messenger: &'a T,\n value: usize,\n max: usize,\n}\n\nimpl<'a, T> LimitTracker<'a, T>\nwhere\n T: Messenger,\n{\n pub fn new(messenger: &T, max: usize) -> LimitTracker {\n LimitTracker {\n messenger,\n value: 0,\n max,\n }\n }\n\n pub fn set_value(&mut self, value: usize) {\n self.value = value;\n\n let percentage_of_max = self.value as f64 / self.max as f64;\n\n if percentage_of_max >= 1.0 {\n self.messenger.send(\"Error: You are over your quota!\");\n } else if percentage_of_max >= 0.9 {\n self.messenger\n .send(\"Urgent warning: You've used up over 90% of your quota!\");\n } else if percentage_of_max >= 0.75 {\n self.messenger\n .send(\"Warning: You've used up over 75% of your quota!\");\n }\n }\n}\n\n#[cfg(test)]\nmod tests {\n use super::*;\n use std::cell::RefCell;\n\n struct MockMessenger {\n sent_messages: RefCell>,\n }\n\n impl MockMessenger {\n fn new() -> MockMessenger {\n MockMessenger {\n sent_messages: RefCell::new(vec![]),\n }\n }\n }\n\n impl Messenger for MockMessenger {\n fn send(&self, message: &str) {\n self.sent_messages.borrow_mut().push(String::from(message)); // call borrow_mut on the RefCell> in self.sent_messages to get a mutable reference to the value inside the RefCell>\n }\n }\n\n #[test]\n fn it_sends_an_over_75_percent_warning_message() {\n // --snip--\n let mock_messenger = MockMessenger::new();\n let mut limit_tracker = LimitTracker::new(&mock_messenger, 100);\n\n limit_tracker.set_value(80);\n\n assert_eq!(mock_messenger.sent_messages.borrow().len(), 1); // call borrow on the RefCell> to get an immutable reference to the vector.\n }\n}\n```\n\n### Keeping Track of Borrows at Runtime with RefCell\nWhen creating immutable and mutable references, we use the & and &mut syntax, respectively. With RefCell, we use the borrow and borrow_mut methods, which are part of the safe API that belongs to RefCell. **The borrow method returns the smart pointer type Ref, and borrow_mut returns the smart pointer type RefMut**. Both types implement Deref, so we can treat them like regular references.\n\nThe RefCell keeps track of how many Ref and RefMut smart pointers are currently active.RefCell lets us have many immutable borrows or one mutable borrow at any point in time. If we try to violate these rules, rather than getting a compiler error as we would with references, the implementation of RefCell will panic at runtime. \n```rust\nimpl Messenger for MockMessenger {\n fn send(&self, message: &str) {\n let mut one_borrow = self.sent_messages.borrow_mut();\n let mut two_borrow = self.sent_messages.borrow_mut();\n\n one_borrow.push(String::from(message));\n two_borrow.push(String::from(message));\n }\n }\n```\nWhen we run the tests for our library, the code in will compile without any errors, but the test will fail\nthread 'main' panicked at 'already borrowed\n\n### Having Multiple Owners of Mutable Data by Combining Rc and RefCell\nA common way to use RefCell is in combination with Rc. If you have an Rc that holds a RefCell, you can get a value that can have multiple owners and that you can mutate!\n```rust\n#[derive(Debug)]\nenum List {\n Cons(Rc>, Rc),\n Nil,\n}\n\nuse crate::List::{Cons, Nil};\nuse std::cell::RefCell;\nuse std::rc::Rc;\n\nfn main() {\n let value = Rc::new(RefCell::new(5)); // We create a value that is an instance of Rc> and store it in a variable named value so we can access it directly later.\n\n let a = Rc::new(Cons(Rc::clone(&value), Rc::new(Nil))); // Then we create a List in a with a Cons variant that holds value. We need to clone value so both a and value have ownership of the inner 5 value rather than transferring ownership from value to a or having a borrow from value. We wrap the list a in an Rc so when we create lists b and c, they can both refer to a\n\n let b = Cons(Rc::new(RefCell::new(3)), Rc::clone(&a));\n let c = Cons(Rc::new(RefCell::new(4)), Rc::clone(&a));\n\n *value.borrow_mut() += 10; // After we’ve created the lists in a, b, and c, we add 10 to the value in value. We do this by calling borrow_mut on value, which uses the automatic dereferencing feature to dereference the Rc to the inner RefCell value. The borrow_mut method returns a RefMut smart pointer, and we use the dereference operator on it and change the inner value.\n\n println!(\"a after = {:?}\", a);\n println!(\"b after = {:?}\", b);\n println!(\"c after = {:?}\", c);\n}\n```\n\n**The standard library has other types that provide interior mutability, such as Cell, which is similar except that instead of giving references to the inner value, the value is copied in and out of the Cell. There’s also Mutex, which offers interior mutability that’s safe to use across threads;** \n\n## Reference Cycles Can Leak Memory\nRust’s memory safety guarantees make it difficult, but not impossible, to accidentally create memory that is never cleaned up (known as a memory leak).\n\n### Creating a Reference Cycle\n```rust\nuse crate::List::{Cons, Nil};\nuse std::cell::RefCell;\nuse std::rc::Rc;\n\n#[derive(Debug)]\nenum List {\n Cons(i32, RefCell>), // The second element in the Cons variant is now RefCell>, meaning that we want to modify which List value a Cons variant is pointing to. \n Nil,\n}\n\nimpl List {\n fn tail(&self) -> Option<&RefCell>> { // We’re also adding a tail method to make it convenient for us to access the second item if we have a Cons variant.\n match self {\n Cons(_, item) => Some(item),\n Nil => None,\n }\n }\n}\n\nfn main() {\n use crate::List::{Cons, Nil};\nuse std::cell::RefCell;\nuse std::rc::Rc;\n\n#[derive(Debug)]\nenum List {\n Cons(i32, RefCell>),\n Nil,\n}\n\nimpl List {\n fn tail(&self) -> Option<&RefCell>> {\n match self {\n Cons(_, item) => Some(item),\n Nil => None,\n }\n }\n}\n\nfn main() {\n let a = Rc::new(Cons(5, RefCell::new(Rc::new(Nil))));\n\n println!(\"a initial rc count = {}\", Rc::strong_count(&a));\n println!(\"a next item = {:?}\", a.tail());\n\n let b = Rc::new(Cons(10, RefCell::new(Rc::clone(&a)))); // his code creates a list in a and a list in b that points to the list in a\n\n println!(\"a rc count after b creation = {}\", Rc::strong_count(&a));\n println!(\"b initial rc count = {}\", Rc::strong_count(&b));\n println!(\"b next item = {:?}\", b.tail());\n\n if let Some(link) = a.tail() {\n *link.borrow_mut() = Rc::clone(&b); // modifies the list in a to point to b, creating a reference cycle\n }\n\n println!(\"b rc count after changing a = {}\", Rc::strong_count(&b));\n println!(\"a rc count after changing a = {}\", Rc::strong_count(&a));\n\n // Uncomment the next line to see that we have a cycle;\n // it will overflow the stack\n // println!(\"a next item = {:?}\", a.tail());\n} // At the end of main, Rust drops the variable b, which decreases the reference count of the Rc instance from 2 to 1. The memory that Rc has on the heap won’t be dropped at this point, because its reference count is 1, not 0. Then Rust drops a, which decreases the reference count of the a Rc instance from 2 to 1 as well. This instance’s memory can’t be dropped either, because the other Rc instance still refers to it. The memory allocated to the list will remain uncollected forever.\n```\n![cycle-ref](/images/rust/pointers/cycle.ref.png)\n\n### Preventing Reference Cycles: Turning an Rc into a Weak\nSo far, we’ve demonstrated that calling Rc::clone increases the strong_count of an Rc instance, and an Rc instance is only cleaned up if its strong_count is 0. You can also create a weak reference to the value within an Rc instance by calling Rc::downgrade and passing a reference to the Rc. When you call Rc::downgrade, you get a smart pointer of type Weak. Instead of increasing the strong_count in the Rc instance by 1, calling Rc::downgrade increases the weak_count by 1. The Rc type uses weak_count to keep track of how many Weak references exist, similar to strong_count. The difference is the weak_count doesn’t need to be 0 for the Rc instance to be cleaned up.\n\nStrong references are how you can share ownership of an Rc instance. Weak references don’t express an ownership relationship. They won’t cause a reference cycle because any cycle involving some weak references will be broken once the strong reference count of values involved is 0.\n\nBecause the value that Weak references might have been dropped, to do anything with the value that a Weak is pointing to, you must make sure the value still exists. Do this by calling the upgrade method on a Weak instance, which will return an Option>. You’ll get a result of Some if the Rc value has not been dropped yet and a result of None if the Rc value has been dropped. \n\n\n### Creating a Tree Data Structure: a Node with Child Nodes\n```rust\nuse std::cell::RefCell;\nuse std::rc::Rc;\n\n#[derive(Debug)]\nstruct Node {\n value: i32,\n parent: RefCell>, // To make the child node aware of its parent, we need to add a parent field to our Node struct definition. The trouble is in deciding what the type of parent should be. We know it can’t contain an Rc, because that would create a reference cycle with leaf.parent pointing to branch and branch.children pointing to leaf, which would cause their strong_count values to never be 0.\n children: RefCell>>,\n}\n\nfn main() {\n\n let leaf = Rc::new(Node {\n value: 3,\n parent: RefCell::new(Weak::new()),\n children: RefCell::new(vec![]),\n });\n\n println!(\"leaf parent = {:?}\", leaf.parent.borrow().upgrade()); // try to get a reference to the parent of leaf by using the upgrade method, we get a None value. \n\n let branch = Rc::new(Node {\n value: 5,\n parent: RefCell::new(Weak::new()),\n children: RefCell::new(vec![Rc::clone(&leaf)]), // We clone the Rc in leaf and store that in branch, meaning the Node in leaf now has two owners: leaf and branch. \n });\n\n *leaf.parent.borrow_mut() = Rc::downgrade(&branch); // use the Rc::downgrade function to create a Weak reference to branch from the Rc in branch.\n\n println!(\"leaf parent = {:?}\", leaf.parent.borrow().upgrade());\n\n \n}\n```\n\n### Visualizing Changes to strong_count and weak_count\n```rust\nuse std::cell::RefCell;\nuse std::rc::{Rc, Weak};\n\n#[derive(Debug)]\nstruct Node {\n value: i32,\n parent: RefCell>,\n children: RefCell>>,\n}\n\nfn main() {\n let leaf = Rc::new(Node {\n value: 3,\n parent: RefCell::new(Weak::new()),\n children: RefCell::new(vec![]),\n });\n\n println!(\n \"leaf strong = {}, weak = {}\",\n Rc::strong_count(&leaf),\n Rc::weak_count(&leaf),\n );\n\n {\n let branch = Rc::new(Node {\n value: 5,\n parent: RefCell::new(Weak::new()),\n children: RefCell::new(vec![Rc::clone(&leaf)]),\n });\n\n *leaf.parent.borrow_mut() = Rc::downgrade(&branch);\n\n println!(\n \"branch strong = {}, weak = {}\",\n Rc::strong_count(&branch),\n Rc::weak_count(&branch),\n );\n\n println!(\n \"leaf strong = {}, weak = {}\",\n Rc::strong_count(&leaf),\n Rc::weak_count(&leaf),\n );\n }\n\n println!(\"leaf parent = {:?}\", leaf.parent.borrow().upgrade());\n println!(\n \"leaf strong = {}, weak = {}\",\n Rc::strong_count(&leaf),\n Rc::weak_count(&leaf),\n );\n}\n```","source":"_posts/rust/rust-06-smart-pointer.md","raw":"---\ntitle: rust smart pointer\ndate: 2022-10-30 11:00:38\ntags: [rust]\n---\n\n## Overview\nThe most common kind of pointer in Rust is a reference (borrow but not own)\nSmart pointers, on the other hand, are data structures that not only act like a pointer but also have additional metadata and capabilities.\nreferences are pointers that only borrow data; in contrast, in many cases, smart pointers own the data they point to.\n\n### smart pointers example\n- String\n- Vec\nBoth these types count as smart pointers because they own some memory and allow you to manipulate it. They also have metadata (such as their capacity) and extra capabilities or guarantees (such as with String ensuring its data will always be valid UTF-8).\n\n### Deref & Drop\nSmart pointers are usually implemented using structs. The characteristic that distinguishes a smart pointer from an ordinary struct is that smart pointers implement the `Deref` and `Drop` traits.\n- The Deref trait allows an instance of the smart pointer struct to behave like a reference so you can write code that works with either references or smart pointers. \n- The Drop trait allows you to customize the code that is run when an instance of the smart pointer goes out of scope.\n\n### the most common smart pointers in the standard library:\n- `Box` for allocating values on the heap\n- `Rc`, a reference counting type that enables multiple ownership\n- `Ref` and `RefMut`, accessed through `RefCell`, a type that enforces the borrowing rules at runtime instead of compile time\n\n## Box\n### when to use\n- When you have a type whose size can’t be known at compile time and you want to use a value of that type in a context that requires an exact size. (such as cons)\n- When you have a large amount of data and you want to transfer ownership but ensure the data won’t be copied when you do so\n- When you want to own a value and you care only that it’s a type that implements a particular trait rather than being of a specific type\n\n\n### enabling recursive types with Box\nAt compile time, Rust needs to know how much space a type takes up\nOne type whose size can’t be known at compile time is a recursive type (cons list), use Box, which only contains a memory address\n\n```rust\nenum List {\n Cons(i32, List),\n Nil,\n}\n\nuse crate::List::{Cons, Nil};\nfn main() {\n let list = Cons(1, Cons(2, Cons(3, Nil))); // not allowed, infinite size\n}\n```\n\nuse Box\n```rust\nenum List {\n Cons(i32, Box),\n Nil,\n}\n\nuse crate::List::{Cons, Nil};\n\nfn main() {\n let list = Cons(1, Box::new(Cons(2, Box::new(Cons(3, Box::new(Nil))))));\n}\n```\n\n## Treating Smart Pointers Like Regular References with the Deref Trait\nImplementing the Deref trait allows you to customize the behavior of the dereference operator, `*`\nBy implementing Deref in such a way that a smart pointer can be treated like a regular reference, you can write code that operates on references and use that code with smart pointers too.\n\n### Defining Our Own Smart Pointer\n```rust\nuse std::ops::Deref;\n\nstruct MyBox(T); // The MyBox type is a tuple struct with one element of type T\n\nimpl MyBox {\n fn new(x: T) -> MyBox {\n MyBox(x)\n }\n}\n\nimpl Deref for MyBox {\n type Target = T;\n\n fn deref(&self) -> &Self::Target {\n &self.0\n }\n}\n\nfn main() {\n let x = 5;\n let y = MyBox::new(x);\n\n assert_eq!(5, x);\n assert_eq!(5, *y);\n}\n```\n- We fill in the body of the deref method with &self.0 so deref returns a reference to the value we want to access with the * operator. \n- behind the scenes Rust actually ran this code: *(y.deref()). Rust substitutes the * operator with a call to the deref method\n- The reason the deref method returns a reference to a value, and that the plain dereference outside the parentheses in *(y.deref()) is still necessary, is the ownership system. If the deref method returned the value directly instead of a reference to the value, the value would be moved out of self. We don’t want to take ownership of the inner value inside MyBox in this case or in most cases where we use the dereference operator.\n\n### Implicit Deref Coercions with Functions and Methods\nDeref coercion is a convenience that Rust performs on arguments to functions and methods. \nDeref coercion works only on types that implement the Deref trait. Deref coercion converts a reference to such a type into a reference to another type. For example, deref coercion can convert &String to &str because String implements the Deref trait such that it returns &str\nA sequence of calls to the deref method converts the type we provided into the type the parameter needs.\n\n```rust\nfn hello(name: &str) {\n println!(\"Hello, {}!\", name);\n}\n\nfn main() {\n let m = MyBox::new(String::from(\"Rust\"));\n hello(\"rust\"); // ok\n hello(&m); // also ok\n hello(&(*m)[..]); // without deref coercion. The (*m) dereferences the MyBox into a String. Then the & and [..] take a string slice of the String that is equal to the whole string to match the signature of hello\n}\n```\nHere we’re calling the hello function with the argument &m, which is a reference to a MyBox value. Because we implemented the Deref trait on MyBox, Rust can turn &MyBox into &String by calling deref. The standard library provides an implementation of Deref on String that returns a string slice. Rust calls deref again to turn the &String into &str, which matches the hello function’s definition.\n\n### How Deref Coercion Interacts with Mutability\nSimilar to how you use the Deref trait to override the * operator on immutable references, you can use the DerefMut trait to override the * operator on mutable references.\n\nRust does deref coercion when it finds types and trait implementations in three cases:\n- From &T to &U when T: Deref\n- From &mut T to &mut U when T: DerefMut\n- From &mut T to &U when T: Deref\n\n## Running Code on Cleanup with the Drop Trait\nDrop, which lets you customize what happens when a value is about to go out of scope. \n\n```rust\nstruct CustomSmartPointer {\n data: String,\n}\n\nimpl Drop for CustomSmartPointer {\n fn drop(&mut self) {\n println!(\"Dropping CustomSmartPointer with data `{}`!\", self.data);\n }\n}\n\nfn main() {\n let c = CustomSmartPointer {\n data: String::from(\"my stuff\"),\n };\n c.drop(); // not allowed\n let d = CustomSmartPointer {\n data: String::from(\"other stuff\"),\n };\n println!(\"CustomSmartPointers created.\");\n}\n\n```\n### Dropping a Value Early with std::mem::drop\nstd::mem::drop is in prelude, can use directly\n```rust\nstruct CustomSmartPointer {\n data: String,\n}\n\nimpl Drop for CustomSmartPointer {\n fn drop(&mut self) {\n println!(\"Dropping CustomSmartPointer with data `{}`!\", self.data);\n }\n}\n\nfn main() {\n let c = CustomSmartPointer {\n data: String::from(\"some data\"),\n };\n println!(\"CustomSmartPointer created.\");\n drop(c); // ok\n println!(\"CustomSmartPointer dropped before the end of main.\");\n} // c goes out of scope, will occur double drop\n\n```\n\n\n## Rc: the Reference Counted Smart Pointer\nIn the majority of cases, ownership is clear: you know exactly which variable owns a given value. However, there are cases when a single value might have multiple owners. \ntype keeps track of the number of references to a value to determine whether or not the value is still in use. If there are zero references to a value, the value can be cleaned up without any references becoming invalid.\n\nWe use the `Rc` type when we want to allocate some data on the heap for multiple parts of our program to read and we can’t determine at compile time which part will finish using the data last.\n\nRc is only for use in single-threaded scenarios\n\n### using Rc to share data\n\n![rc](/images/rust/pointers/rc.png)\nimplement use box will not work, as below\n```rust\nenum List {\n Cons(i32, Box),\n Nil,\n}\n\nuse crate::List::{Cons, Nil};\n\nfn main() {\n let a = Cons(5, Box::new(Cons(10, Box::new(Nil))));\n let b = Cons(3, Box::new(a)); // value moved here\n let c = Cons(4, Box::new(a)); // value used here after move\n}\n```\nuse Rc\n```rust\nenum List {\n Cons(i32, Rc),\n Nil,\n}\n\nuse crate::List::{Cons, Nil};\nuse std::rc::Rc;\n\nfn main() {\n let a = Rc::new(Cons(5, Rc::new(Cons(10, Rc::new(Nil)))));\n let b = Cons(3, Rc::clone(&a)); // shalow copy only reference, not data\n let c = Cons(4, Rc::clone(&a));\n}\n```\n\n\nWe could have called a.clone() rather than Rc::clone(&a), but Rust’s convention is to use Rc::clone in this case. The implementation of Rc::clone doesn’t make a deep copy of all the data like most types’ implementations of clone do. The call to Rc::clone only increments the reference count, which doesn’t take much time.\n\n### Cloning an Rc Increases the Reference Count\n```rust\nenum List {\n Cons(i32, Rc),\n Nil,\n}\n\nuse crate::List::{Cons, Nil};\nuse std::rc::Rc;\n\nfn main() {\n let a = Rc::new(Cons(5, Rc::new(Cons(10, Rc::new(Nil)))));\n println!(\"count after creating a = {}\", Rc::strong_count(&a)); // 1\n let b = Cons(3, Rc::clone(&a));\n println!(\"count after creating b = {}\", Rc::strong_count(&a)); // 2\n {\n let c = Cons(4, Rc::clone(&a));\n println!(\"count after creating c = {}\", Rc::strong_count(&a)); // 3\n }\n println!(\"count after c goes out of scope = {}\", Rc::strong_count(&a)); // 2\n}\n```\nWhat we can’t see in this example is that when b and then a go out of scope at the end of main, the count is then 0, and the Rc is cleaned up completely at that point.\n\n## RefCell and interior mutability pattern\nInterior mutability is a design pattern in Rust that allows you to mutate data even when there are immutable references to that data; normally, this action is disallowed by the borrowing rules.\nTo mutate data, the pattern uses unsafe code inside a data structure to bend Rust’s usual rules that govern mutation and borrowing.\nWe can use types that use the interior mutability pattern when we can ensure that the borrowing rules will be followed at runtime, even though the compiler can’t guarantee that. \nThe unsafe code involved is then wrapped in a safe API, and the outer type is still immutable.\n\n### Enforcing Borrowing Rules at Runtime with RefCell\nUnlike Rc, the RefCell type represents single ownership over the data it holds.\n\nrecall borrowing rules\n- At any given time, you can have either (but not both of) one mutable reference or any number of immutable references.\n- References must always be valid.\nWith references and Box, the borrowing rules’ invariants are enforced at compile time. With RefCell, these invariants are enforced at runtime. \nBecause RefCell allows mutable borrows checked at runtime, you can mutate the value inside the RefCell even when the RefCell is immutable.\n\n### Interior Mutability: A Mutable Borrow to an Immutable Value\n```rust\nfn main() {\n let x = 5;\n let y = &mut x; // not allowed. cannot borrow `x` as mutable, as it is not declared as mutable\n}\n```\n\n### A Use Case for Interior Mutability: Mock Objects\n```rust\npub trait Messenger {\n fn send(&self, msg: &str);\n}\n\npub struct LimitTracker<'a, T: Messenger> {\n messenger: &'a T,\n value: usize,\n max: usize,\n}\n\nimpl<'a, T> LimitTracker<'a, T>\nwhere\n T: Messenger,\n{\n pub fn new(messenger: &T, max: usize) -> LimitTracker {\n LimitTracker {\n messenger,\n value: 0,\n max,\n }\n }\n\n pub fn set_value(&mut self, value: usize) {\n self.value = value;\n\n let percentage_of_max = self.value as f64 / self.max as f64;\n\n if percentage_of_max >= 1.0 {\n self.messenger.send(\"Error: You are over your quota!\");\n } else if percentage_of_max >= 0.9 {\n self.messenger\n .send(\"Urgent warning: You've used up over 90% of your quota!\");\n } else if percentage_of_max >= 0.75 {\n self.messenger\n .send(\"Warning: You've used up over 75% of your quota!\");\n }\n }\n}\n\n```\na problematic usage\n```rust\npub trait Messenger {\n fn send(&self, msg: &str);\n}\n\npub struct LimitTracker<'a, T: Messenger> {\n messenger: &'a T,\n value: usize,\n max: usize,\n}\n\nimpl<'a, T> LimitTracker<'a, T>\nwhere\n T: Messenger,\n{\n pub fn new(messenger: &T, max: usize) -> LimitTracker {\n LimitTracker {\n messenger,\n value: 0,\n max,\n }\n }\n\n pub fn set_value(&mut self, value: usize) {\n self.value = value;\n\n let percentage_of_max = self.value as f64 / self.max as f64;\n\n if percentage_of_max >= 1.0 {\n self.messenger.send(\"Error: You are over your quota!\");\n } else if percentage_of_max >= 0.9 {\n self.messenger\n .send(\"Urgent warning: You've used up over 90% of your quota!\");\n } else if percentage_of_max >= 0.75 {\n self.messenger\n .send(\"Warning: You've used up over 75% of your quota!\");\n }\n }\n}\n\n#[cfg(test)]\nmod tests {\n use super::*;\n\n struct MockMessenger {\n sent_messages: Vec,\n }\n\n impl MockMessenger {\n fn new() -> MockMessenger {\n MockMessenger {\n sent_messages: vec![],\n }\n }\n }\n\n impl Messenger for MockMessenger {\n fn send(&self, message: &str) {\n self.sent_messages.push(String::from(message)); // not allowed. cannot borrow `self.sent_messages` as mutable, as it is behind a `&` reference\n }\n }\n\n #[test]\n fn it_sends_an_over_75_percent_warning_message() {\n let mock_messenger = MockMessenger::new();\n let mut limit_tracker = LimitTracker::new(&mock_messenger, 100);\n\n limit_tracker.set_value(80);\n\n assert_eq!(mock_messenger.sent_messages.len(), 1);\n }\n}\n\n```\nWe can’t modify the MockMessenger to keep track of the messages, because the send method takes an immutable reference to self. We also can’t take the suggestion from the error text to use &mut self instead, because then the signature of send wouldn’t match the signature in the Messenger trait definition\n\nThis is a situation in which interior mutability can help! \n```rust\npub trait Messenger {\n fn send(&self, msg: &str);\n}\n\npub struct LimitTracker<'a, T: Messenger> {\n messenger: &'a T,\n value: usize,\n max: usize,\n}\n\nimpl<'a, T> LimitTracker<'a, T>\nwhere\n T: Messenger,\n{\n pub fn new(messenger: &T, max: usize) -> LimitTracker {\n LimitTracker {\n messenger,\n value: 0,\n max,\n }\n }\n\n pub fn set_value(&mut self, value: usize) {\n self.value = value;\n\n let percentage_of_max = self.value as f64 / self.max as f64;\n\n if percentage_of_max >= 1.0 {\n self.messenger.send(\"Error: You are over your quota!\");\n } else if percentage_of_max >= 0.9 {\n self.messenger\n .send(\"Urgent warning: You've used up over 90% of your quota!\");\n } else if percentage_of_max >= 0.75 {\n self.messenger\n .send(\"Warning: You've used up over 75% of your quota!\");\n }\n }\n}\n\n#[cfg(test)]\nmod tests {\n use super::*;\n use std::cell::RefCell;\n\n struct MockMessenger {\n sent_messages: RefCell>,\n }\n\n impl MockMessenger {\n fn new() -> MockMessenger {\n MockMessenger {\n sent_messages: RefCell::new(vec![]),\n }\n }\n }\n\n impl Messenger for MockMessenger {\n fn send(&self, message: &str) {\n self.sent_messages.borrow_mut().push(String::from(message)); // call borrow_mut on the RefCell> in self.sent_messages to get a mutable reference to the value inside the RefCell>\n }\n }\n\n #[test]\n fn it_sends_an_over_75_percent_warning_message() {\n // --snip--\n let mock_messenger = MockMessenger::new();\n let mut limit_tracker = LimitTracker::new(&mock_messenger, 100);\n\n limit_tracker.set_value(80);\n\n assert_eq!(mock_messenger.sent_messages.borrow().len(), 1); // call borrow on the RefCell> to get an immutable reference to the vector.\n }\n}\n```\n\n### Keeping Track of Borrows at Runtime with RefCell\nWhen creating immutable and mutable references, we use the & and &mut syntax, respectively. With RefCell, we use the borrow and borrow_mut methods, which are part of the safe API that belongs to RefCell. **The borrow method returns the smart pointer type Ref, and borrow_mut returns the smart pointer type RefMut**. Both types implement Deref, so we can treat them like regular references.\n\nThe RefCell keeps track of how many Ref and RefMut smart pointers are currently active.RefCell lets us have many immutable borrows or one mutable borrow at any point in time. If we try to violate these rules, rather than getting a compiler error as we would with references, the implementation of RefCell will panic at runtime. \n```rust\nimpl Messenger for MockMessenger {\n fn send(&self, message: &str) {\n let mut one_borrow = self.sent_messages.borrow_mut();\n let mut two_borrow = self.sent_messages.borrow_mut();\n\n one_borrow.push(String::from(message));\n two_borrow.push(String::from(message));\n }\n }\n```\nWhen we run the tests for our library, the code in will compile without any errors, but the test will fail\nthread 'main' panicked at 'already borrowed\n\n### Having Multiple Owners of Mutable Data by Combining Rc and RefCell\nA common way to use RefCell is in combination with Rc. If you have an Rc that holds a RefCell, you can get a value that can have multiple owners and that you can mutate!\n```rust\n#[derive(Debug)]\nenum List {\n Cons(Rc>, Rc),\n Nil,\n}\n\nuse crate::List::{Cons, Nil};\nuse std::cell::RefCell;\nuse std::rc::Rc;\n\nfn main() {\n let value = Rc::new(RefCell::new(5)); // We create a value that is an instance of Rc> and store it in a variable named value so we can access it directly later.\n\n let a = Rc::new(Cons(Rc::clone(&value), Rc::new(Nil))); // Then we create a List in a with a Cons variant that holds value. We need to clone value so both a and value have ownership of the inner 5 value rather than transferring ownership from value to a or having a borrow from value. We wrap the list a in an Rc so when we create lists b and c, they can both refer to a\n\n let b = Cons(Rc::new(RefCell::new(3)), Rc::clone(&a));\n let c = Cons(Rc::new(RefCell::new(4)), Rc::clone(&a));\n\n *value.borrow_mut() += 10; // After we’ve created the lists in a, b, and c, we add 10 to the value in value. We do this by calling borrow_mut on value, which uses the automatic dereferencing feature to dereference the Rc to the inner RefCell value. The borrow_mut method returns a RefMut smart pointer, and we use the dereference operator on it and change the inner value.\n\n println!(\"a after = {:?}\", a);\n println!(\"b after = {:?}\", b);\n println!(\"c after = {:?}\", c);\n}\n```\n\n**The standard library has other types that provide interior mutability, such as Cell, which is similar except that instead of giving references to the inner value, the value is copied in and out of the Cell. There’s also Mutex, which offers interior mutability that’s safe to use across threads;** \n\n## Reference Cycles Can Leak Memory\nRust’s memory safety guarantees make it difficult, but not impossible, to accidentally create memory that is never cleaned up (known as a memory leak).\n\n### Creating a Reference Cycle\n```rust\nuse crate::List::{Cons, Nil};\nuse std::cell::RefCell;\nuse std::rc::Rc;\n\n#[derive(Debug)]\nenum List {\n Cons(i32, RefCell>), // The second element in the Cons variant is now RefCell>, meaning that we want to modify which List value a Cons variant is pointing to. \n Nil,\n}\n\nimpl List {\n fn tail(&self) -> Option<&RefCell>> { // We’re also adding a tail method to make it convenient for us to access the second item if we have a Cons variant.\n match self {\n Cons(_, item) => Some(item),\n Nil => None,\n }\n }\n}\n\nfn main() {\n use crate::List::{Cons, Nil};\nuse std::cell::RefCell;\nuse std::rc::Rc;\n\n#[derive(Debug)]\nenum List {\n Cons(i32, RefCell>),\n Nil,\n}\n\nimpl List {\n fn tail(&self) -> Option<&RefCell>> {\n match self {\n Cons(_, item) => Some(item),\n Nil => None,\n }\n }\n}\n\nfn main() {\n let a = Rc::new(Cons(5, RefCell::new(Rc::new(Nil))));\n\n println!(\"a initial rc count = {}\", Rc::strong_count(&a));\n println!(\"a next item = {:?}\", a.tail());\n\n let b = Rc::new(Cons(10, RefCell::new(Rc::clone(&a)))); // his code creates a list in a and a list in b that points to the list in a\n\n println!(\"a rc count after b creation = {}\", Rc::strong_count(&a));\n println!(\"b initial rc count = {}\", Rc::strong_count(&b));\n println!(\"b next item = {:?}\", b.tail());\n\n if let Some(link) = a.tail() {\n *link.borrow_mut() = Rc::clone(&b); // modifies the list in a to point to b, creating a reference cycle\n }\n\n println!(\"b rc count after changing a = {}\", Rc::strong_count(&b));\n println!(\"a rc count after changing a = {}\", Rc::strong_count(&a));\n\n // Uncomment the next line to see that we have a cycle;\n // it will overflow the stack\n // println!(\"a next item = {:?}\", a.tail());\n} // At the end of main, Rust drops the variable b, which decreases the reference count of the Rc instance from 2 to 1. The memory that Rc has on the heap won’t be dropped at this point, because its reference count is 1, not 0. Then Rust drops a, which decreases the reference count of the a Rc instance from 2 to 1 as well. This instance’s memory can’t be dropped either, because the other Rc instance still refers to it. The memory allocated to the list will remain uncollected forever.\n```\n![cycle-ref](/images/rust/pointers/cycle.ref.png)\n\n### Preventing Reference Cycles: Turning an Rc into a Weak\nSo far, we’ve demonstrated that calling Rc::clone increases the strong_count of an Rc instance, and an Rc instance is only cleaned up if its strong_count is 0. You can also create a weak reference to the value within an Rc instance by calling Rc::downgrade and passing a reference to the Rc. When you call Rc::downgrade, you get a smart pointer of type Weak. Instead of increasing the strong_count in the Rc instance by 1, calling Rc::downgrade increases the weak_count by 1. The Rc type uses weak_count to keep track of how many Weak references exist, similar to strong_count. The difference is the weak_count doesn’t need to be 0 for the Rc instance to be cleaned up.\n\nStrong references are how you can share ownership of an Rc instance. Weak references don’t express an ownership relationship. They won’t cause a reference cycle because any cycle involving some weak references will be broken once the strong reference count of values involved is 0.\n\nBecause the value that Weak references might have been dropped, to do anything with the value that a Weak is pointing to, you must make sure the value still exists. Do this by calling the upgrade method on a Weak instance, which will return an Option>. You’ll get a result of Some if the Rc value has not been dropped yet and a result of None if the Rc value has been dropped. \n\n\n### Creating a Tree Data Structure: a Node with Child Nodes\n```rust\nuse std::cell::RefCell;\nuse std::rc::Rc;\n\n#[derive(Debug)]\nstruct Node {\n value: i32,\n parent: RefCell>, // To make the child node aware of its parent, we need to add a parent field to our Node struct definition. The trouble is in deciding what the type of parent should be. We know it can’t contain an Rc, because that would create a reference cycle with leaf.parent pointing to branch and branch.children pointing to leaf, which would cause their strong_count values to never be 0.\n children: RefCell>>,\n}\n\nfn main() {\n\n let leaf = Rc::new(Node {\n value: 3,\n parent: RefCell::new(Weak::new()),\n children: RefCell::new(vec![]),\n });\n\n println!(\"leaf parent = {:?}\", leaf.parent.borrow().upgrade()); // try to get a reference to the parent of leaf by using the upgrade method, we get a None value. \n\n let branch = Rc::new(Node {\n value: 5,\n parent: RefCell::new(Weak::new()),\n children: RefCell::new(vec![Rc::clone(&leaf)]), // We clone the Rc in leaf and store that in branch, meaning the Node in leaf now has two owners: leaf and branch. \n });\n\n *leaf.parent.borrow_mut() = Rc::downgrade(&branch); // use the Rc::downgrade function to create a Weak reference to branch from the Rc in branch.\n\n println!(\"leaf parent = {:?}\", leaf.parent.borrow().upgrade());\n\n \n}\n```\n\n### Visualizing Changes to strong_count and weak_count\n```rust\nuse std::cell::RefCell;\nuse std::rc::{Rc, Weak};\n\n#[derive(Debug)]\nstruct Node {\n value: i32,\n parent: RefCell>,\n children: RefCell>>,\n}\n\nfn main() {\n let leaf = Rc::new(Node {\n value: 3,\n parent: RefCell::new(Weak::new()),\n children: RefCell::new(vec![]),\n });\n\n println!(\n \"leaf strong = {}, weak = {}\",\n Rc::strong_count(&leaf),\n Rc::weak_count(&leaf),\n );\n\n {\n let branch = Rc::new(Node {\n value: 5,\n parent: RefCell::new(Weak::new()),\n children: RefCell::new(vec![Rc::clone(&leaf)]),\n });\n\n *leaf.parent.borrow_mut() = Rc::downgrade(&branch);\n\n println!(\n \"branch strong = {}, weak = {}\",\n Rc::strong_count(&branch),\n Rc::weak_count(&branch),\n );\n\n println!(\n \"leaf strong = {}, weak = {}\",\n Rc::strong_count(&leaf),\n Rc::weak_count(&leaf),\n );\n }\n\n println!(\"leaf parent = {:?}\", leaf.parent.borrow().upgrade());\n println!(\n \"leaf strong = {}, weak = {}\",\n Rc::strong_count(&leaf),\n Rc::weak_count(&leaf),\n );\n}\n```","slug":"rust/rust-06-smart-pointer","published":1,"updated":"2023-11-05T04:21:13.734Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8ds0014qwsjeg0uevgt","content":"

Overview

The most common kind of pointer in Rust is a reference (borrow but not own)
Smart pointers, on the other hand, are data structures that not only act like a pointer but also have additional metadata and capabilities.
references are pointers that only borrow data; in contrast, in many cases, smart pointers own the data they point to.

\n

smart pointers example

    \n
  • String
    • \n
  • Vec
    Both these types count as smart pointers because they own some memory and allow you to manipulate it. They also have metadata (such as their capacity) and extra capabilities or guarantees (such as with String ensuring its data will always be valid UTF-8).
    • \n
\n

Deref & Drop

Smart pointers are usually implemented using structs. The characteristic that distinguishes a smart pointer from an ordinary struct is that smart pointers implement the Deref and Drop traits.

\n
    \n
  • The Deref trait allows an instance of the smart pointer struct to behave like a reference so you can write code that works with either references or smart pointers.
    • \n
  • The Drop trait allows you to customize the code that is run when an instance of the smart pointer goes out of scope.
    • \n
\n

the most common smart pointers in the standard library:

    \n
  • Box<T> for allocating values on the heap
    • \n
  • Rc<T>, a reference counting type that enables multiple ownership
    • \n
  • Ref<T> and RefMut<T>, accessed through RefCell<T>, a type that enforces the borrowing rules at runtime instead of compile time
    • \n
\n

Box

when to use

    \n
  • When you have a type whose size can’t be known at compile time and you want to use a value of that type in a context that requires an exact size. (such as cons)
    • \n
  • When you have a large amount of data and you want to transfer ownership but ensure the data won’t be copied when you do so
    • \n
  • When you want to own a value and you care only that it’s a type that implements a particular trait rather than being of a specific type
    • \n
\n

enabling recursive types with Box

At compile time, Rust needs to know how much space a type takes up
One type whose size can’t be known at compile time is a recursive type (cons list), use Box, which only contains a memory address

\n
1
2
3
4
5
6
7
8
9
enum List {
    Cons(i32, List),
    Nil,
}

use crate::List::{Cons, Nil};
fn main() {
    let list = Cons(1, Cons(2, Cons(3, Nil))); // not allowed, infinite size
}
\n\n

use Box

\n
1
2
3
4
5
6
7
8
9
10
enum List {
    Cons(i32, Box<List>),
    Nil,
}

use crate::List::{Cons, Nil};

fn main() {
    let list = Cons(1, Box::new(Cons(2, Box::new(Cons(3, Box::new(Nil))))));
}
\n\n

Treating Smart Pointers Like Regular References with the Deref Trait

Implementing the Deref trait allows you to customize the behavior of the dereference operator, *
By implementing Deref in such a way that a smart pointer can be treated like a regular reference, you can write code that operates on references and use that code with smart pointers too.

\n

Defining Our Own Smart Pointer

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
use std::ops::Deref;

struct MyBox<T>(T);  // The MyBox type is a tuple struct with one element of type T

impl<T> MyBox<T> {
    fn new(x: T) -> MyBox<T> {
        MyBox(x)
    }
}

impl<T> Deref for MyBox<T> {
    type Target = T;

    fn deref(&self) -> &Self::Target {
        &self.0
    }
}

fn main() {
    let x = 5;
    let y = MyBox::new(x);

    assert_eq!(5, x);
    assert_eq!(5, *y);
}
\n
    \n
  • We fill in the body of the deref method with &self.0 so deref returns a reference to the value we want to access with the * operator.
    • \n
  • behind the scenes Rust actually ran this code: *(y.deref()). Rust substitutes the * operator with a call to the deref method
    • \n
  • The reason the deref method returns a reference to a value, and that the plain dereference outside the parentheses in *(y.deref()) is still necessary, is the ownership system. If the deref method returned the value directly instead of a reference to the value, the value would be moved out of self. We don’t want to take ownership of the inner value inside MyBox in this case or in most cases where we use the dereference operator.
    • \n
\n

Implicit Deref Coercions with Functions and Methods

Deref coercion is a convenience that Rust performs on arguments to functions and methods.
Deref coercion works only on types that implement the Deref trait. Deref coercion converts a reference to such a type into a reference to another type. For example, deref coercion can convert &String to &str because String implements the Deref trait such that it returns &str
A sequence of calls to the deref method converts the type we provided into the type the parameter needs.

\n
1
2
3
4
5
6
7
8
9
10
fn hello(name: &str) {
    println!("Hello, {}!", name);
}

fn main() {
    let m = MyBox::new(String::from("Rust"));
    hello("rust");  // ok
    hello(&m);      // also ok
    hello(&(*m)[..]); // without deref coercion. The (*m) dereferences the MyBox<String> into a String. Then the & and [..] take a string slice of the String that is equal to the whole string to match the signature of hello
}
\n

Here we’re calling the hello function with the argument &m, which is a reference to a MyBox value. Because we implemented the Deref trait on MyBox, Rust can turn &MyBox into &String by calling deref. The standard library provides an implementation of Deref on String that returns a string slice. Rust calls deref again to turn the &String into &str, which matches the hello function’s definition.

\n

How Deref Coercion Interacts with Mutability

Similar to how you use the Deref trait to override the * operator on immutable references, you can use the DerefMut trait to override the * operator on mutable references.

\n

Rust does deref coercion when it finds types and trait implementations in three cases:

\n
    \n
  • From &T to &U when T: Deref<Target=U>
    • \n
  • From &mut T to &mut U when T: DerefMut<Target=U>
    • \n
  • From &mut T to &U when T: Deref<Target=U>
    • \n
\n

Running Code on Cleanup with the Drop Trait

Drop, which lets you customize what happens when a value is about to go out of scope.

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
struct CustomSmartPointer {
    data: String,
}

impl Drop for CustomSmartPointer {
    fn drop(&mut self) {
        println!("Dropping CustomSmartPointer with data `{}`!", self.data);
    }
}

fn main() {
    let c = CustomSmartPointer {
        data: String::from("my stuff"),
    };
    c.drop(); // not allowed
    let d = CustomSmartPointer {
        data: String::from("other stuff"),
    };
    println!("CustomSmartPointers created.");
}

\n

Dropping a Value Early with std::mem::drop

std::mem::drop is in prelude, can use directly

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
struct CustomSmartPointer {
    data: String,
}

impl Drop for CustomSmartPointer {
    fn drop(&mut self) {
        println!("Dropping CustomSmartPointer with data `{}`!", self.data);
    }
}

fn main() {
    let c = CustomSmartPointer {
        data: String::from("some data"),
    };
    println!("CustomSmartPointer created.");
    drop(c); // ok
    println!("CustomSmartPointer dropped before the end of main.");
} // c goes out of scope, will occur double drop

\n\n\n

Rc: the Reference Counted Smart Pointer

In the majority of cases, ownership is clear: you know exactly which variable owns a given value. However, there are cases when a single value might have multiple owners.
type keeps track of the number of references to a value to determine whether or not the value is still in use. If there are zero references to a value, the value can be cleaned up without any references becoming invalid.

\n

We use the Rc<T> type when we want to allocate some data on the heap for multiple parts of our program to read and we can’t determine at compile time which part will finish using the data last.

\n

Rc is only for use in single-threaded scenarios

\n

using Rc to share data

\"rc\"
implement use box will not work, as below

\n
1
2
3
4
5
6
7
8
9
10
11
12
enum List {
    Cons(i32, Box<List>),
    Nil,
}

use crate::List::{Cons, Nil};

fn main() {
    let a = Cons(5, Box::new(Cons(10, Box::new(Nil))));
    let b = Cons(3, Box::new(a)); // value moved here
    let c = Cons(4, Box::new(a)); // value used here after move
}
\n

use Rc

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
enum List {
    Cons(i32, Rc<List>),
    Nil,
}

use crate::List::{Cons, Nil};
use std::rc::Rc;

fn main() {
    let a = Rc::new(Cons(5, Rc::new(Cons(10, Rc::new(Nil)))));
    let b = Cons(3, Rc::clone(&a)); // shalow copy only reference, not data
    let c = Cons(4, Rc::clone(&a));
}
\n\n\n

We could have called a.clone() rather than Rc::clone(&a), but Rust’s convention is to use Rc::clone in this case. The implementation of Rc::clone doesn’t make a deep copy of all the data like most types’ implementations of clone do. The call to Rc::clone only increments the reference count, which doesn’t take much time.

\n

Cloning an Rc Increases the Reference Count

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
enum List {
    Cons(i32, Rc<List>),
    Nil,
}

use crate::List::{Cons, Nil};
use std::rc::Rc;

fn main() {
    let a = Rc::new(Cons(5, Rc::new(Cons(10, Rc::new(Nil)))));
    println!("count after creating a = {}", Rc::strong_count(&a));  // 1
    let b = Cons(3, Rc::clone(&a));
    println!("count after creating b = {}", Rc::strong_count(&a));  // 2
    {
        let c = Cons(4, Rc::clone(&a));
        println!("count after creating c = {}", Rc::strong_count(&a)); // 3
    }
    println!("count after c goes out of scope = {}", Rc::strong_count(&a));  // 2
}
\n

What we can’t see in this example is that when b and then a go out of scope at the end of main, the count is then 0, and the Rc is cleaned up completely at that point.

\n

RefCell and interior mutability pattern

Interior mutability is a design pattern in Rust that allows you to mutate data even when there are immutable references to that data; normally, this action is disallowed by the borrowing rules.
To mutate data, the pattern uses unsafe code inside a data structure to bend Rust’s usual rules that govern mutation and borrowing.
We can use types that use the interior mutability pattern when we can ensure that the borrowing rules will be followed at runtime, even though the compiler can’t guarantee that.
The unsafe code involved is then wrapped in a safe API, and the outer type is still immutable.

\n

Enforcing Borrowing Rules at Runtime with RefCell

Unlike Rc, the RefCell type represents single ownership over the data it holds.

\n

recall borrowing rules

\n
    \n
  • At any given time, you can have either (but not both of) one mutable reference or any number of immutable references.
    • \n
  • References must always be valid.
    With references and Box, the borrowing rules’ invariants are enforced at compile time. With RefCell, these invariants are enforced at runtime.
    Because RefCell allows mutable borrows checked at runtime, you can mutate the value inside the RefCell even when the RefCell is immutable.
    • \n
\n

Interior Mutability: A Mutable Borrow to an Immutable Value

1
2
3
4
fn main() {
    let x = 5;
    let y = &mut x; // not allowed. cannot borrow `x` as mutable, as it is not declared as mutable
}
\n\n

A Use Case for Interior Mutability: Mock Objects

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
pub trait Messenger {
    fn send(&self, msg: &str);
}

pub struct LimitTracker<'a, T: Messenger> {
    messenger: &'a T,
    value: usize,
    max: usize,
}

impl<'a, T> LimitTracker<'a, T>
where
    T: Messenger,
{
    pub fn new(messenger: &T, max: usize) -> LimitTracker<T> {
        LimitTracker {
            messenger,
            value: 0,
            max,
        }
    }

    pub fn set_value(&mut self, value: usize) {
        self.value = value;

        let percentage_of_max = self.value as f64 / self.max as f64;

        if percentage_of_max >= 1.0 {
            self.messenger.send("Error: You are over your quota!");
        } else if percentage_of_max >= 0.9 {
            self.messenger
                .send("Urgent warning: You've used up over 90% of your quota!");
        } else if percentage_of_max >= 0.75 {
            self.messenger
                .send("Warning: You've used up over 75% of your quota!");
        }
    }
}

\n

a problematic usage

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
pub trait Messenger {
    fn send(&self, msg: &str);
}

pub struct LimitTracker<'a, T: Messenger> {
    messenger: &'a T,
    value: usize,
    max: usize,
}

impl<'a, T> LimitTracker<'a, T>
where
    T: Messenger,
{
    pub fn new(messenger: &T, max: usize) -> LimitTracker<T> {
        LimitTracker {
            messenger,
            value: 0,
            max,
        }
    }

    pub fn set_value(&mut self, value: usize) {
        self.value = value;

        let percentage_of_max = self.value as f64 / self.max as f64;

        if percentage_of_max >= 1.0 {
            self.messenger.send("Error: You are over your quota!");
        } else if percentage_of_max >= 0.9 {
            self.messenger
                .send("Urgent warning: You've used up over 90% of your quota!");
        } else if percentage_of_max >= 0.75 {
            self.messenger
                .send("Warning: You've used up over 75% of your quota!");
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    struct MockMessenger {
        sent_messages: Vec<String>,
    }

    impl MockMessenger {
        fn new() -> MockMessenger {
            MockMessenger {
                sent_messages: vec![],
            }
        }
    }

    impl Messenger for MockMessenger {
        fn send(&self, message: &str) {
            self.sent_messages.push(String::from(message)); // not allowed. cannot borrow `self.sent_messages` as mutable, as it is behind a `&` reference
        }
    }

    #[test]
    fn it_sends_an_over_75_percent_warning_message() {
        let mock_messenger = MockMessenger::new();
        let mut limit_tracker = LimitTracker::new(&mock_messenger, 100);

        limit_tracker.set_value(80);

        assert_eq!(mock_messenger.sent_messages.len(), 1);
    }
}

\n

We can’t modify the MockMessenger to keep track of the messages, because the send method takes an immutable reference to self. We also can’t take the suggestion from the error text to use &mut self instead, because then the signature of send wouldn’t match the signature in the Messenger trait definition

\n

This is a situation in which interior mutability can help!

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
pub trait Messenger {
    fn send(&self, msg: &str);
}

pub struct LimitTracker<'a, T: Messenger> {
    messenger: &'a T,
    value: usize,
    max: usize,
}

impl<'a, T> LimitTracker<'a, T>
where
    T: Messenger,
{
    pub fn new(messenger: &T, max: usize) -> LimitTracker<T> {
        LimitTracker {
            messenger,
            value: 0,
            max,
        }
    }

    pub fn set_value(&mut self, value: usize) {
        self.value = value;

        let percentage_of_max = self.value as f64 / self.max as f64;

        if percentage_of_max >= 1.0 {
            self.messenger.send("Error: You are over your quota!");
        } else if percentage_of_max >= 0.9 {
            self.messenger
                .send("Urgent warning: You've used up over 90% of your quota!");
        } else if percentage_of_max >= 0.75 {
            self.messenger
                .send("Warning: You've used up over 75% of your quota!");
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::cell::RefCell;

    struct MockMessenger {
        sent_messages: RefCell<Vec<String>>,
    }

    impl MockMessenger {
        fn new() -> MockMessenger {
            MockMessenger {
                sent_messages: RefCell::new(vec![]),
            }
        }
    }

    impl Messenger for MockMessenger {
        fn send(&self, message: &str) {
            self.sent_messages.borrow_mut().push(String::from(message)); // call borrow_mut on the RefCell<Vec<String>> in self.sent_messages to get a mutable reference to the value inside the RefCell<Vec<String>>
        }
    }

    #[test]
    fn it_sends_an_over_75_percent_warning_message() {
        // --snip--
        let mock_messenger = MockMessenger::new();
        let mut limit_tracker = LimitTracker::new(&mock_messenger, 100);

        limit_tracker.set_value(80);

        assert_eq!(mock_messenger.sent_messages.borrow().len(), 1); //  call borrow on the RefCell<Vec<String>> to get an immutable reference to the vector.
    }
}
\n\n

Keeping Track of Borrows at Runtime with RefCell

When creating immutable and mutable references, we use the & and &mut syntax, respectively. With RefCell, we use the borrow and borrow_mut methods, which are part of the safe API that belongs to RefCell. The borrow method returns the smart pointer type Ref, and borrow_mut returns the smart pointer type RefMut. Both types implement Deref, so we can treat them like regular references.

\n

The RefCell keeps track of how many Ref and RefMut smart pointers are currently active.RefCell lets us have many immutable borrows or one mutable borrow at any point in time. If we try to violate these rules, rather than getting a compiler error as we would with references, the implementation of RefCell will panic at runtime.

\n
1
2
3
4
5
6
7
8
9
impl Messenger for MockMessenger {
        fn send(&self, message: &str) {
            let mut one_borrow = self.sent_messages.borrow_mut();
            let mut two_borrow = self.sent_messages.borrow_mut();

            one_borrow.push(String::from(message));
            two_borrow.push(String::from(message));
        }
    }
\n

When we run the tests for our library, the code in will compile without any errors, but the test will fail
thread ‘main’ panicked at ‘already borrowed

\n

Having Multiple Owners of Mutable Data by Combining Rc and RefCell

A common way to use RefCell is in combination with Rc. If you have an Rc that holds a RefCell, you can get a value that can have multiple owners and that you can mutate!

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
#[derive(Debug)]
enum List {
    Cons(Rc<RefCell<i32>>, Rc<List>),
    Nil,
}

use crate::List::{Cons, Nil};
use std::cell::RefCell;
use std::rc::Rc;

fn main() {
    let value = Rc::new(RefCell::new(5)); // We create a value that is an instance of Rc<RefCell<i32>> and store it in a variable named value so we can access it directly later.

    let a = Rc::new(Cons(Rc::clone(&value), Rc::new(Nil))); //  Then we create a List in a with a Cons variant that holds value. We need to clone value so both a and value have ownership of the inner 5 value rather than transferring ownership from value to a or having a borrow from value. We wrap the list a in an Rc<T> so when we create lists b and c, they can both refer to a

    let b = Cons(Rc::new(RefCell::new(3)), Rc::clone(&a));
    let c = Cons(Rc::new(RefCell::new(4)), Rc::clone(&a));

    *value.borrow_mut() += 10; // After we’ve created the lists in a, b, and c, we add 10 to the value in value. We do this by calling borrow_mut on value, which uses the automatic dereferencing feature to dereference the Rc<T> to the inner RefCell<T> value. The borrow_mut method returns a RefMut<T> smart pointer, and we use the dereference operator on it and change the inner value.

    println!("a after = {:?}", a);
    println!("b after = {:?}", b);
    println!("c after = {:?}", c);
}
\n\n

The standard library has other types that provide interior mutability, such as Cell, which is similar except that instead of giving references to the inner value, the value is copied in and out of the Cell. There’s also Mutex, which offers interior mutability that’s safe to use across threads;

\n

Reference Cycles Can Leak Memory

Rust’s memory safety guarantees make it difficult, but not impossible, to accidentally create memory that is never cleaned up (known as a memory leak).

\n

Creating a Reference Cycle

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
use crate::List::{Cons, Nil};
use std::cell::RefCell;
use std::rc::Rc;

#[derive(Debug)]
enum List {
    Cons(i32, RefCell<Rc<List>>), // The second element in the Cons variant is now RefCell<Rc<List>>, meaning that we want to modify which List value a Cons variant is pointing to. 
    Nil,
}

impl List {
    fn tail(&self) -> Option<&RefCell<Rc<List>>> { // We’re also adding a tail method to make it convenient for us to access the second item if we have a Cons variant.
        match self {
            Cons(_, item) => Some(item),
            Nil => None,
        }
    }
}

fn main() {
    use crate::List::{Cons, Nil};
use std::cell::RefCell;
use std::rc::Rc;

#[derive(Debug)]
enum List {
    Cons(i32, RefCell<Rc<List>>),
    Nil,
}

impl List {
    fn tail(&self) -> Option<&RefCell<Rc<List>>> {
        match self {
            Cons(_, item) => Some(item),
            Nil => None,
        }
    }
}

fn main() {
    let a = Rc::new(Cons(5, RefCell::new(Rc::new(Nil))));

    println!("a initial rc count = {}", Rc::strong_count(&a));
    println!("a next item = {:?}", a.tail());

    let b = Rc::new(Cons(10, RefCell::new(Rc::clone(&a)))); // his code creates a list in a and a list in b that points to the list in a

    println!("a rc count after b creation = {}", Rc::strong_count(&a));
    println!("b initial rc count = {}", Rc::strong_count(&b));
    println!("b next item = {:?}", b.tail());

    if let Some(link) = a.tail() {
        *link.borrow_mut() = Rc::clone(&b); // modifies the list in a to point to b, creating a reference cycle
    }

    println!("b rc count after changing a = {}", Rc::strong_count(&b));
    println!("a rc count after changing a = {}", Rc::strong_count(&a));

    // Uncomment the next line to see that we have a cycle;
    // it will overflow the stack
    // println!("a next item = {:?}", a.tail());
} // At the end of main, Rust drops the variable b, which decreases the reference count of the Rc<List> instance from 2 to 1. The memory that Rc<List> has on the heap won’t be dropped at this point, because its reference count is 1, not 0. Then Rust drops a, which decreases the reference count of the a Rc<List> instance from 2 to 1 as well. This instance’s memory can’t be dropped either, because the other Rc<List> instance still refers to it. The memory allocated to the list will remain uncollected forever.
\n

\"cycle-ref\"

\n

Preventing Reference Cycles: Turning an Rc into a Weak

So far, we’ve demonstrated that calling Rc::clone increases the strong_count of an Rc instance, and an Rc instance is only cleaned up if its strong_count is 0. You can also create a weak reference to the value within an Rc instance by calling Rc::downgrade and passing a reference to the Rc. When you call Rc::downgrade, you get a smart pointer of type Weak. Instead of increasing the strong_count in the Rc instance by 1, calling Rc::downgrade increases the weak_count by 1. The Rc type uses weak_count to keep track of how many Weak references exist, similar to strong_count. The difference is the weak_count doesn’t need to be 0 for the Rc instance to be cleaned up.

\n

Strong references are how you can share ownership of an Rc instance. Weak references don’t express an ownership relationship. They won’t cause a reference cycle because any cycle involving some weak references will be broken once the strong reference count of values involved is 0.

\n

Because the value that Weak references might have been dropped, to do anything with the value that a Weak is pointing to, you must make sure the value still exists. Do this by calling the upgrade method on a Weak instance, which will return an Option<Rc>. You’ll get a result of Some if the Rc value has not been dropped yet and a result of None if the Rc value has been dropped.

\n

Creating a Tree Data Structure: a Node with Child Nodes

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
use std::cell::RefCell;
use std::rc::Rc;

#[derive(Debug)]
struct Node {
    value: i32,
    parent: RefCell<Weak<Node>>, // To make the child node aware of its parent, we need to add a parent field to our Node struct definition. The trouble is in deciding what the type of parent should be. We know it can’t contain an Rc<T>, because that would create a reference cycle with leaf.parent pointing to branch and branch.children pointing to leaf, which would cause their strong_count values to never be 0.
    children: RefCell<Vec<Rc<Node>>>,
}

fn main() {

    let leaf = Rc::new(Node {
        value: 3,
        parent: RefCell::new(Weak::new()),
        children: RefCell::new(vec![]),
    });

    println!("leaf parent = {:?}", leaf.parent.borrow().upgrade()); // try to get a reference to the parent of leaf by using the upgrade method, we get a None value. 

    let branch = Rc::new(Node {
        value: 5,
        parent: RefCell::new(Weak::new()),
        children: RefCell::new(vec![Rc::clone(&leaf)]), // We clone the Rc<Node> in leaf and store that in branch, meaning the Node in leaf now has two owners: leaf and branch. 
    });

    *leaf.parent.borrow_mut() = Rc::downgrade(&branch); // use the Rc::downgrade function to create a Weak<Node> reference to branch from the Rc<Node> in branch.

    println!("leaf parent = {:?}", leaf.parent.borrow().upgrade());

   
}
\n\n

Visualizing Changes to strong_count and weak_count

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
use std::cell::RefCell;
use std::rc::{Rc, Weak};

#[derive(Debug)]
struct Node {
    value: i32,
    parent: RefCell<Weak<Node>>,
    children: RefCell<Vec<Rc<Node>>>,
}

fn main() {
    let leaf = Rc::new(Node {
        value: 3,
        parent: RefCell::new(Weak::new()),
        children: RefCell::new(vec![]),
    });

    println!(
        "leaf strong = {}, weak = {}",
        Rc::strong_count(&leaf),
        Rc::weak_count(&leaf),
    );

    {
        let branch = Rc::new(Node {
            value: 5,
            parent: RefCell::new(Weak::new()),
            children: RefCell::new(vec![Rc::clone(&leaf)]),
        });

        *leaf.parent.borrow_mut() = Rc::downgrade(&branch);

        println!(
            "branch strong = {}, weak = {}",
            Rc::strong_count(&branch),
            Rc::weak_count(&branch),
        );

        println!(
            "leaf strong = {}, weak = {}",
            Rc::strong_count(&leaf),
            Rc::weak_count(&leaf),
        );
    }

    println!("leaf parent = {:?}", leaf.parent.borrow().upgrade());
    println!(
        "leaf strong = {}, weak = {}",
        Rc::strong_count(&leaf),
        Rc::weak_count(&leaf),
    );
}
","site":{"data":{}},"excerpt":"","more":"

Overview

The most common kind of pointer in Rust is a reference (borrow but not own)
Smart pointers, on the other hand, are data structures that not only act like a pointer but also have additional metadata and capabilities.
references are pointers that only borrow data; in contrast, in many cases, smart pointers own the data they point to.

\n

smart pointers example

    \n
  • String
    • \n
  • Vec
    Both these types count as smart pointers because they own some memory and allow you to manipulate it. They also have metadata (such as their capacity) and extra capabilities or guarantees (such as with String ensuring its data will always be valid UTF-8).
    • \n
\n

Deref & Drop

Smart pointers are usually implemented using structs. The characteristic that distinguishes a smart pointer from an ordinary struct is that smart pointers implement the Deref and Drop traits.

\n
    \n
  • The Deref trait allows an instance of the smart pointer struct to behave like a reference so you can write code that works with either references or smart pointers.
    • \n
  • The Drop trait allows you to customize the code that is run when an instance of the smart pointer goes out of scope.
    • \n
\n

the most common smart pointers in the standard library:

    \n
  • Box<T> for allocating values on the heap
    • \n
  • Rc<T>, a reference counting type that enables multiple ownership
    • \n
  • Ref<T> and RefMut<T>, accessed through RefCell<T>, a type that enforces the borrowing rules at runtime instead of compile time
    • \n
\n

Box

when to use

    \n
  • When you have a type whose size can’t be known at compile time and you want to use a value of that type in a context that requires an exact size. (such as cons)
    • \n
  • When you have a large amount of data and you want to transfer ownership but ensure the data won’t be copied when you do so
    • \n
  • When you want to own a value and you care only that it’s a type that implements a particular trait rather than being of a specific type
    • \n
\n

enabling recursive types with Box

At compile time, Rust needs to know how much space a type takes up
One type whose size can’t be known at compile time is a recursive type (cons list), use Box, which only contains a memory address

\n
1
2
3
4
5
6
7
8
9
enum List {
    Cons(i32, List),
    Nil,
}

use crate::List::{Cons, Nil};
fn main() {
    let list = Cons(1, Cons(2, Cons(3, Nil))); // not allowed, infinite size
}
\n\n

use Box

\n
1
2
3
4
5
6
7
8
9
10
enum List {
    Cons(i32, Box<List>),
    Nil,
}

use crate::List::{Cons, Nil};

fn main() {
    let list = Cons(1, Box::new(Cons(2, Box::new(Cons(3, Box::new(Nil))))));
}
\n\n

Treating Smart Pointers Like Regular References with the Deref Trait

Implementing the Deref trait allows you to customize the behavior of the dereference operator, *
By implementing Deref in such a way that a smart pointer can be treated like a regular reference, you can write code that operates on references and use that code with smart pointers too.

\n

Defining Our Own Smart Pointer

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
use std::ops::Deref;

struct MyBox<T>(T);  // The MyBox type is a tuple struct with one element of type T

impl<T> MyBox<T> {
    fn new(x: T) -> MyBox<T> {
        MyBox(x)
    }
}

impl<T> Deref for MyBox<T> {
    type Target = T;

    fn deref(&self) -> &Self::Target {
        &self.0
    }
}

fn main() {
    let x = 5;
    let y = MyBox::new(x);

    assert_eq!(5, x);
    assert_eq!(5, *y);
}
\n
    \n
  • We fill in the body of the deref method with &self.0 so deref returns a reference to the value we want to access with the * operator.
    • \n
  • behind the scenes Rust actually ran this code: *(y.deref()). Rust substitutes the * operator with a call to the deref method
    • \n
  • The reason the deref method returns a reference to a value, and that the plain dereference outside the parentheses in *(y.deref()) is still necessary, is the ownership system. If the deref method returned the value directly instead of a reference to the value, the value would be moved out of self. We don’t want to take ownership of the inner value inside MyBox in this case or in most cases where we use the dereference operator.
    • \n
\n

Implicit Deref Coercions with Functions and Methods

Deref coercion is a convenience that Rust performs on arguments to functions and methods.
Deref coercion works only on types that implement the Deref trait. Deref coercion converts a reference to such a type into a reference to another type. For example, deref coercion can convert &String to &str because String implements the Deref trait such that it returns &str
A sequence of calls to the deref method converts the type we provided into the type the parameter needs.

\n
1
2
3
4
5
6
7
8
9
10
fn hello(name: &str) {
    println!("Hello, {}!", name);
}

fn main() {
    let m = MyBox::new(String::from("Rust"));
    hello("rust");  // ok
    hello(&m);      // also ok
    hello(&(*m)[..]); // without deref coercion. The (*m) dereferences the MyBox<String> into a String. Then the & and [..] take a string slice of the String that is equal to the whole string to match the signature of hello
}
\n

Here we’re calling the hello function with the argument &m, which is a reference to a MyBox value. Because we implemented the Deref trait on MyBox, Rust can turn &MyBox into &String by calling deref. The standard library provides an implementation of Deref on String that returns a string slice. Rust calls deref again to turn the &String into &str, which matches the hello function’s definition.

\n

How Deref Coercion Interacts with Mutability

Similar to how you use the Deref trait to override the * operator on immutable references, you can use the DerefMut trait to override the * operator on mutable references.

\n

Rust does deref coercion when it finds types and trait implementations in three cases:

\n
    \n
  • From &T to &U when T: Deref<Target=U>
    • \n
  • From &mut T to &mut U when T: DerefMut<Target=U>
    • \n
  • From &mut T to &U when T: Deref<Target=U>
    • \n
\n

Running Code on Cleanup with the Drop Trait

Drop, which lets you customize what happens when a value is about to go out of scope.

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
struct CustomSmartPointer {
    data: String,
}

impl Drop for CustomSmartPointer {
    fn drop(&mut self) {
        println!("Dropping CustomSmartPointer with data `{}`!", self.data);
    }
}

fn main() {
    let c = CustomSmartPointer {
        data: String::from("my stuff"),
    };
    c.drop(); // not allowed
    let d = CustomSmartPointer {
        data: String::from("other stuff"),
    };
    println!("CustomSmartPointers created.");
}

\n

Dropping a Value Early with std::mem::drop

std::mem::drop is in prelude, can use directly

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
struct CustomSmartPointer {
    data: String,
}

impl Drop for CustomSmartPointer {
    fn drop(&mut self) {
        println!("Dropping CustomSmartPointer with data `{}`!", self.data);
    }
}

fn main() {
    let c = CustomSmartPointer {
        data: String::from("some data"),
    };
    println!("CustomSmartPointer created.");
    drop(c); // ok
    println!("CustomSmartPointer dropped before the end of main.");
} // c goes out of scope, will occur double drop

\n\n\n

Rc: the Reference Counted Smart Pointer

In the majority of cases, ownership is clear: you know exactly which variable owns a given value. However, there are cases when a single value might have multiple owners.
type keeps track of the number of references to a value to determine whether or not the value is still in use. If there are zero references to a value, the value can be cleaned up without any references becoming invalid.

\n

We use the Rc<T> type when we want to allocate some data on the heap for multiple parts of our program to read and we can’t determine at compile time which part will finish using the data last.

\n

Rc is only for use in single-threaded scenarios

\n

using Rc to share data

\"rc\"
implement use box will not work, as below

\n
1
2
3
4
5
6
7
8
9
10
11
12
enum List {
    Cons(i32, Box<List>),
    Nil,
}

use crate::List::{Cons, Nil};

fn main() {
    let a = Cons(5, Box::new(Cons(10, Box::new(Nil))));
    let b = Cons(3, Box::new(a)); // value moved here
    let c = Cons(4, Box::new(a)); // value used here after move
}
\n

use Rc

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
enum List {
    Cons(i32, Rc<List>),
    Nil,
}

use crate::List::{Cons, Nil};
use std::rc::Rc;

fn main() {
    let a = Rc::new(Cons(5, Rc::new(Cons(10, Rc::new(Nil)))));
    let b = Cons(3, Rc::clone(&a)); // shalow copy only reference, not data
    let c = Cons(4, Rc::clone(&a));
}
\n\n\n

We could have called a.clone() rather than Rc::clone(&a), but Rust’s convention is to use Rc::clone in this case. The implementation of Rc::clone doesn’t make a deep copy of all the data like most types’ implementations of clone do. The call to Rc::clone only increments the reference count, which doesn’t take much time.

\n

Cloning an Rc Increases the Reference Count

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
enum List {
    Cons(i32, Rc<List>),
    Nil,
}

use crate::List::{Cons, Nil};
use std::rc::Rc;

fn main() {
    let a = Rc::new(Cons(5, Rc::new(Cons(10, Rc::new(Nil)))));
    println!("count after creating a = {}", Rc::strong_count(&a));  // 1
    let b = Cons(3, Rc::clone(&a));
    println!("count after creating b = {}", Rc::strong_count(&a));  // 2
    {
        let c = Cons(4, Rc::clone(&a));
        println!("count after creating c = {}", Rc::strong_count(&a)); // 3
    }
    println!("count after c goes out of scope = {}", Rc::strong_count(&a));  // 2
}
\n

What we can’t see in this example is that when b and then a go out of scope at the end of main, the count is then 0, and the Rc is cleaned up completely at that point.

\n

RefCell and interior mutability pattern

Interior mutability is a design pattern in Rust that allows you to mutate data even when there are immutable references to that data; normally, this action is disallowed by the borrowing rules.
To mutate data, the pattern uses unsafe code inside a data structure to bend Rust’s usual rules that govern mutation and borrowing.
We can use types that use the interior mutability pattern when we can ensure that the borrowing rules will be followed at runtime, even though the compiler can’t guarantee that.
The unsafe code involved is then wrapped in a safe API, and the outer type is still immutable.

\n

Enforcing Borrowing Rules at Runtime with RefCell

Unlike Rc, the RefCell type represents single ownership over the data it holds.

\n

recall borrowing rules

\n
    \n
  • At any given time, you can have either (but not both of) one mutable reference or any number of immutable references.
    • \n
  • References must always be valid.
    With references and Box, the borrowing rules’ invariants are enforced at compile time. With RefCell, these invariants are enforced at runtime.
    Because RefCell allows mutable borrows checked at runtime, you can mutate the value inside the RefCell even when the RefCell is immutable.
    • \n
\n

Interior Mutability: A Mutable Borrow to an Immutable Value

1
2
3
4
fn main() {
    let x = 5;
    let y = &mut x; // not allowed. cannot borrow `x` as mutable, as it is not declared as mutable
}
\n\n

A Use Case for Interior Mutability: Mock Objects

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
pub trait Messenger {
    fn send(&self, msg: &str);
}

pub struct LimitTracker<'a, T: Messenger> {
    messenger: &'a T,
    value: usize,
    max: usize,
}

impl<'a, T> LimitTracker<'a, T>
where
    T: Messenger,
{
    pub fn new(messenger: &T, max: usize) -> LimitTracker<T> {
        LimitTracker {
            messenger,
            value: 0,
            max,
        }
    }

    pub fn set_value(&mut self, value: usize) {
        self.value = value;

        let percentage_of_max = self.value as f64 / self.max as f64;

        if percentage_of_max >= 1.0 {
            self.messenger.send("Error: You are over your quota!");
        } else if percentage_of_max >= 0.9 {
            self.messenger
                .send("Urgent warning: You've used up over 90% of your quota!");
        } else if percentage_of_max >= 0.75 {
            self.messenger
                .send("Warning: You've used up over 75% of your quota!");
        }
    }
}

\n

a problematic usage

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
pub trait Messenger {
    fn send(&self, msg: &str);
}

pub struct LimitTracker<'a, T: Messenger> {
    messenger: &'a T,
    value: usize,
    max: usize,
}

impl<'a, T> LimitTracker<'a, T>
where
    T: Messenger,
{
    pub fn new(messenger: &T, max: usize) -> LimitTracker<T> {
        LimitTracker {
            messenger,
            value: 0,
            max,
        }
    }

    pub fn set_value(&mut self, value: usize) {
        self.value = value;

        let percentage_of_max = self.value as f64 / self.max as f64;

        if percentage_of_max >= 1.0 {
            self.messenger.send("Error: You are over your quota!");
        } else if percentage_of_max >= 0.9 {
            self.messenger
                .send("Urgent warning: You've used up over 90% of your quota!");
        } else if percentage_of_max >= 0.75 {
            self.messenger
                .send("Warning: You've used up over 75% of your quota!");
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    struct MockMessenger {
        sent_messages: Vec<String>,
    }

    impl MockMessenger {
        fn new() -> MockMessenger {
            MockMessenger {
                sent_messages: vec![],
            }
        }
    }

    impl Messenger for MockMessenger {
        fn send(&self, message: &str) {
            self.sent_messages.push(String::from(message)); // not allowed. cannot borrow `self.sent_messages` as mutable, as it is behind a `&` reference
        }
    }

    #[test]
    fn it_sends_an_over_75_percent_warning_message() {
        let mock_messenger = MockMessenger::new();
        let mut limit_tracker = LimitTracker::new(&mock_messenger, 100);

        limit_tracker.set_value(80);

        assert_eq!(mock_messenger.sent_messages.len(), 1);
    }
}

\n

We can’t modify the MockMessenger to keep track of the messages, because the send method takes an immutable reference to self. We also can’t take the suggestion from the error text to use &mut self instead, because then the signature of send wouldn’t match the signature in the Messenger trait definition

\n

This is a situation in which interior mutability can help!

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
pub trait Messenger {
    fn send(&self, msg: &str);
}

pub struct LimitTracker<'a, T: Messenger> {
    messenger: &'a T,
    value: usize,
    max: usize,
}

impl<'a, T> LimitTracker<'a, T>
where
    T: Messenger,
{
    pub fn new(messenger: &T, max: usize) -> LimitTracker<T> {
        LimitTracker {
            messenger,
            value: 0,
            max,
        }
    }

    pub fn set_value(&mut self, value: usize) {
        self.value = value;

        let percentage_of_max = self.value as f64 / self.max as f64;

        if percentage_of_max >= 1.0 {
            self.messenger.send("Error: You are over your quota!");
        } else if percentage_of_max >= 0.9 {
            self.messenger
                .send("Urgent warning: You've used up over 90% of your quota!");
        } else if percentage_of_max >= 0.75 {
            self.messenger
                .send("Warning: You've used up over 75% of your quota!");
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::cell::RefCell;

    struct MockMessenger {
        sent_messages: RefCell<Vec<String>>,
    }

    impl MockMessenger {
        fn new() -> MockMessenger {
            MockMessenger {
                sent_messages: RefCell::new(vec![]),
            }
        }
    }

    impl Messenger for MockMessenger {
        fn send(&self, message: &str) {
            self.sent_messages.borrow_mut().push(String::from(message)); // call borrow_mut on the RefCell<Vec<String>> in self.sent_messages to get a mutable reference to the value inside the RefCell<Vec<String>>
        }
    }

    #[test]
    fn it_sends_an_over_75_percent_warning_message() {
        // --snip--
        let mock_messenger = MockMessenger::new();
        let mut limit_tracker = LimitTracker::new(&mock_messenger, 100);

        limit_tracker.set_value(80);

        assert_eq!(mock_messenger.sent_messages.borrow().len(), 1); //  call borrow on the RefCell<Vec<String>> to get an immutable reference to the vector.
    }
}
\n\n

Keeping Track of Borrows at Runtime with RefCell

When creating immutable and mutable references, we use the & and &mut syntax, respectively. With RefCell, we use the borrow and borrow_mut methods, which are part of the safe API that belongs to RefCell. The borrow method returns the smart pointer type Ref, and borrow_mut returns the smart pointer type RefMut. Both types implement Deref, so we can treat them like regular references.

\n

The RefCell keeps track of how many Ref and RefMut smart pointers are currently active.RefCell lets us have many immutable borrows or one mutable borrow at any point in time. If we try to violate these rules, rather than getting a compiler error as we would with references, the implementation of RefCell will panic at runtime.

\n
1
2
3
4
5
6
7
8
9
impl Messenger for MockMessenger {
        fn send(&self, message: &str) {
            let mut one_borrow = self.sent_messages.borrow_mut();
            let mut two_borrow = self.sent_messages.borrow_mut();

            one_borrow.push(String::from(message));
            two_borrow.push(String::from(message));
        }
    }
\n

When we run the tests for our library, the code in will compile without any errors, but the test will fail
thread ‘main’ panicked at ‘already borrowed

\n

Having Multiple Owners of Mutable Data by Combining Rc and RefCell

A common way to use RefCell is in combination with Rc. If you have an Rc that holds a RefCell, you can get a value that can have multiple owners and that you can mutate!

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
#[derive(Debug)]
enum List {
    Cons(Rc<RefCell<i32>>, Rc<List>),
    Nil,
}

use crate::List::{Cons, Nil};
use std::cell::RefCell;
use std::rc::Rc;

fn main() {
    let value = Rc::new(RefCell::new(5)); // We create a value that is an instance of Rc<RefCell<i32>> and store it in a variable named value so we can access it directly later.

    let a = Rc::new(Cons(Rc::clone(&value), Rc::new(Nil))); //  Then we create a List in a with a Cons variant that holds value. We need to clone value so both a and value have ownership of the inner 5 value rather than transferring ownership from value to a or having a borrow from value. We wrap the list a in an Rc<T> so when we create lists b and c, they can both refer to a

    let b = Cons(Rc::new(RefCell::new(3)), Rc::clone(&a));
    let c = Cons(Rc::new(RefCell::new(4)), Rc::clone(&a));

    *value.borrow_mut() += 10; // After we’ve created the lists in a, b, and c, we add 10 to the value in value. We do this by calling borrow_mut on value, which uses the automatic dereferencing feature to dereference the Rc<T> to the inner RefCell<T> value. The borrow_mut method returns a RefMut<T> smart pointer, and we use the dereference operator on it and change the inner value.

    println!("a after = {:?}", a);
    println!("b after = {:?}", b);
    println!("c after = {:?}", c);
}
\n\n

The standard library has other types that provide interior mutability, such as Cell, which is similar except that instead of giving references to the inner value, the value is copied in and out of the Cell. There’s also Mutex, which offers interior mutability that’s safe to use across threads;

\n

Reference Cycles Can Leak Memory

Rust’s memory safety guarantees make it difficult, but not impossible, to accidentally create memory that is never cleaned up (known as a memory leak).

\n

Creating a Reference Cycle

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
use crate::List::{Cons, Nil};
use std::cell::RefCell;
use std::rc::Rc;

#[derive(Debug)]
enum List {
    Cons(i32, RefCell<Rc<List>>), // The second element in the Cons variant is now RefCell<Rc<List>>, meaning that we want to modify which List value a Cons variant is pointing to. 
    Nil,
}

impl List {
    fn tail(&self) -> Option<&RefCell<Rc<List>>> { // We’re also adding a tail method to make it convenient for us to access the second item if we have a Cons variant.
        match self {
            Cons(_, item) => Some(item),
            Nil => None,
        }
    }
}

fn main() {
    use crate::List::{Cons, Nil};
use std::cell::RefCell;
use std::rc::Rc;

#[derive(Debug)]
enum List {
    Cons(i32, RefCell<Rc<List>>),
    Nil,
}

impl List {
    fn tail(&self) -> Option<&RefCell<Rc<List>>> {
        match self {
            Cons(_, item) => Some(item),
            Nil => None,
        }
    }
}

fn main() {
    let a = Rc::new(Cons(5, RefCell::new(Rc::new(Nil))));

    println!("a initial rc count = {}", Rc::strong_count(&a));
    println!("a next item = {:?}", a.tail());

    let b = Rc::new(Cons(10, RefCell::new(Rc::clone(&a)))); // his code creates a list in a and a list in b that points to the list in a

    println!("a rc count after b creation = {}", Rc::strong_count(&a));
    println!("b initial rc count = {}", Rc::strong_count(&b));
    println!("b next item = {:?}", b.tail());

    if let Some(link) = a.tail() {
        *link.borrow_mut() = Rc::clone(&b); // modifies the list in a to point to b, creating a reference cycle
    }

    println!("b rc count after changing a = {}", Rc::strong_count(&b));
    println!("a rc count after changing a = {}", Rc::strong_count(&a));

    // Uncomment the next line to see that we have a cycle;
    // it will overflow the stack
    // println!("a next item = {:?}", a.tail());
} // At the end of main, Rust drops the variable b, which decreases the reference count of the Rc<List> instance from 2 to 1. The memory that Rc<List> has on the heap won’t be dropped at this point, because its reference count is 1, not 0. Then Rust drops a, which decreases the reference count of the a Rc<List> instance from 2 to 1 as well. This instance’s memory can’t be dropped either, because the other Rc<List> instance still refers to it. The memory allocated to the list will remain uncollected forever.
\n

\"cycle-ref\"

\n

Preventing Reference Cycles: Turning an Rc into a Weak

So far, we’ve demonstrated that calling Rc::clone increases the strong_count of an Rc instance, and an Rc instance is only cleaned up if its strong_count is 0. You can also create a weak reference to the value within an Rc instance by calling Rc::downgrade and passing a reference to the Rc. When you call Rc::downgrade, you get a smart pointer of type Weak. Instead of increasing the strong_count in the Rc instance by 1, calling Rc::downgrade increases the weak_count by 1. The Rc type uses weak_count to keep track of how many Weak references exist, similar to strong_count. The difference is the weak_count doesn’t need to be 0 for the Rc instance to be cleaned up.

\n

Strong references are how you can share ownership of an Rc instance. Weak references don’t express an ownership relationship. They won’t cause a reference cycle because any cycle involving some weak references will be broken once the strong reference count of values involved is 0.

\n

Because the value that Weak references might have been dropped, to do anything with the value that a Weak is pointing to, you must make sure the value still exists. Do this by calling the upgrade method on a Weak instance, which will return an Option<Rc>. You’ll get a result of Some if the Rc value has not been dropped yet and a result of None if the Rc value has been dropped.

\n

Creating a Tree Data Structure: a Node with Child Nodes

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
use std::cell::RefCell;
use std::rc::Rc;

#[derive(Debug)]
struct Node {
    value: i32,
    parent: RefCell<Weak<Node>>, // To make the child node aware of its parent, we need to add a parent field to our Node struct definition. The trouble is in deciding what the type of parent should be. We know it can’t contain an Rc<T>, because that would create a reference cycle with leaf.parent pointing to branch and branch.children pointing to leaf, which would cause their strong_count values to never be 0.
    children: RefCell<Vec<Rc<Node>>>,
}

fn main() {

    let leaf = Rc::new(Node {
        value: 3,
        parent: RefCell::new(Weak::new()),
        children: RefCell::new(vec![]),
    });

    println!("leaf parent = {:?}", leaf.parent.borrow().upgrade()); // try to get a reference to the parent of leaf by using the upgrade method, we get a None value. 

    let branch = Rc::new(Node {
        value: 5,
        parent: RefCell::new(Weak::new()),
        children: RefCell::new(vec![Rc::clone(&leaf)]), // We clone the Rc<Node> in leaf and store that in branch, meaning the Node in leaf now has two owners: leaf and branch. 
    });

    *leaf.parent.borrow_mut() = Rc::downgrade(&branch); // use the Rc::downgrade function to create a Weak<Node> reference to branch from the Rc<Node> in branch.

    println!("leaf parent = {:?}", leaf.parent.borrow().upgrade());

   
}
\n\n

Visualizing Changes to strong_count and weak_count

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
use std::cell::RefCell;
use std::rc::{Rc, Weak};

#[derive(Debug)]
struct Node {
    value: i32,
    parent: RefCell<Weak<Node>>,
    children: RefCell<Vec<Rc<Node>>>,
}

fn main() {
    let leaf = Rc::new(Node {
        value: 3,
        parent: RefCell::new(Weak::new()),
        children: RefCell::new(vec![]),
    });

    println!(
        "leaf strong = {}, weak = {}",
        Rc::strong_count(&leaf),
        Rc::weak_count(&leaf),
    );

    {
        let branch = Rc::new(Node {
            value: 5,
            parent: RefCell::new(Weak::new()),
            children: RefCell::new(vec![Rc::clone(&leaf)]),
        });

        *leaf.parent.borrow_mut() = Rc::downgrade(&branch);

        println!(
            "branch strong = {}, weak = {}",
            Rc::strong_count(&branch),
            Rc::weak_count(&branch),
        );

        println!(
            "leaf strong = {}, weak = {}",
            Rc::strong_count(&leaf),
            Rc::weak_count(&leaf),
        );
    }

    println!("leaf parent = {:?}", leaf.parent.borrow().upgrade());
    println!(
        "leaf strong = {}, weak = {}",
        Rc::strong_count(&leaf),
        Rc::weak_count(&leaf),
    );
}
"},{"title":"rust memory layout","date":"2022-10-25T02:54:13.000Z","_content":"\n\n## trait object\na reference to a trait type (or Box) is called trait object\n\n### two ways convert concrete type to trait object\n1. assigning to variable\n```rust\nuse std::io::Write;\n\nlet mut buffer: Vec = vec![];\nlet w: &mut dyn Write = &mut buffer;\n```\n2. pass a concrete type as a argument to a function\n```rust\nfn main() {\n let mut buffer: Vec = vec![];\n writer(&mut buffer);\n}\n\nfn writer(w: &mut dyn Write) {\n // ...\n}\n```\nin both ways, buffer is converted to a trait object implements Write\n\nin memory, a trait object (in the example, w) is a fat point consists two pointers\n![trait_object_memory](/images/rust/memory/trait_object_memory.png)\nthe vtable is generated once at compile time and shared by all objects of the same type. the vtable contains pointers to the machine code of the functions that must be present for a type to be a \"Writer\"\n\n\n## references\n- https://www.youtube.com/watch?v=rDoqT-a6UFg","source":"_posts/rust/rust-05-memory-layout.md","raw":"---\ntitle: rust memory layout\ndate: 2022-10-25 10:54:13\ntags: [rust]\n---\n\n\n## trait object\na reference to a trait type (or Box) is called trait object\n\n### two ways convert concrete type to trait object\n1. assigning to variable\n```rust\nuse std::io::Write;\n\nlet mut buffer: Vec = vec![];\nlet w: &mut dyn Write = &mut buffer;\n```\n2. pass a concrete type as a argument to a function\n```rust\nfn main() {\n let mut buffer: Vec = vec![];\n writer(&mut buffer);\n}\n\nfn writer(w: &mut dyn Write) {\n // ...\n}\n```\nin both ways, buffer is converted to a trait object implements Write\n\nin memory, a trait object (in the example, w) is a fat point consists two pointers\n![trait_object_memory](/images/rust/memory/trait_object_memory.png)\nthe vtable is generated once at compile time and shared by all objects of the same type. the vtable contains pointers to the machine code of the functions that must be present for a type to be a \"Writer\"\n\n\n## references\n- https://www.youtube.com/watch?v=rDoqT-a6UFg","slug":"rust/rust-05-memory-layout","published":1,"updated":"2023-11-05T04:21:13.734Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8ds0015qwsjh6j437e0","content":"

trait object

a reference to a trait type (or Box) is called trait object

\n

two ways convert concrete type to trait object

    \n
  1. assigning to variable
    1
    2
    3
    4
    use std::io::Write;

    let mut buffer: Vec<u8> = vec![];
    let w: &mut dyn Write = &mut buffer;
    2. \n
  2. pass a concrete type as a argument to a function
    1
    2
    3
    4
    5
    6
    7
    8
    fn main() {
       let mut buffer: Vec<u8> = vec![];
       writer(&mut buffer);
    }

    fn writer(w: &mut dyn Write) {
        // ...
    }
    \nin both ways, buffer is converted to a trait object implements Write
    3. \n
\n

in memory, a trait object (in the example, w) is a fat point consists two pointers
\"trait_object_memory\"
the vtable is generated once at compile time and shared by all objects of the same type. the vtable contains pointers to the machine code of the functions that must be present for a type to be a “Writer”

\n

references

\n","site":{"data":{}},"excerpt":"","more":"

trait object

a reference to a trait type (or Box) is called trait object

\n

two ways convert concrete type to trait object

    \n
  1. assigning to variable
    1
    2
    3
    4
    use std::io::Write;

    let mut buffer: Vec<u8> = vec![];
    let w: &mut dyn Write = &mut buffer;
    2. \n
  2. pass a concrete type as a argument to a function
    1
    2
    3
    4
    5
    6
    7
    8
    fn main() {
       let mut buffer: Vec<u8> = vec![];
       writer(&mut buffer);
    }

    fn writer(w: &mut dyn Write) {
        // ...
    }
    \nin both ways, buffer is converted to a trait object implements Write
    3. \n
\n

in memory, a trait object (in the example, w) is a fat point consists two pointers
\"trait_object_memory\"
the vtable is generated once at compile time and shared by all objects of the same type. the vtable contains pointers to the machine code of the functions that must be present for a type to be a “Writer”

\n

references

\n"},{"title":"rust reflect and macro","date":"2022-12-28T02:24:21.000Z","_content":"\n## reflect\n### trait object\nRust provides dynamic dispatch through a feature called `trait objects`. Trait objects, like &Foo or Box, are normal values that store a value of any type that implements the given trait, where the precise type can only be known at **runtime**. more details can be found on [references 1]\n\n### any\nThis module (std::any) contains the `Any` trait, which enables dynamic typing of any 'static type through runtime reflection. It also contains the `Provider` trait and accompanying API, which enable trait objects to provide data based on typed requests, an alternate form of runtime reflection.\nAny itself can be used to get a TypeId\n\n```rust\nuse std::fmt::Debug;\nuse std::any::Any;\n\nfn log(value: &Any) {\n let value_any = value as &dyn Any; // &dyn Any (a borrowed trait object), Note that &dyn Any is limited to testing whether a value is of a specified concrete type, and cannot be used to test whether a type implements a trait.\n\n match value_any.downcast_ref::() {\n Some(val_str) -> {\n // do with string\n },\n None => {\n // \n }\n }\n}\n```\n\n### porpular crates using Any\n- [oso](https://docs.osohq.com/)\nThe Oso Library is a batteries-included framework for building authorization in your application\n- [bevy](https://bevyengine.org/)\nA refreshingly simple data-driven game engine built in Rust\n\n## macro\n### rust compile process\n![rust compile process](/images/rust/macros/16.compile_process.png)\n\n### front end: rustc\n1. lexical analysis: Code Text -> TokenStream\n2. syntax analysis: TokenStream -> AST (abstract syntax tree)\n3. semantic analyzer: \n AST -> HIR (High-Level Intermediate Representation) -> Type HIR (static type analysis, syntactic desugar, e.g `for` changed to `loop`) -> MIR: (Mid-Level Intermediate Representation, scope, reference & borrow check)\n\n### back end: LLVM\nLLVM IR -> machine code\n\n\n### macros in compiling\n- declarative macros: TokenStream - expand -> TokenStream\n- procedule macros: self defined AST with the help or third party crate such as syn, quote\n\n## declarative macro: macro_rules!\nDeclarative macros allow us to write match-like code. The match expression is a control structure that receives an expression and then matches the result of the expression with multiple patterns. Once a pattern is matched, the code associated with the pattern will be executed\n```rust\n#![allow(unused)]\nfn main() {\n match target {\n match_pattern_1 => expr_1,\n match_pattern_2 => {\n statement1;\n statement2;\n expr_2\n },\n _ => expr_3\n }\n}\n```\n\n### example 1, simplified `vec!`\nbelow example use macro_rules to implement a simplified version of vec!\n```rust\n#[macro_export]\nmacro_rules! vec {\n ( $( $x:expr ),* ) => {\n {\n let mut temp_vec = Vec::new();\n $(\n temp_vec.push($x);\n )*\n temp_vec\n }\n };\n}\n\n#![allow(unused)]\nfn main() {\n let v: Vec = vec![1, 2, 3];\n}\n```\n\n### example 2, unless\n```rust\nmacro_rules! unless {\n ( ($arg:expr) => $branch:tt ) => ( if !$arg {$branch};);\n}\n\nfn cmp(a:i32, b:i32) {\n unless!{\n (a>b) => {println!(\"{} < {}\", a,b);}\n }\n}\nfn main() {\n cmp(1,2); /// print \"1<2\" as the condition is true !(a>b)\n cmp(3,2); /// print nothing\n}\n```\n### example 3, HashMap\n```rust\nmacro_rules! hashmap {\n // match for \"a\" => 1, \"b\" => 2,\n ( $($key:expr => $value:expr,)* ) =>\n { hashmap!($($key => $value),*) }; // recuisive\n // match for \"a\" => 1, \"b\" => 2\n ( $($key:expr => $value:expr),* ) => { \n {\n let mut _map = ::std::collections::HashMap::new();\n $(\n _map.insert($key, $value);\n )*\n _map\n }\n \n };\n}\n\nmacro_rules! hashmap_equivalent {\n ( $($key:expr => $value:expr),* $(,)*) => { \n {\n let mut _map = ::std::collections::HashMap::new();\n $(\n _map.insert($key, $value);\n )*\n _map\n }\n \n };\n}\nfn main() {\n let map = hashmap!{\n \"a\" => 1,\n \"b\" => 2, // with or without ,\n };\n let map_2 = hashmap_equivalent!{\n \"a\" => 1, \n \"b\" => 2, // with or without ,\n };\n}\n```\n\n### metavariables\n- item: an Item\n- stmt: a Statement without the trailing semicolon (except for item statements that require semicolons)\n- expr: an Expression\n- ty: a Type\n- ident: an IDENTIFIER_OR_KEYWORD or RAW_IDENTIFIER\n- path: a TypePath style path\n- tt: a TokenTree (a single token or tokens in matching delimiters (), [], or {})\n- meta: an Attr, the contents of an attribute\n- lifetime: a LIFETIME_TOKEN\n- vis: a possibly empty Visibility qualifier\n- literal: matches LiteralExpression\ndetails to be found [here](https://doc.rust-lang.org/reference/macros-by-example.html)\n\n## procedures macro\n\n## references\n1. [rust trait object](https://web.mit.edu/rust-lang_v1.25/arch/amd64_ubuntu1404/share/doc/rust/html/book/first-edition/trait-objects.html)","source":"_posts/rust/rust-07-macro.md","raw":"---\ntitle: rust reflect and macro\ndate: 2022-12-28 10:24:21\ntags: [rust]\n---\n\n## reflect\n### trait object\nRust provides dynamic dispatch through a feature called `trait objects`. Trait objects, like &Foo or Box, are normal values that store a value of any type that implements the given trait, where the precise type can only be known at **runtime**. more details can be found on [references 1]\n\n### any\nThis module (std::any) contains the `Any` trait, which enables dynamic typing of any 'static type through runtime reflection. It also contains the `Provider` trait and accompanying API, which enable trait objects to provide data based on typed requests, an alternate form of runtime reflection.\nAny itself can be used to get a TypeId\n\n```rust\nuse std::fmt::Debug;\nuse std::any::Any;\n\nfn log(value: &Any) {\n let value_any = value as &dyn Any; // &dyn Any (a borrowed trait object), Note that &dyn Any is limited to testing whether a value is of a specified concrete type, and cannot be used to test whether a type implements a trait.\n\n match value_any.downcast_ref::() {\n Some(val_str) -> {\n // do with string\n },\n None => {\n // \n }\n }\n}\n```\n\n### porpular crates using Any\n- [oso](https://docs.osohq.com/)\nThe Oso Library is a batteries-included framework for building authorization in your application\n- [bevy](https://bevyengine.org/)\nA refreshingly simple data-driven game engine built in Rust\n\n## macro\n### rust compile process\n![rust compile process](/images/rust/macros/16.compile_process.png)\n\n### front end: rustc\n1. lexical analysis: Code Text -> TokenStream\n2. syntax analysis: TokenStream -> AST (abstract syntax tree)\n3. semantic analyzer: \n AST -> HIR (High-Level Intermediate Representation) -> Type HIR (static type analysis, syntactic desugar, e.g `for` changed to `loop`) -> MIR: (Mid-Level Intermediate Representation, scope, reference & borrow check)\n\n### back end: LLVM\nLLVM IR -> machine code\n\n\n### macros in compiling\n- declarative macros: TokenStream - expand -> TokenStream\n- procedule macros: self defined AST with the help or third party crate such as syn, quote\n\n## declarative macro: macro_rules!\nDeclarative macros allow us to write match-like code. The match expression is a control structure that receives an expression and then matches the result of the expression with multiple patterns. Once a pattern is matched, the code associated with the pattern will be executed\n```rust\n#![allow(unused)]\nfn main() {\n match target {\n match_pattern_1 => expr_1,\n match_pattern_2 => {\n statement1;\n statement2;\n expr_2\n },\n _ => expr_3\n }\n}\n```\n\n### example 1, simplified `vec!`\nbelow example use macro_rules to implement a simplified version of vec!\n```rust\n#[macro_export]\nmacro_rules! vec {\n ( $( $x:expr ),* ) => {\n {\n let mut temp_vec = Vec::new();\n $(\n temp_vec.push($x);\n )*\n temp_vec\n }\n };\n}\n\n#![allow(unused)]\nfn main() {\n let v: Vec = vec![1, 2, 3];\n}\n```\n\n### example 2, unless\n```rust\nmacro_rules! unless {\n ( ($arg:expr) => $branch:tt ) => ( if !$arg {$branch};);\n}\n\nfn cmp(a:i32, b:i32) {\n unless!{\n (a>b) => {println!(\"{} < {}\", a,b);}\n }\n}\nfn main() {\n cmp(1,2); /// print \"1<2\" as the condition is true !(a>b)\n cmp(3,2); /// print nothing\n}\n```\n### example 3, HashMap\n```rust\nmacro_rules! hashmap {\n // match for \"a\" => 1, \"b\" => 2,\n ( $($key:expr => $value:expr,)* ) =>\n { hashmap!($($key => $value),*) }; // recuisive\n // match for \"a\" => 1, \"b\" => 2\n ( $($key:expr => $value:expr),* ) => { \n {\n let mut _map = ::std::collections::HashMap::new();\n $(\n _map.insert($key, $value);\n )*\n _map\n }\n \n };\n}\n\nmacro_rules! hashmap_equivalent {\n ( $($key:expr => $value:expr),* $(,)*) => { \n {\n let mut _map = ::std::collections::HashMap::new();\n $(\n _map.insert($key, $value);\n )*\n _map\n }\n \n };\n}\nfn main() {\n let map = hashmap!{\n \"a\" => 1,\n \"b\" => 2, // with or without ,\n };\n let map_2 = hashmap_equivalent!{\n \"a\" => 1, \n \"b\" => 2, // with or without ,\n };\n}\n```\n\n### metavariables\n- item: an Item\n- stmt: a Statement without the trailing semicolon (except for item statements that require semicolons)\n- expr: an Expression\n- ty: a Type\n- ident: an IDENTIFIER_OR_KEYWORD or RAW_IDENTIFIER\n- path: a TypePath style path\n- tt: a TokenTree (a single token or tokens in matching delimiters (), [], or {})\n- meta: an Attr, the contents of an attribute\n- lifetime: a LIFETIME_TOKEN\n- vis: a possibly empty Visibility qualifier\n- literal: matches LiteralExpression\ndetails to be found [here](https://doc.rust-lang.org/reference/macros-by-example.html)\n\n## procedures macro\n\n## references\n1. [rust trait object](https://web.mit.edu/rust-lang_v1.25/arch/amd64_ubuntu1404/share/doc/rust/html/book/first-edition/trait-objects.html)","slug":"rust/rust-07-macro","published":1,"updated":"2023-11-05T04:21:13.734Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8ds0017qwsj9lqm1w2r","content":"

reflect

trait object

Rust provides dynamic dispatch through a feature called trait objects. Trait objects, like &Foo or Box, are normal values that store a value of any type that implements the given trait, where the precise type can only be known at runtime. more details can be found on [references 1]

\n

any

This module (std::any) contains the Any trait, which enables dynamic typing of any 'static type through runtime reflection. It also contains the Provider trait and accompanying API, which enable trait objects to provide data based on typed requests, an alternate form of runtime reflection.
Any itself can be used to get a TypeId

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
use std::fmt::Debug;
use std::any::Any;

fn log<T: Any + Debug>(value: &Any) {
    let value_any = value as &dyn Any;  // &dyn Any (a borrowed trait object), Note that &dyn Any is limited to testing whether a value is of a specified concrete type, and cannot be used to test whether a type implements a trait.

    match value_any.downcast_ref::<String>() {
        Some(val_str) -> {
            // do with string
        },
        None => {
            // 
        }
    }
}
\n\n

porpular crates using Any

    \n
  • oso
    The Oso Library is a batteries-included framework for building authorization in your application
    • \n
  • bevy
    A refreshingly simple data-driven game engine built in Rust
    • \n
\n

macro

rust compile process

\"rust

\n

front end: rustc

    \n
  1. lexical analysis: Code Text -> TokenStream
    2. \n
  2. syntax analysis: TokenStream -> AST (abstract syntax tree)
    3. \n
  3. semantic analyzer:
    AST -> HIR (High-Level Intermediate Representation) -> Type HIR (static type analysis, syntactic desugar, e.g for changed to loop) -> MIR: (Mid-Level Intermediate Representation, scope, reference & borrow check)
    4. \n
\n

back end: LLVM

LLVM IR -> machine code

\n

macros in compiling

    \n
  • declarative macros: TokenStream - expand -> TokenStream
    • \n
  • procedule macros: self defined AST with the help or third party crate such as syn, quote
    • \n
\n

declarative macro: macro_rules!

Declarative macros allow us to write match-like code. The match expression is a control structure that receives an expression and then matches the result of the expression with multiple patterns. Once a pattern is matched, the code associated with the pattern will be executed

\n
1
2
3
4
5
6
7
8
9
10
11
12
#![allow(unused)]
fn main() {
    match target {
        match_pattern_1 => expr_1,
        match_pattern_2 => {
            statement1;
            statement2;
            expr_2
        },
        _ => expr_3
    }
}
\n\n

example 1, simplified vec!

below example use macro_rules to implement a simplified version of vec!

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
#[macro_export]
macro_rules! vec {
    ( $( $x:expr ),* ) => {
        {
            let mut temp_vec = Vec::new();
            $(
                temp_vec.push($x);
            )*
            temp_vec
        }
    };
}

#![allow(unused)]
fn main() {
    let v: Vec<u32> = vec![1, 2, 3];
}
\n\n

example 2, unless

1
2
3
4
5
6
7
8
9
10
11
12
13
macro_rules! unless {
    ( ($arg:expr) => $branch:tt ) => ( if !$arg {$branch};);
}

fn cmp(a:i32, b:i32) {
    unless!{
        (a>b) => {println!("{} < {}", a,b);}
    }
}
fn main() {
    cmp(1,2);  /// print "1<2" as the condition is true !(a>b)
    cmp(3,2);  /// print nothing
}
\n

example 3, HashMap

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
macro_rules! hashmap {
    // match for "a" => 1, "b" => 2,
    ( $($key:expr => $value:expr,)* ) =>
        { hashmap!($($key => $value),*) }; // recuisive
    // match for "a" => 1, "b" => 2
    ( $($key:expr => $value:expr),* ) => { 
        {
            let mut _map = ::std::collections::HashMap::new();
            $(
                _map.insert($key, $value);
            )*
            _map
        }
       
    };
}

macro_rules! hashmap_equivalent {
    ( $($key:expr => $value:expr),* $(,)*) => { 
        {
            let mut _map = ::std::collections::HashMap::new();
            $(
                _map.insert($key, $value);
            )*
            _map
        }
       
    };
}
fn main() {
    let map = hashmap!{
        "a" => 1,
        "b" => 2,  // with or without ,
    };
    let map_2 = hashmap_equivalent!{
        "a" => 1, 
        "b" => 2,  // with or without ,
    };
}
\n\n

metavariables

    \n
  • item: an Item
    • \n
  • stmt: a Statement without the trailing semicolon (except for item statements that require semicolons)
    • \n
  • expr: an Expression
    • \n
  • ty: a Type
    • \n
  • ident: an IDENTIFIER_OR_KEYWORD or RAW_IDENTIFIER
    • \n
  • path: a TypePath style path
    • \n
  • tt: a TokenTree (a single token or tokens in matching delimiters (), [], or {})
    • \n
  • meta: an Attr, the contents of an attribute
    • \n
  • lifetime: a LIFETIME_TOKEN
    • \n
  • vis: a possibly empty Visibility qualifier
    • \n
  • literal: matches LiteralExpression
    details to be found here
    • \n
\n

procedures macro

references

    \n
  1. rust trait object
    2. \n
\n","site":{"data":{}},"excerpt":"","more":"

reflect

trait object

Rust provides dynamic dispatch through a feature called trait objects. Trait objects, like &Foo or Box, are normal values that store a value of any type that implements the given trait, where the precise type can only be known at runtime. more details can be found on [references 1]

\n

any

This module (std::any) contains the Any trait, which enables dynamic typing of any 'static type through runtime reflection. It also contains the Provider trait and accompanying API, which enable trait objects to provide data based on typed requests, an alternate form of runtime reflection.
Any itself can be used to get a TypeId

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
use std::fmt::Debug;
use std::any::Any;

fn log<T: Any + Debug>(value: &Any) {
    let value_any = value as &dyn Any;  // &dyn Any (a borrowed trait object), Note that &dyn Any is limited to testing whether a value is of a specified concrete type, and cannot be used to test whether a type implements a trait.

    match value_any.downcast_ref::<String>() {
        Some(val_str) -> {
            // do with string
        },
        None => {
            // 
        }
    }
}
\n\n

porpular crates using Any

    \n
  • oso
    The Oso Library is a batteries-included framework for building authorization in your application
    • \n
  • bevy
    A refreshingly simple data-driven game engine built in Rust
    • \n
\n

macro

rust compile process

\"rust

\n

front end: rustc

    \n
  1. lexical analysis: Code Text -> TokenStream
    2. \n
  2. syntax analysis: TokenStream -> AST (abstract syntax tree)
    3. \n
  3. semantic analyzer:
    AST -> HIR (High-Level Intermediate Representation) -> Type HIR (static type analysis, syntactic desugar, e.g for changed to loop) -> MIR: (Mid-Level Intermediate Representation, scope, reference & borrow check)
    4. \n
\n

back end: LLVM

LLVM IR -> machine code

\n

macros in compiling

    \n
  • declarative macros: TokenStream - expand -> TokenStream
    • \n
  • procedule macros: self defined AST with the help or third party crate such as syn, quote
    • \n
\n

declarative macro: macro_rules!

Declarative macros allow us to write match-like code. The match expression is a control structure that receives an expression and then matches the result of the expression with multiple patterns. Once a pattern is matched, the code associated with the pattern will be executed

\n
1
2
3
4
5
6
7
8
9
10
11
12
#![allow(unused)]
fn main() {
    match target {
        match_pattern_1 => expr_1,
        match_pattern_2 => {
            statement1;
            statement2;
            expr_2
        },
        _ => expr_3
    }
}
\n\n

example 1, simplified vec!

below example use macro_rules to implement a simplified version of vec!

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
#[macro_export]
macro_rules! vec {
    ( $( $x:expr ),* ) => {
        {
            let mut temp_vec = Vec::new();
            $(
                temp_vec.push($x);
            )*
            temp_vec
        }
    };
}

#![allow(unused)]
fn main() {
    let v: Vec<u32> = vec![1, 2, 3];
}
\n\n

example 2, unless

1
2
3
4
5
6
7
8
9
10
11
12
13
macro_rules! unless {
    ( ($arg:expr) => $branch:tt ) => ( if !$arg {$branch};);
}

fn cmp(a:i32, b:i32) {
    unless!{
        (a>b) => {println!("{} < {}", a,b);}
    }
}
fn main() {
    cmp(1,2);  /// print "1<2" as the condition is true !(a>b)
    cmp(3,2);  /// print nothing
}
\n

example 3, HashMap

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
macro_rules! hashmap {
    // match for "a" => 1, "b" => 2,
    ( $($key:expr => $value:expr,)* ) =>
        { hashmap!($($key => $value),*) }; // recuisive
    // match for "a" => 1, "b" => 2
    ( $($key:expr => $value:expr),* ) => { 
        {
            let mut _map = ::std::collections::HashMap::new();
            $(
                _map.insert($key, $value);
            )*
            _map
        }
       
    };
}

macro_rules! hashmap_equivalent {
    ( $($key:expr => $value:expr),* $(,)*) => { 
        {
            let mut _map = ::std::collections::HashMap::new();
            $(
                _map.insert($key, $value);
            )*
            _map
        }
       
    };
}
fn main() {
    let map = hashmap!{
        "a" => 1,
        "b" => 2,  // with or without ,
    };
    let map_2 = hashmap_equivalent!{
        "a" => 1, 
        "b" => 2,  // with or without ,
    };
}
\n\n

metavariables

    \n
  • item: an Item
    • \n
  • stmt: a Statement without the trailing semicolon (except for item statements that require semicolons)
    • \n
  • expr: an Expression
    • \n
  • ty: a Type
    • \n
  • ident: an IDENTIFIER_OR_KEYWORD or RAW_IDENTIFIER
    • \n
  • path: a TypePath style path
    • \n
  • tt: a TokenTree (a single token or tokens in matching delimiters (), [], or {})
    • \n
  • meta: an Attr, the contents of an attribute
    • \n
  • lifetime: a LIFETIME_TOKEN
    • \n
  • vis: a possibly empty Visibility qualifier
    • \n
  • literal: matches LiteralExpression
    details to be found here
    • \n
\n

procedures macro

references

    \n
  1. rust trait object
    2. \n
\n"},{"title":"rust project management","date":"2022-11-06T07:33:55.000Z","_content":"\n## basic concepts\n- **Packages**: A Cargo feature that lets you build, test, and share crates\n- **Crates**: A tree of modules that produces a library or executable\n- **Modules and use**: Let you control the organization, scope, and privacy of paths\n- **Paths**: A way of naming an item, such as a struct, function, or module\n\n### package\n1. one package can only has one library Crate. default is src/lib.rs\n2. one package can have multiple binary Crates (under src/bin). default src/main.rs\n3. one package has at least one crate，no matter lib or bin。\n4. one package could simultaneously have src/main.rs and src/lib.rs (crate name same to package name)\n```\ncargo new my-project --lib ## create a library project\n```\n### crate\n- binary or library\n- The crate root is a source file that the Rust compiler starts from and makes up the root module of your crate\n\n### module\n- keyword `mod` \n- module could be nested。\n- module includes(struct、enum、const、trait、func, etc)\n\n### path\n- absolute path: use crate name or `crate`\n- relative path:，use self, super etc\n```rust\nfn serve_order() {}\nmod back_of_house {\n fn fix_incorrect_order() {\n cook_order();\n super::serve_order();\n }\n fn cook_order() {}\n}\n```\n```rust\nmod front_of_house {\n pub mod hosting {\n pub fn add_to_waitlist() {}\n }\n}\npub fn eat_at_restaurant() {\n // Absolute path\n crate::front_of_house::hosting::add_to_waitlist();\n // Relative path\n front_of_house::hosting::add_to_waitlist();\n}\n```\n\n### Bringing Paths into Scope with the use Keyword\n```rust\nuse std::io;\nuse std::io::Write;\nuse std::io:: { self, Write} ;\nuse std::{cmp::Ordering, io};\nuse std::fmt::Result;\nuse std::io::Result as IoResult;\nuse std::collections::*;\n\npub use crate::front_of_house::hosting; // re-exporting names with pub use\n```\n### Separating Modules into Different Files\nFilename: src/lib.rs\n```rust\nmod front_of_house;\n\npub use crate::front_of_house::hosting;\n\npub fn eat_at_restaurant() {\n hosting::add_to_waitlist();\n hosting::add_to_waitlist();\n hosting::add_to_waitlist();\n}\n```\nFilename: src/front_of_house.rs\n```rust\npub mod hosting {\n pub fn add_to_waitlist() {}\n}\n```\n\n## profile\n```\ncargo build ## dev: [unoptimized + debuginfo]\ncargo build --release ## [optimized]\n```\n- config\n```\n[profile.dev]\nopt-level = 0\n\n[profile.release]\nopt-level = 3\n\n# profile for the wasm example (from project ethers-rs)\n[profile.release.package.ethers-wasm]\nopt-level = \"s\" # Tell `rustc` to optimize for small code size.\n```\n","source":"_posts/rust/rust-08-project-management.md","raw":"---\ntitle: rust project management\ndate: 2022-11-06 15:33:55\ntags: [rust]\n---\n\n## basic concepts\n- **Packages**: A Cargo feature that lets you build, test, and share crates\n- **Crates**: A tree of modules that produces a library or executable\n- **Modules and use**: Let you control the organization, scope, and privacy of paths\n- **Paths**: A way of naming an item, such as a struct, function, or module\n\n### package\n1. one package can only has one library Crate. default is src/lib.rs\n2. one package can have multiple binary Crates (under src/bin). default src/main.rs\n3. one package has at least one crate，no matter lib or bin。\n4. one package could simultaneously have src/main.rs and src/lib.rs (crate name same to package name)\n```\ncargo new my-project --lib ## create a library project\n```\n### crate\n- binary or library\n- The crate root is a source file that the Rust compiler starts from and makes up the root module of your crate\n\n### module\n- keyword `mod` \n- module could be nested。\n- module includes(struct、enum、const、trait、func, etc)\n\n### path\n- absolute path: use crate name or `crate`\n- relative path:，use self, super etc\n```rust\nfn serve_order() {}\nmod back_of_house {\n fn fix_incorrect_order() {\n cook_order();\n super::serve_order();\n }\n fn cook_order() {}\n}\n```\n```rust\nmod front_of_house {\n pub mod hosting {\n pub fn add_to_waitlist() {}\n }\n}\npub fn eat_at_restaurant() {\n // Absolute path\n crate::front_of_house::hosting::add_to_waitlist();\n // Relative path\n front_of_house::hosting::add_to_waitlist();\n}\n```\n\n### Bringing Paths into Scope with the use Keyword\n```rust\nuse std::io;\nuse std::io::Write;\nuse std::io:: { self, Write} ;\nuse std::{cmp::Ordering, io};\nuse std::fmt::Result;\nuse std::io::Result as IoResult;\nuse std::collections::*;\n\npub use crate::front_of_house::hosting; // re-exporting names with pub use\n```\n### Separating Modules into Different Files\nFilename: src/lib.rs\n```rust\nmod front_of_house;\n\npub use crate::front_of_house::hosting;\n\npub fn eat_at_restaurant() {\n hosting::add_to_waitlist();\n hosting::add_to_waitlist();\n hosting::add_to_waitlist();\n}\n```\nFilename: src/front_of_house.rs\n```rust\npub mod hosting {\n pub fn add_to_waitlist() {}\n}\n```\n\n## profile\n```\ncargo build ## dev: [unoptimized + debuginfo]\ncargo build --release ## [optimized]\n```\n- config\n```\n[profile.dev]\nopt-level = 0\n\n[profile.release]\nopt-level = 3\n\n# profile for the wasm example (from project ethers-rs)\n[profile.release.package.ethers-wasm]\nopt-level = \"s\" # Tell `rustc` to optimize for small code size.\n```\n","slug":"rust/rust-08-project-management","published":1,"updated":"2023-11-05T04:21:13.734Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8ds0018qwsj3eer6n3l","content":"

basic concepts

    \n
  • Packages: A Cargo feature that lets you build, test, and share crates
    • \n
  • Crates: A tree of modules that produces a library or executable
    • \n
  • Modules and use: Let you control the organization, scope, and privacy of paths
    • \n
  • Paths: A way of naming an item, such as a struct, function, or module
    • \n
\n

package

    \n
  1. one package can only has one library Crate. default is src/lib.rs
    2. \n
  2. one package can have multiple binary Crates (under src/bin). default src/main.rs
    3. \n
  3. one package has at least one crate，no matter lib or bin。
    4. \n
  4. one package could simultaneously have src/main.rs and src/lib.rs (crate name same to package name)
    1
    cargo new my-project --lib  ## create a library project
    5. \n
\n

crate

    \n
  • binary or library
    • \n
  • The crate root is a source file that the Rust compiler starts from and makes up the root module of your crate
    • \n
\n

module

    \n
  • keyword mod
    • \n
  • module could be nested。
    • \n
  • module includes(struct、enum、const、trait、func, etc)
    • \n
\n

path

    \n
  • absolute path: use crate name or crate
    • \n
  • relative path:，use self, super etc
    1
    2
    3
    4
    5
    6
    7
    8
    fn serve_order() {}
    mod back_of_house {
        fn fix_incorrect_order() {
            cook_order();
            super::serve_order();
        }
        fn cook_order() {}
    }
    \n
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    mod front_of_house {
        pub mod hosting {
            pub fn add_to_waitlist() {}
        }
    }
    pub fn eat_at_restaurant() {
        // Absolute path
        crate::front_of_house::hosting::add_to_waitlist();
        // Relative path
        front_of_house::hosting::add_to_waitlist();
    }
    • \n
\n

Bringing Paths into Scope with the use Keyword

1
2
3
4
5
6
7
8
9
use std::io;
use std::io::Write;
use std::io:: { self, Write} ;
use std::{cmp::Ordering, io};
use std::fmt::Result;
use std::io::Result as IoResult;
use std::collections::*;

pub use crate::front_of_house::hosting; // re-exporting names with pub use
\n

Separating Modules into Different Files

Filename: src/lib.rs

\n
1
2
3
4
5
6
7
8
9
mod front_of_house;

pub use crate::front_of_house::hosting;

pub fn eat_at_restaurant() {
    hosting::add_to_waitlist();
    hosting::add_to_waitlist();
    hosting::add_to_waitlist();
}
\n

Filename: src/front_of_house.rs

\n
1
2
3
pub mod hosting {
    pub fn add_to_waitlist() {}
}
\n\n

profile

1
2
cargo build ## dev: [unoptimized + debuginfo]
cargo build --release ## [optimized]
\n
    \n
  • config
    1
    2
    3
    4
    5
    6
    7
    8
    9
    [profile.dev]
    opt-level = 0

    [profile.release]
    opt-level = 3

    # profile for the wasm example (from project ethers-rs)
    [profile.release.package.ethers-wasm]
    opt-level = "s"  # Tell `rustc` to optimize for small code size.
    • \n
\n","site":{"data":{}},"excerpt":"","more":"

basic concepts

    \n
  • Packages: A Cargo feature that lets you build, test, and share crates
    • \n
  • Crates: A tree of modules that produces a library or executable
    • \n
  • Modules and use: Let you control the organization, scope, and privacy of paths
    • \n
  • Paths: A way of naming an item, such as a struct, function, or module
    • \n
\n

package

    \n
  1. one package can only has one library Crate. default is src/lib.rs
    2. \n
  2. one package can have multiple binary Crates (under src/bin). default src/main.rs
    3. \n
  3. one package has at least one crate，no matter lib or bin。
    4. \n
  4. one package could simultaneously have src/main.rs and src/lib.rs (crate name same to package name)
    1
    cargo new my-project --lib  ## create a library project
    5. \n
\n

crate

    \n
  • binary or library
    • \n
  • The crate root is a source file that the Rust compiler starts from and makes up the root module of your crate
    • \n
\n

module

    \n
  • keyword mod
    • \n
  • module could be nested。
    • \n
  • module includes(struct、enum、const、trait、func, etc)
    • \n
\n

path

    \n
  • absolute path: use crate name or crate
    • \n
  • relative path:，use self, super etc
    1
    2
    3
    4
    5
    6
    7
    8
    fn serve_order() {}
    mod back_of_house {
        fn fix_incorrect_order() {
            cook_order();
            super::serve_order();
        }
        fn cook_order() {}
    }
    \n
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    mod front_of_house {
        pub mod hosting {
            pub fn add_to_waitlist() {}
        }
    }
    pub fn eat_at_restaurant() {
        // Absolute path
        crate::front_of_house::hosting::add_to_waitlist();
        // Relative path
        front_of_house::hosting::add_to_waitlist();
    }
    • \n
\n

Bringing Paths into Scope with the use Keyword

1
2
3
4
5
6
7
8
9
use std::io;
use std::io::Write;
use std::io:: { self, Write} ;
use std::{cmp::Ordering, io};
use std::fmt::Result;
use std::io::Result as IoResult;
use std::collections::*;

pub use crate::front_of_house::hosting; // re-exporting names with pub use
\n

Separating Modules into Different Files

Filename: src/lib.rs

\n
1
2
3
4
5
6
7
8
9
mod front_of_house;

pub use crate::front_of_house::hosting;

pub fn eat_at_restaurant() {
    hosting::add_to_waitlist();
    hosting::add_to_waitlist();
    hosting::add_to_waitlist();
}
\n

Filename: src/front_of_house.rs

\n
1
2
3
pub mod hosting {
    pub fn add_to_waitlist() {}
}
\n\n

profile

1
2
cargo build ## dev: [unoptimized + debuginfo]
cargo build --release ## [optimized]
\n
    \n
  • config
    1
    2
    3
    4
    5
    6
    7
    8
    9
    [profile.dev]
    opt-level = 0

    [profile.release]
    opt-level = 3

    # profile for the wasm example (from project ethers-rs)
    [profile.release.package.ethers-wasm]
    opt-level = "s"  # Tell `rustc` to optimize for small code size.
    • \n
\n"},{"title":"rust functional programming","date":"2023-04-11T14:04:38.000Z","_content":"\n## iterator\n- `iter()` Returns an iterator over the **slice**\n- `into_iter()` Creates a consuming iterator, that is, one that moves each value out of the vector\n- `iter_mut()` Returns an iterator that allows modifying each value.\n\n## flat_map\n```rust\n/// Creates an iterator that works like map, but flattens nested structure.\n///\n/// The [`map`] adapter is very useful, but only when the closure\n/// argument produces values. If it produces an iterator instead, there's\n/// an extra layer of indirection. `flat_map()` will remove this extra layer\n/// on its own.\n```\n### Examples\n```rust\nlet words = [\"alpha\", \"beta\", \"gamma\"];\n// chars() returns an iterator\nlet merged: String = words.iter()\n .flat_map(|s| s.chars())\n .collect();\nassert_eq!(merged, \"alphabetagamma\");\n```\n","source":"_posts/rust/rust-09-functional.md","raw":"---\ntitle: rust functional programming\ndate: 2023-04-11 22:04:38\ntags: [rust]\n---\n\n## iterator\n- `iter()` Returns an iterator over the **slice**\n- `into_iter()` Creates a consuming iterator, that is, one that moves each value out of the vector\n- `iter_mut()` Returns an iterator that allows modifying each value.\n\n## flat_map\n```rust\n/// Creates an iterator that works like map, but flattens nested structure.\n///\n/// The [`map`] adapter is very useful, but only when the closure\n/// argument produces values. If it produces an iterator instead, there's\n/// an extra layer of indirection. `flat_map()` will remove this extra layer\n/// on its own.\n```\n### Examples\n```rust\nlet words = [\"alpha\", \"beta\", \"gamma\"];\n// chars() returns an iterator\nlet merged: String = words.iter()\n .flat_map(|s| s.chars())\n .collect();\nassert_eq!(merged, \"alphabetagamma\");\n```\n","slug":"rust/rust-09-functional","published":1,"updated":"2023-11-05T04:21:13.735Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8ds001aqwsjdxc70zfv","content":"

iterator

    \n
  • iter() Returns an iterator over the slice
    • \n
  • into_iter() Creates a consuming iterator, that is, one that moves each value out of the vector
    • \n
  • iter_mut() Returns an iterator that allows modifying each value.
    • \n
\n

flat_map

1
2
3
4
5
6
/// Creates an iterator that works like map, but flattens nested structure.
///
/// The [`map`] adapter is very useful, but only when the closure
/// argument produces values. If it produces an iterator instead, there's
/// an extra layer of indirection. `flat_map()` will remove this extra layer
/// on its own.
\n

Examples

1
2
3
4
5
6
let words = ["alpha", "beta", "gamma"];
// chars() returns an iterator
let merged: String = words.iter()
                          .flat_map(|s| s.chars())
                          .collect();
assert_eq!(merged, "alphabetagamma");
\n","site":{"data":{}},"excerpt":"","more":"

iterator

    \n
  • iter() Returns an iterator over the slice
    • \n
  • into_iter() Creates a consuming iterator, that is, one that moves each value out of the vector
    • \n
  • iter_mut() Returns an iterator that allows modifying each value.
    • \n
\n

flat_map

1
2
3
4
5
6
/// Creates an iterator that works like map, but flattens nested structure.
///
/// The [`map`] adapter is very useful, but only when the closure
/// argument produces values. If it produces an iterator instead, there's
/// an extra layer of indirection. `flat_map()` will remove this extra layer
/// on its own.
\n

Examples

1
2
3
4
5
6
let words = ["alpha", "beta", "gamma"];
// chars() returns an iterator
let merged: String = words.iter()
                          .flat_map(|s| s.chars())
                          .collect();
assert_eq!(merged, "alphabetagamma");
\n"},{"title":"rust async","date":"2023-01-13T09:17:10.000Z","_content":" \n\n## I/O\nI/O：在计算机中指Input/Output。由于程序和运行时数据是在内存中驻留，由cpu来执行，涉及到数据交换的地方，通常是磁盘、网卡等，就需要IO接口\n\n\n### I/O 模型\n```plantuml\n@startmindmap\n+ **I/O模型**\n++ 同步I/O\n'tag::details[]\n+++_ 阻塞I/O (BIO)\n+++_ 非阻塞I/O (NIO)\n+++_ I/O多路复用\n+++_ 信号驱动I/O\n'end::details[]\n++ 异步I/O\n'tag::details[]\n+++_ linux (AIO, io_uring)\n+++_ windows (IOCP)\n'end::details[]\n@endmindmap\n```\n#### 同步阻塞\n- 当用户线程发起IO请求后，会进行系统调用（system call）来让内核（Kernel）进行IO操作（系统调用是用户空间和内核空间的一个通道）；\n- 此时用户线程阻塞，等待内核将数据准备好；\n- 内核将数据准备好后会将数据从内核空间拷贝到用户空间，并返回给用户线程结束阻塞。\n\n#### 同步非阻塞\n- 由用户线程发起IO请求，进行系统调用来让内核进行IO操作；\n- 此时如果内核没有准备好数据则会直接返回error，并不会阻塞用户线程，用户线程可以重复发起IO请求；\n- 当用户线程发起请求并且内核已经将数据准备好后，会将数据从内核空间拷贝到用户空间（这个过程是需要阻塞用户线程的），返回给用户。\n\n#### 同步多路复用\n- 用户线程调用select后进行系统调用（内核会监视所有select负责的socket）\n- 当用户将数据准备好后就会返回，并通知用户线程进行读取操作，此时内核将数据拷贝到用户空间并返回。此时用户线程被阻塞；\n\n#### 异步I/O\n- 用户线程进行aio_read，进行系统调用切换到内核；\n- 内核立即返回，并不会阻塞用户线程；\n- 内核准备好数据后会将数据从内核空间拷贝到用户空间并通知用户线程（发送信号）操作已完成。\n\n### 流程图\n#### 同步blocking I/O\n\n```plantuml\n@startuml Test Diagram\n\nparticipant \"application\" as app\nparticipant \"kernel\" as kernel\n\nactivate app\nactivate kernel\napp -> kernel: syscall: Read recvfrom\nkernel -> kernel: wait for data (no datagram ready)\nkernel -> kernel: copy datagram to user (datagram ready)\nkernel -> app: return\n@enduml\n```\n\n#### I/O多路复用\n\n### 异步编程\n\n\n## the Future Trait\nA Future is an asynchronous computation that can produce a value. A simplified version of the future trait might look something like this\n\n```rust\n\ntrait SimpleFuture {\n type Output;\n fn poll(&mut self, wake: fn()) -> Poll;\n}\n\nenum Poll {\n Ready(T),\n Pending,\n}\n```\n\nFor example, consider the case where we want to read from a socket that may or may not have data available already.\n```rust\npub struct SocketRead<'a> {\n socket: &'a Socket,\n}\n\nimpl SimpleFuture for SocketRead<'_> {\n type Output = Vec;\n\n fn poll(&mut self, wake: fn()) -> Poll {\n if self.socket.has_data_to_read() {\n // The socket has data -- read it into a buffer and return it.\n Poll::Ready(self.socket.read_buf())\n } else {\n // The socket does not yet have data.\n //\n // Arrange for `wake` to be called once data is available.\n // When data becomes available, `wake` will be called, and the\n // user of this `Future` will know to call `poll` again and\n // receive data.\n self.socket.set_readable_callback(wake);\n Poll::Pending\n }\n }\n}\n\n```\n\n the real Future trait and how it is different\n```rust\ntrait Future {\n type Output;\n fn poll(\n // Note the change from `&mut self` to `Pin<&mut Self>`:\n self: Pin<&mut Self>,\n // and the change from `wake: fn()` to `cx: &mut Context<'_>`:\n cx: &mut Context<'_>,\n ) -> Poll;\n}\n\n```\nThe first change you'll notice is that our self type is no longer &mut Self, but has changed to Pin<&mut Self>. We'll talk more about pinning in a later section, but for now know that it allows us to create futures that are immovable. Immovable objects can store pointers between their fields, e.g. struct MyFut { a: i32, ptr_to_a: *const i32 }. Pinning is necessary to enable async/await.\n\nSecondly, wake: fn() has changed to &mut Context<'_>. In SimpleFuture, we used a call to a function pointer (fn()) to tell the future executor that the future in question should be polled. However, since fn() is just a function pointer, it can't store any data about which Future called wake.\n\nIn a real-world scenario, a complex application like a web server may have thousands of different connections whose wakeups should all be managed separately. The Context type solves this by providing access to a value of type Waker, which can be used to wake up a specific task.\n\n\n## task wakeups with Waker\nWaker provides a wake() method that can be used to tell the executor that the associated task should be awoken. When wake() is called, the executor knows that the task associated with the Waker is ready to make progress, and its future should be polled again.\n\n## referencs\n[csdn blog](https://blog.csdn.net/XMJYever/article/details/111560976)\n","source":"_posts/rust/rust-async.md","raw":"---\ntitle: rust async\ndate: 2023-01-13 17:17:10\ntags: [rust]\n--- \n\n## I/O\nI/O：在计算机中指Input/Output。由于程序和运行时数据是在内存中驻留，由cpu来执行，涉及到数据交换的地方，通常是磁盘、网卡等，就需要IO接口\n\n\n### I/O 模型\n```plantuml\n@startmindmap\n+ **I/O模型**\n++ 同步I/O\n'tag::details[]\n+++_ 阻塞I/O (BIO)\n+++_ 非阻塞I/O (NIO)\n+++_ I/O多路复用\n+++_ 信号驱动I/O\n'end::details[]\n++ 异步I/O\n'tag::details[]\n+++_ linux (AIO, io_uring)\n+++_ windows (IOCP)\n'end::details[]\n@endmindmap\n```\n#### 同步阻塞\n- 当用户线程发起IO请求后，会进行系统调用（system call）来让内核（Kernel）进行IO操作（系统调用是用户空间和内核空间的一个通道）；\n- 此时用户线程阻塞，等待内核将数据准备好；\n- 内核将数据准备好后会将数据从内核空间拷贝到用户空间，并返回给用户线程结束阻塞。\n\n#### 同步非阻塞\n- 由用户线程发起IO请求，进行系统调用来让内核进行IO操作；\n- 此时如果内核没有准备好数据则会直接返回error，并不会阻塞用户线程，用户线程可以重复发起IO请求；\n- 当用户线程发起请求并且内核已经将数据准备好后，会将数据从内核空间拷贝到用户空间（这个过程是需要阻塞用户线程的），返回给用户。\n\n#### 同步多路复用\n- 用户线程调用select后进行系统调用（内核会监视所有select负责的socket）\n- 当用户将数据准备好后就会返回，并通知用户线程进行读取操作，此时内核将数据拷贝到用户空间并返回。此时用户线程被阻塞；\n\n#### 异步I/O\n- 用户线程进行aio_read，进行系统调用切换到内核；\n- 内核立即返回，并不会阻塞用户线程；\n- 内核准备好数据后会将数据从内核空间拷贝到用户空间并通知用户线程（发送信号）操作已完成。\n\n### 流程图\n#### 同步blocking I/O\n\n```plantuml\n@startuml Test Diagram\n\nparticipant \"application\" as app\nparticipant \"kernel\" as kernel\n\nactivate app\nactivate kernel\napp -> kernel: syscall: Read recvfrom\nkernel -> kernel: wait for data (no datagram ready)\nkernel -> kernel: copy datagram to user (datagram ready)\nkernel -> app: return\n@enduml\n```\n\n#### I/O多路复用\n\n### 异步编程\n\n\n## the Future Trait\nA Future is an asynchronous computation that can produce a value. A simplified version of the future trait might look something like this\n\n```rust\n\ntrait SimpleFuture {\n type Output;\n fn poll(&mut self, wake: fn()) -> Poll;\n}\n\nenum Poll {\n Ready(T),\n Pending,\n}\n```\n\nFor example, consider the case where we want to read from a socket that may or may not have data available already.\n```rust\npub struct SocketRead<'a> {\n socket: &'a Socket,\n}\n\nimpl SimpleFuture for SocketRead<'_> {\n type Output = Vec;\n\n fn poll(&mut self, wake: fn()) -> Poll {\n if self.socket.has_data_to_read() {\n // The socket has data -- read it into a buffer and return it.\n Poll::Ready(self.socket.read_buf())\n } else {\n // The socket does not yet have data.\n //\n // Arrange for `wake` to be called once data is available.\n // When data becomes available, `wake` will be called, and the\n // user of this `Future` will know to call `poll` again and\n // receive data.\n self.socket.set_readable_callback(wake);\n Poll::Pending\n }\n }\n}\n\n```\n\n the real Future trait and how it is different\n```rust\ntrait Future {\n type Output;\n fn poll(\n // Note the change from `&mut self` to `Pin<&mut Self>`:\n self: Pin<&mut Self>,\n // and the change from `wake: fn()` to `cx: &mut Context<'_>`:\n cx: &mut Context<'_>,\n ) -> Poll;\n}\n\n```\nThe first change you'll notice is that our self type is no longer &mut Self, but has changed to Pin<&mut Self>. We'll talk more about pinning in a later section, but for now know that it allows us to create futures that are immovable. Immovable objects can store pointers between their fields, e.g. struct MyFut { a: i32, ptr_to_a: *const i32 }. Pinning is necessary to enable async/await.\n\nSecondly, wake: fn() has changed to &mut Context<'_>. In SimpleFuture, we used a call to a function pointer (fn()) to tell the future executor that the future in question should be polled. However, since fn() is just a function pointer, it can't store any data about which Future called wake.\n\nIn a real-world scenario, a complex application like a web server may have thousands of different connections whose wakeups should all be managed separately. The Context type solves this by providing access to a value of type Waker, which can be used to wake up a specific task.\n\n\n## task wakeups with Waker\nWaker provides a wake() method that can be used to tell the executor that the associated task should be awoken. When wake() is called, the executor knows that the task associated with the Waker is ready to make progress, and its future should be polled again.\n\n## referencs\n[csdn blog](https://blog.csdn.net/XMJYever/article/details/111560976)\n","slug":"rust/rust-async","published":1,"updated":"2023-11-05T04:21:13.735Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8dt001cqwsj0grhcnus","content":"

I/O

I/O：在计算机中指Input/Output。由于程序和运行时数据是在内存中驻留，由cpu来执行，涉及到数据交换的地方，通常是磁盘、网卡等，就需要IO接口

\n

I/O 模型

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
@startmindmap
+ **I/O模型**
++ 同步I/O
'tag::details[]
+++_ 阻塞I/O (BIO)
+++_ 非阻塞I/O (NIO)
+++_ I/O多路复用
+++_ 信号驱动I/O
'end::details[]
++ 异步I/O
'tag::details[]
+++_ linux (AIO, io_uring)
+++_ windows (IOCP)
'end::details[]
@endmindmap
\n

同步阻塞

    \n
  • 当用户线程发起IO请求后，会进行系统调用（system call）来让内核（Kernel）进行IO操作（系统调用是用户空间和内核空间的一个通道）；
    • \n
  • 此时用户线程阻塞，等待内核将数据准备好；
    • \n
  • 内核将数据准备好后会将数据从内核空间拷贝到用户空间，并返回给用户线程结束阻塞。
    • \n
\n

同步非阻塞

    \n
  • 由用户线程发起IO请求，进行系统调用来让内核进行IO操作；
    • \n
  • 此时如果内核没有准备好数据则会直接返回error，并不会阻塞用户线程，用户线程可以重复发起IO请求；
    • \n
  • 当用户线程发起请求并且内核已经将数据准备好后，会将数据从内核空间拷贝到用户空间（这个过程是需要阻塞用户线程的），返回给用户。
    • \n
\n

同步多路复用

    \n
  • 用户线程调用select后进行系统调用（内核会监视所有select负责的socket）
    • \n
  • 当用户将数据准备好后就会返回，并通知用户线程进行读取操作，此时内核将数据拷贝到用户空间并返回。此时用户线程被阻塞；
    • \n
\n

异步I/O

    \n
  • 用户线程进行aio_read，进行系统调用切换到内核；
    • \n
  • 内核立即返回，并不会阻塞用户线程；
    • \n
  • 内核准备好数据后会将数据从内核空间拷贝到用户空间并通知用户线程（发送信号）操作已完成。
    • \n
\n

流程图

同步blocking I/O

1
2
3
4
5
6
7
8
9
10
11
12
@startuml Test Diagram

participant "application" as app
participant "kernel" as kernel

activate app
activate kernel
app -> kernel: syscall: Read recvfrom
kernel -> kernel: wait for data (no datagram ready)
kernel -> kernel: copy datagram to user (datagram ready)
kernel -> app: return
@enduml
\n\n

I/O多路复用

异步编程

the Future Trait

A Future is an asynchronous computation that can produce a value. A simplified version of the future trait might look something like this

\n
1
2
3
4
5
6
7
8
9
10

trait SimpleFuture {
    type Output;
    fn poll(&mut self, wake: fn()) -> Poll<Self::Output>;
}

enum Poll<T> {
    Ready(T),
    Pending,
}
\n\n

For example, consider the case where we want to read from a socket that may or may not have data available already.

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
pub struct SocketRead<'a> {
    socket: &'a Socket,
}

impl SimpleFuture for SocketRead<'_> {
    type Output = Vec<u8>;

    fn poll(&mut self, wake: fn()) -> Poll<Self::Output> {
        if self.socket.has_data_to_read() {
            // The socket has data -- read it into a buffer and return it.
            Poll::Ready(self.socket.read_buf())
        } else {
            // The socket does not yet have data.
            //
            // Arrange for `wake` to be called once data is available.
            // When data becomes available, `wake` will be called, and the
            // user of this `Future` will know to call `poll` again and
            // receive data.
            self.socket.set_readable_callback(wake);
            Poll::Pending
        }
    }
}

\n\n

the real Future trait and how it is different

\n
1
2
3
4
5
6
7
8
9
10
trait Future {
    type Output;
    fn poll(
        // Note the change from `&mut self` to `Pin<&mut Self>`:
        self: Pin<&mut Self>,
        // and the change from `wake: fn()` to `cx: &mut Context<'_>`:
        cx: &mut Context<'_>,
    ) -> Poll<Self::Output>;
}

\n

The first change you’ll notice is that our self type is no longer &mut Self, but has changed to Pin<&mut Self>. We’ll talk more about pinning in a later section, but for now know that it allows us to create futures that are immovable. Immovable objects can store pointers between their fields, e.g. struct MyFut { a: i32, ptr_to_a: *const i32 }. Pinning is necessary to enable async/await.

\n

Secondly, wake: fn() has changed to &mut Context<’_>. In SimpleFuture, we used a call to a function pointer (fn()) to tell the future executor that the future in question should be polled. However, since fn() is just a function pointer, it can’t store any data about which Future called wake.

\n

In a real-world scenario, a complex application like a web server may have thousands of different connections whose wakeups should all be managed separately. The Context type solves this by providing access to a value of type Waker, which can be used to wake up a specific task.

\n

task wakeups with Waker

Waker provides a wake() method that can be used to tell the executor that the associated task should be awoken. When wake() is called, the executor knows that the task associated with the Waker is ready to make progress, and its future should be polled again.

\n

referencs

csdn blog

\n","site":{"data":{}},"excerpt":"","more":"

I/O

I/O：在计算机中指Input/Output。由于程序和运行时数据是在内存中驻留，由cpu来执行，涉及到数据交换的地方，通常是磁盘、网卡等，就需要IO接口

\n

I/O 模型

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
@startmindmap
+ **I/O模型**
++ 同步I/O
'tag::details[]
+++_ 阻塞I/O (BIO)
+++_ 非阻塞I/O (NIO)
+++_ I/O多路复用
+++_ 信号驱动I/O
'end::details[]
++ 异步I/O
'tag::details[]
+++_ linux (AIO, io_uring)
+++_ windows (IOCP)
'end::details[]
@endmindmap
\n

同步阻塞

    \n
  • 当用户线程发起IO请求后，会进行系统调用（system call）来让内核（Kernel）进行IO操作（系统调用是用户空间和内核空间的一个通道）；
    • \n
  • 此时用户线程阻塞，等待内核将数据准备好；
    • \n
  • 内核将数据准备好后会将数据从内核空间拷贝到用户空间，并返回给用户线程结束阻塞。
    • \n
\n

同步非阻塞

    \n
  • 由用户线程发起IO请求，进行系统调用来让内核进行IO操作；
    • \n
  • 此时如果内核没有准备好数据则会直接返回error，并不会阻塞用户线程，用户线程可以重复发起IO请求；
    • \n
  • 当用户线程发起请求并且内核已经将数据准备好后，会将数据从内核空间拷贝到用户空间（这个过程是需要阻塞用户线程的），返回给用户。
    • \n
\n

同步多路复用

    \n
  • 用户线程调用select后进行系统调用（内核会监视所有select负责的socket）
    • \n
  • 当用户将数据准备好后就会返回，并通知用户线程进行读取操作，此时内核将数据拷贝到用户空间并返回。此时用户线程被阻塞；
    • \n
\n

异步I/O

    \n
  • 用户线程进行aio_read，进行系统调用切换到内核；
    • \n
  • 内核立即返回，并不会阻塞用户线程；
    • \n
  • 内核准备好数据后会将数据从内核空间拷贝到用户空间并通知用户线程（发送信号）操作已完成。
    • \n
\n

流程图

同步blocking I/O

1
2
3
4
5
6
7
8
9
10
11
12
@startuml Test Diagram

participant "application" as app
participant "kernel" as kernel

activate app
activate kernel
app -> kernel: syscall: Read recvfrom
kernel -> kernel: wait for data (no datagram ready)
kernel -> kernel: copy datagram to user (datagram ready)
kernel -> app: return
@enduml
\n\n

I/O多路复用

异步编程

the Future Trait

A Future is an asynchronous computation that can produce a value. A simplified version of the future trait might look something like this

\n
1
2
3
4
5
6
7
8
9
10

trait SimpleFuture {
    type Output;
    fn poll(&mut self, wake: fn()) -> Poll<Self::Output>;
}

enum Poll<T> {
    Ready(T),
    Pending,
}
\n\n

For example, consider the case where we want to read from a socket that may or may not have data available already.

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
pub struct SocketRead<'a> {
    socket: &'a Socket,
}

impl SimpleFuture for SocketRead<'_> {
    type Output = Vec<u8>;

    fn poll(&mut self, wake: fn()) -> Poll<Self::Output> {
        if self.socket.has_data_to_read() {
            // The socket has data -- read it into a buffer and return it.
            Poll::Ready(self.socket.read_buf())
        } else {
            // The socket does not yet have data.
            //
            // Arrange for `wake` to be called once data is available.
            // When data becomes available, `wake` will be called, and the
            // user of this `Future` will know to call `poll` again and
            // receive data.
            self.socket.set_readable_callback(wake);
            Poll::Pending
        }
    }
}

\n\n

the real Future trait and how it is different

\n
1
2
3
4
5
6
7
8
9
10
trait Future {
    type Output;
    fn poll(
        // Note the change from `&mut self` to `Pin<&mut Self>`:
        self: Pin<&mut Self>,
        // and the change from `wake: fn()` to `cx: &mut Context<'_>`:
        cx: &mut Context<'_>,
    ) -> Poll<Self::Output>;
}

\n

The first change you’ll notice is that our self type is no longer &mut Self, but has changed to Pin<&mut Self>. We’ll talk more about pinning in a later section, but for now know that it allows us to create futures that are immovable. Immovable objects can store pointers between their fields, e.g. struct MyFut { a: i32, ptr_to_a: *const i32 }. Pinning is necessary to enable async/await.

\n

Secondly, wake: fn() has changed to &mut Context<’_>. In SimpleFuture, we used a call to a function pointer (fn()) to tell the future executor that the future in question should be polled. However, since fn() is just a function pointer, it can’t store any data about which Future called wake.

\n

In a real-world scenario, a complex application like a web server may have thousands of different connections whose wakeups should all be managed separately. The Context type solves this by providing access to a value of type Waker, which can be used to wake up a specific task.

\n

task wakeups with Waker

Waker provides a wake() method that can be used to tell the executor that the associated task should be awoken. When wake() is called, the executor knows that the task associated with the Waker is ready to make progress, and its future should be polled again.

\n

referencs

csdn blog

\n"},{"title":"rust concurrency","date":"2023-06-01T14:04:38.000Z","_content":"\n## Send and Sync\n- A type is Send if it is safe to send it to another thread.\n- A type is Sync if it is safe to share between threads (T is Sync if and only if &T is Send).\n- raw pointers are neither Send nor Sync (because they have no safety guards).\n- UnsafeCell isn't Sync (and therefore Cell and RefCell aren't).\n- Rc isn't Send or Sync (because the refcount is shared and unsynchronized).\n\nTypes that aren't automatically derived can simply implement them if desired:\n```rust\nstruct MyBox(*mut u8);\n\nunsafe impl Send for MyBox {}\nunsafe impl Sync for MyBox {}\n```\none can also unimplement Send and Sync:\n```rust\n#![feature(negative_impls)]\n\n// I have some magic semantics for some synchronization primitive!\nstruct SpecialThreadToken(u8);\n\nimpl !Send for SpecialThreadToken {}\nimpl !Sync for SpecialThreadToken {}\n```\n\n## reference\n- [rustonomicon](https://doc.rust-lang.org/nomicon/send-and-sync.html)","source":"_posts/rust/rust-10-concurrency.md","raw":"---\ntitle: rust concurrency\ndate: 2023-06-01 22:04:38\ntags: [rust]\n---\n\n## Send and Sync\n- A type is Send if it is safe to send it to another thread.\n- A type is Sync if it is safe to share between threads (T is Sync if and only if &T is Send).\n- raw pointers are neither Send nor Sync (because they have no safety guards).\n- UnsafeCell isn't Sync (and therefore Cell and RefCell aren't).\n- Rc isn't Send or Sync (because the refcount is shared and unsynchronized).\n\nTypes that aren't automatically derived can simply implement them if desired:\n```rust\nstruct MyBox(*mut u8);\n\nunsafe impl Send for MyBox {}\nunsafe impl Sync for MyBox {}\n```\none can also unimplement Send and Sync:\n```rust\n#![feature(negative_impls)]\n\n// I have some magic semantics for some synchronization primitive!\nstruct SpecialThreadToken(u8);\n\nimpl !Send for SpecialThreadToken {}\nimpl !Sync for SpecialThreadToken {}\n```\n\n## reference\n- [rustonomicon](https://doc.rust-lang.org/nomicon/send-and-sync.html)","slug":"rust/rust-10-concurrency","published":1,"updated":"2023-11-05T04:21:13.735Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8dt001fqwsjbi7t4e7s","content":"

Send and Sync

    \n
  • A type is Send if it is safe to send it to another thread.
    • \n
  • A type is Sync if it is safe to share between threads (T is Sync if and only if &T is Send).
    • \n
  • raw pointers are neither Send nor Sync (because they have no safety guards).
    • \n
  • UnsafeCell isn’t Sync (and therefore Cell and RefCell aren’t).
    • \n
  • Rc isn’t Send or Sync (because the refcount is shared and unsynchronized).
    • \n
\n

Types that aren’t automatically derived can simply implement them if desired:

\n
1
2
3
4
struct MyBox(*mut u8);

unsafe impl Send for MyBox {}
unsafe impl Sync for MyBox {}
\n

one can also unimplement Send and Sync:

\n
1
2
3
4
5
6
7
#![feature(negative_impls)]

// I have some magic semantics for some synchronization primitive!
struct SpecialThreadToken(u8);

impl !Send for SpecialThreadToken {}
impl !Sync for SpecialThreadToken {}
\n\n

reference

\n","site":{"data":{}},"excerpt":"","more":"

Send and Sync

    \n
  • A type is Send if it is safe to send it to another thread.
    • \n
  • A type is Sync if it is safe to share between threads (T is Sync if and only if &T is Send).
    • \n
  • raw pointers are neither Send nor Sync (because they have no safety guards).
    • \n
  • UnsafeCell isn’t Sync (and therefore Cell and RefCell aren’t).
    • \n
  • Rc isn’t Send or Sync (because the refcount is shared and unsynchronized).
    • \n
\n

Types that aren’t automatically derived can simply implement them if desired:

\n
1
2
3
4
struct MyBox(*mut u8);

unsafe impl Send for MyBox {}
unsafe impl Sync for MyBox {}
\n

one can also unimplement Send and Sync:

\n
1
2
3
4
5
6
7
#![feature(negative_impls)]

// I have some magic semantics for some synchronization primitive!
struct SpecialThreadToken(u8);

impl !Send for SpecialThreadToken {}
impl !Sync for SpecialThreadToken {}
\n\n

reference

\n"},{"title":"rust cargo all in one","date":"2022-12-06T09:05:07.000Z","_content":"\n## useful cmd\n```\ncargo new ${crate_name} --lib ## create a lib crate\ncargo build --verbose ## print out each rustc invocation\n```\n\n## specifying dependencies\n- specifying dependencies from crates.io\n```toml\n[dependencies]\ntime = \"0.1.12\"\n```\n- specifying dependencies from other registries\n```toml\n[dependencies]\nsome-crate = { version = \"1.0\", registry = \"my-registry\" }\n```\n- specifying dependencies form git repositories\n```toml\n[dependencies]\nregex = { git = \"https://github.com/rust-lang/regex.git\" }\n```\n- path dependencies\n```toml\n[dependencies]\nhello_utils = { path = \"hello_utils\" }\n```\n- platform specific dependencies\n```toml\n[target.'cfg(unix)'.dependencies]\nopenssl = \"1.0.1\"\n\n[target.'cfg(target_arch = \"x86\")'.dependencies]\nnative-i686 = { path = \"native/i686\" }\n```\nLike with Rust, the syntax here supports the not, any, and all operators to combine various cfg name/value pairs.\nIf you want to know which cfg targets are available on your platform, run ```rustc --print=cfg``` from the command line.\nIf you want to know which cfg targets are available for another platform, such as 64-bit Windows, run ```rustc --print=cfg --target=x86_64-pc-windows-msvc```\n- custom target specifications\n```toml\n[target.bar.dependencies]\nwinhttp = \"0.4.0\"\n\n[target.my-special-i686-platform.dependencies]\nopenssl = \"1.0.1\"\nnative = { path = \"native/i686\" }\n```\n- development dependencies\n [dev-dependencies]\n Dev-dependencies are not used when compiling a package for building, but are used for compiling tests, examples, and benchmarks.\n These dependencies are not propagated to other packages which depend on this package.\n ```toml\n [target.'cfg(unix)'.dev-dependencies]\n mio = \"0.0.1\"\n ```\n\n- build dependencies\n ```toml\n [build-dependencies]\n cc = \"1.0.3\"\n ```\n The build script does not have access to the dependencies listed in the dependencies or dev-dependencies section. Build dependencies will likewise not be available to the package itself unless listed under the dependencies section as well.\n- choosing features\n ```toml\n [dependencies.awesome]\n version = \"1.3.5\"\n default-features = false # do not include the default features, and optionally\n # cherry-pick individual features\n features = [\"secure-password\", \"civet\"]\n ```\n- renaming dependencies in Cargo.toml\n When writing a [dependencies] section in Cargo.toml the key you write for a dependency typically matches up to the name of the crate you import from in the code. For some projects, though, you may wish to reference the crate with a different name in the code regardless of how it's published on crates.io. For example you may wish to: Avoid the need to use foo as bar in Rust source.\n (more to be found in original book)\n\n## references\n[cargo book](https://doc.rust-lang.org/cargo/)","source":"_posts/rust/rust-cargo-all-in-one.md","raw":"---\ntitle: rust cargo all in one\ndate: 2022-12-06 17:05:07\ntags: [rust]\n---\n\n## useful cmd\n```\ncargo new ${crate_name} --lib ## create a lib crate\ncargo build --verbose ## print out each rustc invocation\n```\n\n## specifying dependencies\n- specifying dependencies from crates.io\n```toml\n[dependencies]\ntime = \"0.1.12\"\n```\n- specifying dependencies from other registries\n```toml\n[dependencies]\nsome-crate = { version = \"1.0\", registry = \"my-registry\" }\n```\n- specifying dependencies form git repositories\n```toml\n[dependencies]\nregex = { git = \"https://github.com/rust-lang/regex.git\" }\n```\n- path dependencies\n```toml\n[dependencies]\nhello_utils = { path = \"hello_utils\" }\n```\n- platform specific dependencies\n```toml\n[target.'cfg(unix)'.dependencies]\nopenssl = \"1.0.1\"\n\n[target.'cfg(target_arch = \"x86\")'.dependencies]\nnative-i686 = { path = \"native/i686\" }\n```\nLike with Rust, the syntax here supports the not, any, and all operators to combine various cfg name/value pairs.\nIf you want to know which cfg targets are available on your platform, run ```rustc --print=cfg``` from the command line.\nIf you want to know which cfg targets are available for another platform, such as 64-bit Windows, run ```rustc --print=cfg --target=x86_64-pc-windows-msvc```\n- custom target specifications\n```toml\n[target.bar.dependencies]\nwinhttp = \"0.4.0\"\n\n[target.my-special-i686-platform.dependencies]\nopenssl = \"1.0.1\"\nnative = { path = \"native/i686\" }\n```\n- development dependencies\n [dev-dependencies]\n Dev-dependencies are not used when compiling a package for building, but are used for compiling tests, examples, and benchmarks.\n These dependencies are not propagated to other packages which depend on this package.\n ```toml\n [target.'cfg(unix)'.dev-dependencies]\n mio = \"0.0.1\"\n ```\n\n- build dependencies\n ```toml\n [build-dependencies]\n cc = \"1.0.3\"\n ```\n The build script does not have access to the dependencies listed in the dependencies or dev-dependencies section. Build dependencies will likewise not be available to the package itself unless listed under the dependencies section as well.\n- choosing features\n ```toml\n [dependencies.awesome]\n version = \"1.3.5\"\n default-features = false # do not include the default features, and optionally\n # cherry-pick individual features\n features = [\"secure-password\", \"civet\"]\n ```\n- renaming dependencies in Cargo.toml\n When writing a [dependencies] section in Cargo.toml the key you write for a dependency typically matches up to the name of the crate you import from in the code. For some projects, though, you may wish to reference the crate with a different name in the code regardless of how it's published on crates.io. For example you may wish to: Avoid the need to use foo as bar in Rust source.\n (more to be found in original book)\n\n## references\n[cargo book](https://doc.rust-lang.org/cargo/)","slug":"rust/rust-cargo-all-in-one","published":1,"updated":"2023-11-05T04:21:13.735Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8dt001hqwsj4zoehfes","content":"

useful cmd

1
2
cargo new ${crate_name} --lib ## create a lib crate
cargo build --verbose ## print out each rustc invocation
\n\n

specifying dependencies

    \n

  • specifying dependencies from crates.io

    \n
    1
    2
    [dependencies]
    time = "0.1.12"
    • \n

  • specifying dependencies from other registries

    \n
    1
    2
    [dependencies]
    some-crate = { version = "1.0", registry = "my-registry" }
    • \n

  • specifying dependencies form git repositories

    \n
    1
    2
    [dependencies]
    regex = { git = "https://github.com/rust-lang/regex.git" }
    • \n

  • path dependencies

    \n
    1
    2
    [dependencies]
    hello_utils = { path = "hello_utils" }
    • \n

  • platform specific dependencies

    \n
    1
    2
    3
    4
    5
    [target.'cfg(unix)'.dependencies]
    openssl = "1.0.1"

    [target.'cfg(target_arch = "x86")'.dependencies]
    native-i686 = { path = "native/i686" }
    \n

    Like with Rust, the syntax here supports the not, any, and all operators to combine various cfg name/value pairs.
    If you want to know which cfg targets are available on your platform, run rustc --print=cfg from the command line.
    If you want to know which cfg targets are available for another platform, such as 64-bit Windows, run rustc --print=cfg --target=x86_64-pc-windows-msvc

    \n
    • \n

  • custom target specifications

    \n
    1
    2
    3
    4
    5
    6
    [target.bar.dependencies]
    winhttp = "0.4.0"

    [target.my-special-i686-platform.dependencies]
    openssl = "1.0.1"
    native = { path = "native/i686" }
    • \n

  • development dependencies
    [dev-dependencies]
    Dev-dependencies are not used when compiling a package for building, but are used for compiling tests, examples, and benchmarks.
    These dependencies are not propagated to other packages which depend on this package.

    \n 
    1
    2
    [target.'cfg(unix)'.dev-dependencies]
    mio = "0.0.1"
    \n
    • \n

  • build dependencies

    \n 
    1
    2
    [build-dependencies]
    cc = "1.0.3"
    \n

    The build script does not have access to the dependencies listed in the dependencies or dev-dependencies section. Build dependencies will likewise not be available to the package itself unless listed under the dependencies section as well.

    \n
    • \n

  • choosing features

    \n 
    1
    2
    3
    4
    5
    [dependencies.awesome]
    version = "1.3.5"
    default-features = false # do not include the default features, and optionally
                            # cherry-pick individual features
    features = ["secure-password", "civet"]
    • \n

  • renaming dependencies in Cargo.toml
    When writing a [dependencies] section in Cargo.toml the key you write for a dependency typically matches up to the name of the crate you import from in the code. For some projects, though, you may wish to reference the crate with a different name in the code regardless of how it’s published on crates.io. For example you may wish to: Avoid the need to use foo as bar in Rust source.
    (more to be found in original book)

    \n
    • \n
\n

references

cargo book

\n","site":{"data":{}},"excerpt":"","more":"

useful cmd

1
2
cargo new ${crate_name} --lib ## create a lib crate
cargo build --verbose ## print out each rustc invocation
\n\n

specifying dependencies

    \n

  • specifying dependencies from crates.io

    \n
    1
    2
    [dependencies]
    time = "0.1.12"
    • \n

  • specifying dependencies from other registries

    \n
    1
    2
    [dependencies]
    some-crate = { version = "1.0", registry = "my-registry" }
    • \n

  • specifying dependencies form git repositories

    \n
    1
    2
    [dependencies]
    regex = { git = "https://github.com/rust-lang/regex.git" }
    • \n

  • path dependencies

    \n
    1
    2
    [dependencies]
    hello_utils = { path = "hello_utils" }
    • \n

  • platform specific dependencies

    \n
    1
    2
    3
    4
    5
    [target.'cfg(unix)'.dependencies]
    openssl = "1.0.1"

    [target.'cfg(target_arch = "x86")'.dependencies]
    native-i686 = { path = "native/i686" }
    \n

    Like with Rust, the syntax here supports the not, any, and all operators to combine various cfg name/value pairs.
    If you want to know which cfg targets are available on your platform, run rustc --print=cfg from the command line.
    If you want to know which cfg targets are available for another platform, such as 64-bit Windows, run rustc --print=cfg --target=x86_64-pc-windows-msvc

    \n
    • \n

  • custom target specifications

    \n
    1
    2
    3
    4
    5
    6
    [target.bar.dependencies]
    winhttp = "0.4.0"

    [target.my-special-i686-platform.dependencies]
    openssl = "1.0.1"
    native = { path = "native/i686" }
    • \n

  • development dependencies
    [dev-dependencies]
    Dev-dependencies are not used when compiling a package for building, but are used for compiling tests, examples, and benchmarks.
    These dependencies are not propagated to other packages which depend on this package.

    \n 
    1
    2
    [target.'cfg(unix)'.dev-dependencies]
    mio = "0.0.1"
    \n
    • \n

  • build dependencies

    \n 
    1
    2
    [build-dependencies]
    cc = "1.0.3"
    \n

    The build script does not have access to the dependencies listed in the dependencies or dev-dependencies section. Build dependencies will likewise not be available to the package itself unless listed under the dependencies section as well.

    \n
    • \n

  • choosing features

    \n 
    1
    2
    3
    4
    5
    [dependencies.awesome]
    version = "1.3.5"
    default-features = false # do not include the default features, and optionally
                            # cherry-pick individual features
    features = ["secure-password", "civet"]
    • \n

  • renaming dependencies in Cargo.toml
    When writing a [dependencies] section in Cargo.toml the key you write for a dependency typically matches up to the name of the crate you import from in the code. For some projects, though, you may wish to reference the crate with a different name in the code regardless of how it’s published on crates.io. For example you may wish to: Avoid the need to use foo as bar in Rust source.
    (more to be found in original book)

    \n
    • \n
\n

references

cargo book

\n"},{"title":"rust similar concepts comparison","date":"2022-11-23T07:52:34.000Z","_content":"\n## ref vs &\n`ref` annotates pattern bindings to make them borrow rather than move. It is **not** a part of the pattern as far as matching is concerned: it does not affect whether a value is matched, only how it is matched.\nBy default, match statements consume all they can, which can sometimes be a problem, when you don't really need the value to be moved and owned:\n```rust\nlet maybe_name = Some(String::from(\"Alice\"));\n// Using `ref`, the value is borrowed, not moved ...\nmatch maybe_name {\n Some(ref n) => println!(\"Hello, {n}\"),\n _ => println!(\"Hello, world\"),\n}\n// ... so it's available here!\nprintln!(\"Hello again, {}\", maybe_name.unwrap_or(\"world\".into()));\n```\n\n- `&` denotes that your pattern expects a reference to an object. Hence `&` is a part of said pattern: `&Foo` matches different objects than `Foo` does.\n- `ref` indicates that you want a reference to an unpacked value. It is not matched against: `Foo(ref foo)` matches the same objects as `Foo(foo)`.\n\n## Clone vs Copy\n### Copy 的含义\n`Copy` 的全名是 `std::marker::Copy`。`std::marker` 这个模块里面的所有的 trait 都是特殊的。目前稳定的有四个，它们是 `Copy`、`Send`、`Sized`、`Sync`。它们的特殊之处在于它们是跟编译器密切绑定的，impl 这些 trait 对编译器的行为有重要影响。这几个 trait 内部都没有方法，它们的唯一任务是，给类型打一个“标记”，表明它符合某种约定。\n\n如果一个类型 impl 了 Copy trait，意味着任何时候，我们可以通过简单的内存拷贝(C语言的按位拷贝memcpy)实现该类型的复制，而不会产生任何问题。\n\n一旦一个类型实现了 Copy trait，那么它在变量绑定、函数参数传递、函数返回值传递等场景下，它都是 copy 语义，而不再是默认的 move 语义。\n\n### Copy 的实现条件\n并不是所有的类型都可以实现Copy trait。Rust规定，对于自定义类型，只有所有的成员都实现了 Copy trait，这个类型才有资格实现 Copy trait。\n\n常见的数字类型、bool类型、共享借用指针&，都是具有 Copy 属性的类型。而 Box、Vec、可写借用指针&mut 等类型都是不具备 Copy 属性的类型。\n\n我们可以认为，Rust中只有 POD(C++语言中的Plain Old Data) 类型才有资格实现Copy trait。在Rust中，如果一个类型只包含POD数据类型的成员，没有指针类型的成员，并且没有自定义析构函数（实现Drop trait），那它就是POD类型。比如整数、浮点数、只包含POD类型的数组等。而Box、 String、 Vec等，不能按 bit 位拷贝的类型，都不属于POD类型。但是，反过来讲，并不是所有的POD类型都应该实现Copy trait。\n\n### Clone 的含义\nClone 的全名是 std::clone::Clone。它的完整声明是这样的：\n```rust\npub trait Clone : Sized {\n fn clone(&self) -> Self;\n fn clone_from(&mut self, source: &Self) {\n *self = source.clone()\n }\n}\n```\n它有两个关联方法，其中 clone_from 是有默认实现的，它依赖于 clone 方法的实现。clone 方法没有默认实现，需要我们手动实现。\n\nclone 方法一般用于“基于语义的复制”操作。所以，它做什么事情，跟具体类型的作用息息相关。比如对于 Box 类型，clone 就是执行的“深拷贝”，而对于 Rc 类型，clone 做的事情就是把引用计数值加1。\n\n虽然说，Rust中 clone 方法一般是用来执行复制操作的，但是你如果在自定义的 clone 函数中做点什么别的工作编译器也没法禁止，你可以根据情况在 clone 函数中编写任意的逻辑。但是有一条规则需要注意：对于实现了 Copy 的类型，它的 clone 方法应该跟 Copy 语义相容，等同于按位拷贝。\n\n### 自动 derive\n绝大多数情况下，实现 Copy Clone 这样的 trait 都是一个重复而无聊的工作。因此，Rust提供了一个 attribute，让我们可以利用编译器自动生成这部分代码。示例如下：\n\n```rust\n#[derive(Copy, Clone)]\nstruct MyStruct(i32);\n```\n这里的 derive 会让编译器帮我们自动生成 impl Copy 和 impl Clone 这样的代码。自动生成的 clone 方法，就是依次调用每个成员的 clone 方法。\n\n通过 derive 方式自动实现 Copy 和手工实现 Copy 有一丁点的微小区别。当类型具有泛型参数的时候，比如 struct MyStruct{}，通过 derive 自动生成的代码会自动添加一个 T: Copy 的约束。\n\n目前，只有一部分固定的特殊 trait 可以通过 derive 来自动实现。将来 Rust 会允许自定义的 derive 行为，让我们自己的 trait 也可以通过 derive 的方式自动实现。\n\n## Cell vs RefCell\n- Cell 是操作T(values), RefCell操作&T(references). Cell get的时候要求T impl Copy。比如String类型没有实现Copy trait, 那么Cell::new(String::from(\"Hello\")).get()会报错\n- Cell 在编译器检查，运行时不会panic；RefCell在运行时检查，使用不当会发生panic\n- 一般来说，Cell内部实现会发生内存的分配，性能较之RefCell有点大\n\n## AsRef vs Borrow\n[WIP]\n","source":"_posts/rust/rust-similar-concepts-comparison.md","raw":"---\ntitle: rust similar concepts comparison\ndate: 2022-11-23 15:52:34\ntags: [rust]\n---\n\n## ref vs &\n`ref` annotates pattern bindings to make them borrow rather than move. It is **not** a part of the pattern as far as matching is concerned: it does not affect whether a value is matched, only how it is matched.\nBy default, match statements consume all they can, which can sometimes be a problem, when you don't really need the value to be moved and owned:\n```rust\nlet maybe_name = Some(String::from(\"Alice\"));\n// Using `ref`, the value is borrowed, not moved ...\nmatch maybe_name {\n Some(ref n) => println!(\"Hello, {n}\"),\n _ => println!(\"Hello, world\"),\n}\n// ... so it's available here!\nprintln!(\"Hello again, {}\", maybe_name.unwrap_or(\"world\".into()));\n```\n\n- `&` denotes that your pattern expects a reference to an object. Hence `&` is a part of said pattern: `&Foo` matches different objects than `Foo` does.\n- `ref` indicates that you want a reference to an unpacked value. It is not matched against: `Foo(ref foo)` matches the same objects as `Foo(foo)`.\n\n## Clone vs Copy\n### Copy 的含义\n`Copy` 的全名是 `std::marker::Copy`。`std::marker` 这个模块里面的所有的 trait 都是特殊的。目前稳定的有四个，它们是 `Copy`、`Send`、`Sized`、`Sync`。它们的特殊之处在于它们是跟编译器密切绑定的，impl 这些 trait 对编译器的行为有重要影响。这几个 trait 内部都没有方法，它们的唯一任务是，给类型打一个“标记”，表明它符合某种约定。\n\n如果一个类型 impl 了 Copy trait，意味着任何时候，我们可以通过简单的内存拷贝(C语言的按位拷贝memcpy)实现该类型的复制，而不会产生任何问题。\n\n一旦一个类型实现了 Copy trait，那么它在变量绑定、函数参数传递、函数返回值传递等场景下，它都是 copy 语义，而不再是默认的 move 语义。\n\n### Copy 的实现条件\n并不是所有的类型都可以实现Copy trait。Rust规定，对于自定义类型，只有所有的成员都实现了 Copy trait，这个类型才有资格实现 Copy trait。\n\n常见的数字类型、bool类型、共享借用指针&，都是具有 Copy 属性的类型。而 Box、Vec、可写借用指针&mut 等类型都是不具备 Copy 属性的类型。\n\n我们可以认为，Rust中只有 POD(C++语言中的Plain Old Data) 类型才有资格实现Copy trait。在Rust中，如果一个类型只包含POD数据类型的成员，没有指针类型的成员，并且没有自定义析构函数（实现Drop trait），那它就是POD类型。比如整数、浮点数、只包含POD类型的数组等。而Box、 String、 Vec等，不能按 bit 位拷贝的类型，都不属于POD类型。但是，反过来讲，并不是所有的POD类型都应该实现Copy trait。\n\n### Clone 的含义\nClone 的全名是 std::clone::Clone。它的完整声明是这样的：\n```rust\npub trait Clone : Sized {\n fn clone(&self) -> Self;\n fn clone_from(&mut self, source: &Self) {\n *self = source.clone()\n }\n}\n```\n它有两个关联方法，其中 clone_from 是有默认实现的，它依赖于 clone 方法的实现。clone 方法没有默认实现，需要我们手动实现。\n\nclone 方法一般用于“基于语义的复制”操作。所以，它做什么事情，跟具体类型的作用息息相关。比如对于 Box 类型，clone 就是执行的“深拷贝”，而对于 Rc 类型，clone 做的事情就是把引用计数值加1。\n\n虽然说，Rust中 clone 方法一般是用来执行复制操作的，但是你如果在自定义的 clone 函数中做点什么别的工作编译器也没法禁止，你可以根据情况在 clone 函数中编写任意的逻辑。但是有一条规则需要注意：对于实现了 Copy 的类型，它的 clone 方法应该跟 Copy 语义相容，等同于按位拷贝。\n\n### 自动 derive\n绝大多数情况下，实现 Copy Clone 这样的 trait 都是一个重复而无聊的工作。因此，Rust提供了一个 attribute，让我们可以利用编译器自动生成这部分代码。示例如下：\n\n```rust\n#[derive(Copy, Clone)]\nstruct MyStruct(i32);\n```\n这里的 derive 会让编译器帮我们自动生成 impl Copy 和 impl Clone 这样的代码。自动生成的 clone 方法，就是依次调用每个成员的 clone 方法。\n\n通过 derive 方式自动实现 Copy 和手工实现 Copy 有一丁点的微小区别。当类型具有泛型参数的时候，比如 struct MyStruct{}，通过 derive 自动生成的代码会自动添加一个 T: Copy 的约束。\n\n目前，只有一部分固定的特殊 trait 可以通过 derive 来自动实现。将来 Rust 会允许自定义的 derive 行为，让我们自己的 trait 也可以通过 derive 的方式自动实现。\n\n## Cell vs RefCell\n- Cell 是操作T(values), RefCell操作&T(references). Cell get的时候要求T impl Copy。比如String类型没有实现Copy trait, 那么Cell::new(String::from(\"Hello\")).get()会报错\n- Cell 在编译器检查，运行时不会panic；RefCell在运行时检查，使用不当会发生panic\n- 一般来说，Cell内部实现会发生内存的分配，性能较之RefCell有点大\n\n## AsRef vs Borrow\n[WIP]\n","slug":"rust/rust-similar-concepts-comparison","published":1,"updated":"2023-11-05T04:21:13.735Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8dt001kqwsj843kh526","content":"

ref vs &

ref annotates pattern bindings to make them borrow rather than move. It is not a part of the pattern as far as matching is concerned: it does not affect whether a value is matched, only how it is matched.
By default, match statements consume all they can, which can sometimes be a problem, when you don’t really need the value to be moved and owned:

\n
1
2
3
4
5
6
7
8
let maybe_name = Some(String::from("Alice"));
// Using `ref`, the value is borrowed, not moved ...
match maybe_name {
    Some(ref n) => println!("Hello, {n}"),
    _ => println!("Hello, world"),
}
// ... so it's available here!
println!("Hello again, {}", maybe_name.unwrap_or("world".into()));
\n\n
    \n
  • & denotes that your pattern expects a reference to an object. Hence & is a part of said pattern: &Foo matches different objects than Foo does.
    • \n
  • ref indicates that you want a reference to an unpacked value. It is not matched against: Foo(ref foo) matches the same objects as Foo(foo).
    • \n
\n

Clone vs Copy

Copy 的含义

Copy 的全名是 std::marker::Copy。std::marker 这个模块里面的所有的 trait 都是特殊的。目前稳定的有四个，它们是 Copy、Send、Sized、Sync。它们的特殊之处在于它们是跟编译器密切绑定的，impl 这些 trait 对编译器的行为有重要影响。这几个 trait 内部都没有方法，它们的唯一任务是，给类型打一个“标记”，表明它符合某种约定。

\n

如果一个类型 impl 了 Copy trait，意味着任何时候，我们可以通过简单的内存拷贝(C语言的按位拷贝memcpy)实现该类型的复制，而不会产生任何问题。

\n

一旦一个类型实现了 Copy trait，那么它在变量绑定、函数参数传递、函数返回值传递等场景下，它都是 copy 语义，而不再是默认的 move 语义。

\n

Copy 的实现条件

并不是所有的类型都可以实现Copy trait。Rust规定，对于自定义类型，只有所有的成员都实现了 Copy trait，这个类型才有资格实现 Copy trait。

\n

常见的数字类型、bool类型、共享借用指针&，都是具有 Copy 属性的类型。而 Box、Vec、可写借用指针&mut 等类型都是不具备 Copy 属性的类型。

\n

我们可以认为，Rust中只有 POD(C++语言中的Plain Old Data) 类型才有资格实现Copy trait。在Rust中，如果一个类型只包含POD数据类型的成员，没有指针类型的成员，并且没有自定义析构函数（实现Drop trait），那它就是POD类型。比如整数、浮点数、只包含POD类型的数组等。而Box、 String、 Vec等，不能按 bit 位拷贝的类型，都不属于POD类型。但是，反过来讲，并不是所有的POD类型都应该实现Copy trait。

\n

Clone 的含义

Clone 的全名是 std::clone::Clone。它的完整声明是这样的：

\n
1
2
3
4
5
6
pub trait Clone : Sized {
    fn clone(&self) -> Self;
    fn clone_from(&mut self, source: &Self) {
        *self = source.clone()
    }
}
\n

它有两个关联方法，其中 clone_from 是有默认实现的，它依赖于 clone 方法的实现。clone 方法没有默认实现，需要我们手动实现。

\n

clone 方法一般用于“基于语义的复制”操作。所以，它做什么事情，跟具体类型的作用息息相关。比如对于 Box 类型，clone 就是执行的“深拷贝”，而对于 Rc 类型，clone 做的事情就是把引用计数值加1。

\n

虽然说，Rust中 clone 方法一般是用来执行复制操作的，但是你如果在自定义的 clone 函数中做点什么别的工作编译器也没法禁止，你可以根据情况在 clone 函数中编写任意的逻辑。但是有一条规则需要注意：对于实现了 Copy 的类型，它的 clone 方法应该跟 Copy 语义相容，等同于按位拷贝。

\n

自动 derive

绝大多数情况下，实现 Copy Clone 这样的 trait 都是一个重复而无聊的工作。因此，Rust提供了一个 attribute，让我们可以利用编译器自动生成这部分代码。示例如下：

\n
1
2
#[derive(Copy, Clone)]
struct MyStruct(i32);
\n

这里的 derive 会让编译器帮我们自动生成 impl Copy 和 impl Clone 这样的代码。自动生成的 clone 方法，就是依次调用每个成员的 clone 方法。

\n

通过 derive 方式自动实现 Copy 和手工实现 Copy 有一丁点的微小区别。当类型具有泛型参数的时候，比如 struct MyStruct{}，通过 derive 自动生成的代码会自动添加一个 T: Copy 的约束。

\n

目前，只有一部分固定的特殊 trait 可以通过 derive 来自动实现。将来 Rust 会允许自定义的 derive 行为，让我们自己的 trait 也可以通过 derive 的方式自动实现。

\n

Cell vs RefCell

    \n
  • Cell 是操作T(values), RefCell操作&T(references). Cell get的时候要求T impl Copy。比如String类型没有实现Copy trait, 那么Cell::new(String::from(“Hello”)).get()会报错
    • \n
  • Cell 在编译器检查，运行时不会panic；RefCell在运行时检查，使用不当会发生panic
    • \n
  • 一般来说，Cell内部实现会发生内存的分配，性能较之RefCell有点大
    • \n
\n

AsRef vs Borrow

[WIP]

\n","site":{"data":{}},"excerpt":"","more":"

ref vs &

ref annotates pattern bindings to make them borrow rather than move. It is not a part of the pattern as far as matching is concerned: it does not affect whether a value is matched, only how it is matched.
By default, match statements consume all they can, which can sometimes be a problem, when you don’t really need the value to be moved and owned:

\n
1
2
3
4
5
6
7
8
let maybe_name = Some(String::from("Alice"));
// Using `ref`, the value is borrowed, not moved ...
match maybe_name {
    Some(ref n) => println!("Hello, {n}"),
    _ => println!("Hello, world"),
}
// ... so it's available here!
println!("Hello again, {}", maybe_name.unwrap_or("world".into()));
\n\n
    \n
  • & denotes that your pattern expects a reference to an object. Hence & is a part of said pattern: &Foo matches different objects than Foo does.
    • \n
  • ref indicates that you want a reference to an unpacked value. It is not matched against: Foo(ref foo) matches the same objects as Foo(foo).
    • \n
\n

Clone vs Copy

Copy 的含义

Copy 的全名是 std::marker::Copy。std::marker 这个模块里面的所有的 trait 都是特殊的。目前稳定的有四个，它们是 Copy、Send、Sized、Sync。它们的特殊之处在于它们是跟编译器密切绑定的，impl 这些 trait 对编译器的行为有重要影响。这几个 trait 内部都没有方法，它们的唯一任务是，给类型打一个“标记”，表明它符合某种约定。

\n

如果一个类型 impl 了 Copy trait，意味着任何时候，我们可以通过简单的内存拷贝(C语言的按位拷贝memcpy)实现该类型的复制，而不会产生任何问题。

\n

一旦一个类型实现了 Copy trait，那么它在变量绑定、函数参数传递、函数返回值传递等场景下，它都是 copy 语义，而不再是默认的 move 语义。

\n

Copy 的实现条件

并不是所有的类型都可以实现Copy trait。Rust规定，对于自定义类型，只有所有的成员都实现了 Copy trait，这个类型才有资格实现 Copy trait。

\n

常见的数字类型、bool类型、共享借用指针&，都是具有 Copy 属性的类型。而 Box、Vec、可写借用指针&mut 等类型都是不具备 Copy 属性的类型。

\n

我们可以认为，Rust中只有 POD(C++语言中的Plain Old Data) 类型才有资格实现Copy trait。在Rust中，如果一个类型只包含POD数据类型的成员，没有指针类型的成员，并且没有自定义析构函数（实现Drop trait），那它就是POD类型。比如整数、浮点数、只包含POD类型的数组等。而Box、 String、 Vec等，不能按 bit 位拷贝的类型，都不属于POD类型。但是，反过来讲，并不是所有的POD类型都应该实现Copy trait。

\n

Clone 的含义

Clone 的全名是 std::clone::Clone。它的完整声明是这样的：

\n
1
2
3
4
5
6
pub trait Clone : Sized {
    fn clone(&self) -> Self;
    fn clone_from(&mut self, source: &Self) {
        *self = source.clone()
    }
}
\n

它有两个关联方法，其中 clone_from 是有默认实现的，它依赖于 clone 方法的实现。clone 方法没有默认实现，需要我们手动实现。

\n

clone 方法一般用于“基于语义的复制”操作。所以，它做什么事情，跟具体类型的作用息息相关。比如对于 Box 类型，clone 就是执行的“深拷贝”，而对于 Rc 类型，clone 做的事情就是把引用计数值加1。

\n

虽然说，Rust中 clone 方法一般是用来执行复制操作的，但是你如果在自定义的 clone 函数中做点什么别的工作编译器也没法禁止，你可以根据情况在 clone 函数中编写任意的逻辑。但是有一条规则需要注意：对于实现了 Copy 的类型，它的 clone 方法应该跟 Copy 语义相容，等同于按位拷贝。

\n

自动 derive

绝大多数情况下，实现 Copy Clone 这样的 trait 都是一个重复而无聊的工作。因此，Rust提供了一个 attribute，让我们可以利用编译器自动生成这部分代码。示例如下：

\n
1
2
#[derive(Copy, Clone)]
struct MyStruct(i32);
\n

这里的 derive 会让编译器帮我们自动生成 impl Copy 和 impl Clone 这样的代码。自动生成的 clone 方法，就是依次调用每个成员的 clone 方法。

\n

通过 derive 方式自动实现 Copy 和手工实现 Copy 有一丁点的微小区别。当类型具有泛型参数的时候，比如 struct MyStruct{}，通过 derive 自动生成的代码会自动添加一个 T: Copy 的约束。

\n

目前，只有一部分固定的特殊 trait 可以通过 derive 来自动实现。将来 Rust 会允许自定义的 derive 行为，让我们自己的 trait 也可以通过 derive 的方式自动实现。

\n

Cell vs RefCell

    \n
  • Cell 是操作T(values), RefCell操作&T(references). Cell get的时候要求T impl Copy。比如String类型没有实现Copy trait, 那么Cell::new(String::from(“Hello”)).get()会报错
    • \n
  • Cell 在编译器检查，运行时不会panic；RefCell在运行时检查，使用不当会发生panic
    • \n
  • 一般来说，Cell内部实现会发生内存的分配，性能较之RefCell有点大
    • \n
\n

AsRef vs Borrow

[WIP]

\n"},{"title":"cargo doc","date":"2022-11-13T07:41:59.000Z","_content":"\n## 文档注释\n### 用于生成文档\n - 使用 ///\n - 支持 Markdown\n - 放置在被说没条目之前\n### 例子\n```rust\n/// adds one to the number given\n/// \n/// # Examples\n/// ```\n/// let arg = 5;\n/// let answer = my_crate::add_one(arg);\n/// \n/// assert_eq!(6, answer);\n/// ```\npub fn add_one(x: i32) -> i32 {\n x + 1;\n}\n```\n### 命令\n```\ncargo doc ## 生成的html放在 target/doc 目录下\ncargo doc --open ## 构建当前crate的文档 (也包含crate依赖项的文档)\n```\n### 常用章节\n- `# Examples`\n- `Panics`: 可能panic的场景\n- `Errors`: 如果fn返回Result, 描述可能的错误种类, 以及导致错误的条件\n- `Safety`: 如果fn出入unsafe调用, 解释unsafe的原因, 以及调用者确保的使用前提\n\n### 文档注释作为测试\n- 运行cargo test, doc中用# Example标记的实例代码会用来测试运行\n\n### 为包含注释的项添加文档注释\n- 符号: //!\n- 这类注释通常用描述crate和模块:\n - crate root (按惯例 src/lib.rs)\n - 一个模块内，将crate火模块作为一个整体进行记录\n\n## 注释\n//! - 模块级稳定注释, 置于模块头部\n//!! - 模块级稳定注释, 但是和上面注释置于同一行\n\n//! - 模块级稳定注释, 会换行\n\n/*! - 模块级稳定注释 */\n/*!！ - 模块级稳定注释, 和上面同一行 */\n\n// 普通行注释\n/// 行级文档注释\n//// 普通行注释\n\n/* 普通块级注释 */\n/** 会级文档注释 */","source":"_posts/rust/rust-cargo-doc.md","raw":"---\ntitle: cargo doc\ndate: 2022-11-13 15:41:59\ntags: [rust,cargo]\n---\n\n## 文档注释\n### 用于生成文档\n - 使用 ///\n - 支持 Markdown\n - 放置在被说没条目之前\n### 例子\n```rust\n/// adds one to the number given\n/// \n/// # Examples\n/// ```\n/// let arg = 5;\n/// let answer = my_crate::add_one(arg);\n/// \n/// assert_eq!(6, answer);\n/// ```\npub fn add_one(x: i32) -> i32 {\n x + 1;\n}\n```\n### 命令\n```\ncargo doc ## 生成的html放在 target/doc 目录下\ncargo doc --open ## 构建当前crate的文档 (也包含crate依赖项的文档)\n```\n### 常用章节\n- `# Examples`\n- `Panics`: 可能panic的场景\n- `Errors`: 如果fn返回Result, 描述可能的错误种类, 以及导致错误的条件\n- `Safety`: 如果fn出入unsafe调用, 解释unsafe的原因, 以及调用者确保的使用前提\n\n### 文档注释作为测试\n- 运行cargo test, doc中用# Example标记的实例代码会用来测试运行\n\n### 为包含注释的项添加文档注释\n- 符号: //!\n- 这类注释通常用描述crate和模块:\n - crate root (按惯例 src/lib.rs)\n - 一个模块内，将crate火模块作为一个整体进行记录\n\n## 注释\n//! - 模块级稳定注释, 置于模块头部\n//!! - 模块级稳定注释, 但是和上面注释置于同一行\n\n//! - 模块级稳定注释, 会换行\n\n/*! - 模块级稳定注释 */\n/*!！ - 模块级稳定注释, 和上面同一行 */\n\n// 普通行注释\n/// 行级文档注释\n//// 普通行注释\n\n/* 普通块级注释 */\n/** 会级文档注释 */","slug":"rust/rust-cargo-doc","published":1,"updated":"2023-11-05T04:21:13.735Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8dt001mqwsjfk9x8s37","content":"

文档注释

用于生成文档

    \n
  • 使用 ///
    • \n
  • 支持 Markdown
    • \n
  • 放置在被说没条目之前
    • \n
\n

例子

1
2
3
4
5
6
7
8
9
10
11
12
/// adds one to the number given
/// 
/// # Examples
/// ```
/// let arg = 5;
/// let answer = my_crate::add_one(arg);
/// 
/// assert_eq!(6, answer);
/// ```
pub fn add_one(x: i32) -> i32 {
    x + 1;
}
\n

命令

1
2
cargo doc  ## 生成的html放在 target/doc 目录下
cargo doc --open ## 构建当前crate的文档 (也包含crate依赖项的文档)
\n

常用章节

    \n
  • # Examples
    • \n
  • Panics: 可能panic的场景
    • \n
  • Errors: 如果fn返回Result, 描述可能的错误种类, 以及导致错误的条件
    • \n
  • Safety: 如果fn出入unsafe调用, 解释unsafe的原因, 以及调用者确保的使用前提
    • \n
\n

文档注释作为测试

    \n
  • 运行cargo test, doc中用# Example标记的实例代码会用来测试运行
    • \n
\n

为包含注释的项添加文档注释

    \n
  • 符号: //!
    • \n
  • 这类注释通常用描述crate和模块:
      \n
    • crate root (按惯例 src/lib.rs)
      • \n
    • 一个模块内，将crate火模块作为一个整体进行记录
      • \n
    \n
    • \n
\n

注释

//! - 模块级稳定注释, 置于模块头部
//!! - 模块级稳定注释, 但是和上面注释置于同一行

\n

//! - 模块级稳定注释, 会换行

\n

/*! - 模块级稳定注释 /
/!！ - 模块级稳定注释, 和上面同一行 */

\n

// 普通行注释
/// 行级文档注释
//// 普通行注释

\n

/* 普通块级注释 /
/* 会级文档注释 */

\n","site":{"data":{}},"excerpt":"","more":"

文档注释

用于生成文档

    \n
  • 使用 ///
    • \n
  • 支持 Markdown
    • \n
  • 放置在被说没条目之前
    • \n
\n

例子

1
2
3
4
5
6
7
8
9
10
11
12
/// adds one to the number given
/// 
/// # Examples
/// ```
/// let arg = 5;
/// let answer = my_crate::add_one(arg);
/// 
/// assert_eq!(6, answer);
/// ```
pub fn add_one(x: i32) -> i32 {
    x + 1;
}
\n

命令

1
2
cargo doc  ## 生成的html放在 target/doc 目录下
cargo doc --open ## 构建当前crate的文档 (也包含crate依赖项的文档)
\n

常用章节

    \n
  • # Examples
    • \n
  • Panics: 可能panic的场景
    • \n
  • Errors: 如果fn返回Result, 描述可能的错误种类, 以及导致错误的条件
    • \n
  • Safety: 如果fn出入unsafe调用, 解释unsafe的原因, 以及调用者确保的使用前提
    • \n
\n

文档注释作为测试

    \n
  • 运行cargo test, doc中用# Example标记的实例代码会用来测试运行
    • \n
\n

为包含注释的项添加文档注释

    \n
  • 符号: //!
    • \n
  • 这类注释通常用描述crate和模块:
      \n
    • crate root (按惯例 src/lib.rs)
      • \n
    • 一个模块内，将crate火模块作为一个整体进行记录
      • \n
    \n
    • \n
\n

注释

//! - 模块级稳定注释, 置于模块头部
//!! - 模块级稳定注释, 但是和上面注释置于同一行

\n

//! - 模块级稳定注释, 会换行

\n

/*! - 模块级稳定注释 /
/!！ - 模块级稳定注释, 和上面同一行 */

\n

// 普通行注释
/// 行级文档注释
//// 普通行注释

\n

/* 普通块级注释 /
/* 会级文档注释 */

\n"},{"title":"rust sugar","date":"2022-11-17T07:47:13.000Z","_content":"\n## formatted print\n```rust\n// Positional arguments can be used\nprintln!(\"{0}, this is {1}. {1}, this is {0}\", \"Alice\", \"Bob\");\n// As can named arguments.\nprintln!(\"{subject} {verb} {object}\",\n object=\"the lazy dog\",\n subject=\"the quick brown fox\",\n verb=\"jumps over\");\n// Different formatting can be invoked by specifying the format character\n// after a `:`.\nprintln!(\"Base 10: {}\", 69420); // 69420\nprintln!(\"Base 2 (binary): {:b}\", 69420); // 10000111100101100\nprintln!(\"Base 8 (octal): {:o}\", 69420); // 207454\nprintln!(\"Base 16 (hexadecimal): {:x}\", 69420); // 10f2c\nprintln!(\"Base 16 (hexadecimal): {:X}\", 69420); // 10F2C\n// You can right-justify text with a specified width. This will\n// output \" 1\". (Four white spaces and a \"1\", for a total width of 5.)\nprintln!(\"{number:>5}\", number=1);\n// You can pad numbers with extra zeroes,\n// and left-adjust by flipping the sign. This will output \"10000\".\nprintln!(\"{number:0<5}\", number=1);\n// You can use named arguments in the format specifier by appending a `$`.\nprintln!(\"{number:0>width$}\", number=1, width=5);\n```\n- [reference](https://doc.rust-lang.org/rust-by-example/hello/print.html)\n\n\n## syntax\n```rust\n/// print to stderr\neprintln!(\"server error: {}\", e);\n\nstruct MyTupleStruct(M);\nlet myTupleStruct = MyTupleStruct::(String::from(\"hello\"));\n\nvec.iter().position()\nvec.iter().find()\nvec.iter().any()\n\n\nstatic mut A: u32 = 0;\n```\n\n## attributes\n```rust\n#![allow(warnings)]\n#[allow(dead_code)]\n#![allow(unused)]\n// Suppress all warnings from casts which overflow.\n#![allow(overflowing_literals)]\n#![allow(unreachable_code)]\n```\n\n## memory\nThe compiler will not rearrange the memory layout\n```rust\n#[repr(C)]\nstruct A {\n a:u8,\n b:u32,\n c:u16\n}\n```\n\n## ptr\n```rust\nNonNull::new_unchecked()\n\n#![feature(new_uninit)]\nlet mut five = Box::::new_uninit();\nlet five = unsafe {\n // Deferred initialization:\n five.as_mut_ptr().write(5);\n five.assume_init()\n};\nassert_eq!(*five, 5)\n\nlet zero = Box::::new_zeroed();\nlet zero = unsafe { zero.assume_init() };\nassert_eq!(*zero, 0)\n\nuse std::alloc::{alloc, Layout};\nunsafe {\n let ptr = alloc(Layout::new::()) as *mut i32;\n ptr.write(5);\n let x = Box::from_raw(ptr);\n}\n```","source":"_posts/rust/rust-sugar.md","raw":"---\ntitle: rust sugar\ndate: 2022-11-17 15:47:13\ntags: [rust]\n---\n\n## formatted print\n```rust\n// Positional arguments can be used\nprintln!(\"{0}, this is {1}. {1}, this is {0}\", \"Alice\", \"Bob\");\n// As can named arguments.\nprintln!(\"{subject} {verb} {object}\",\n object=\"the lazy dog\",\n subject=\"the quick brown fox\",\n verb=\"jumps over\");\n// Different formatting can be invoked by specifying the format character\n// after a `:`.\nprintln!(\"Base 10: {}\", 69420); // 69420\nprintln!(\"Base 2 (binary): {:b}\", 69420); // 10000111100101100\nprintln!(\"Base 8 (octal): {:o}\", 69420); // 207454\nprintln!(\"Base 16 (hexadecimal): {:x}\", 69420); // 10f2c\nprintln!(\"Base 16 (hexadecimal): {:X}\", 69420); // 10F2C\n// You can right-justify text with a specified width. This will\n// output \" 1\". (Four white spaces and a \"1\", for a total width of 5.)\nprintln!(\"{number:>5}\", number=1);\n// You can pad numbers with extra zeroes,\n// and left-adjust by flipping the sign. This will output \"10000\".\nprintln!(\"{number:0<5}\", number=1);\n// You can use named arguments in the format specifier by appending a `$`.\nprintln!(\"{number:0>width$}\", number=1, width=5);\n```\n- [reference](https://doc.rust-lang.org/rust-by-example/hello/print.html)\n\n\n## syntax\n```rust\n/// print to stderr\neprintln!(\"server error: {}\", e);\n\nstruct MyTupleStruct(M);\nlet myTupleStruct = MyTupleStruct::(String::from(\"hello\"));\n\nvec.iter().position()\nvec.iter().find()\nvec.iter().any()\n\n\nstatic mut A: u32 = 0;\n```\n\n## attributes\n```rust\n#![allow(warnings)]\n#[allow(dead_code)]\n#![allow(unused)]\n// Suppress all warnings from casts which overflow.\n#![allow(overflowing_literals)]\n#![allow(unreachable_code)]\n```\n\n## memory\nThe compiler will not rearrange the memory layout\n```rust\n#[repr(C)]\nstruct A {\n a:u8,\n b:u32,\n c:u16\n}\n```\n\n## ptr\n```rust\nNonNull::new_unchecked()\n\n#![feature(new_uninit)]\nlet mut five = Box::::new_uninit();\nlet five = unsafe {\n // Deferred initialization:\n five.as_mut_ptr().write(5);\n five.assume_init()\n};\nassert_eq!(*five, 5)\n\nlet zero = Box::::new_zeroed();\nlet zero = unsafe { zero.assume_init() };\nassert_eq!(*zero, 0)\n\nuse std::alloc::{alloc, Layout};\nunsafe {\n let ptr = alloc(Layout::new::()) as *mut i32;\n ptr.write(5);\n let x = Box::from_raw(ptr);\n}\n```","slug":"rust/rust-sugar","published":1,"updated":"2023-11-05T04:21:13.735Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8du001pqwsj1k4uhzw0","content":"

formatted print

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
// Positional arguments can be used
println!("{0}, this is {1}. {1}, this is {0}", "Alice", "Bob");
// As can named arguments.
println!("{subject} {verb} {object}",
            object="the lazy dog",
            subject="the quick brown fox",
            verb="jumps over");
// Different formatting can be invoked by specifying the format character
// after a `:`.
println!("Base 10:               {}",   69420); // 69420
println!("Base 2 (binary):       {:b}", 69420); // 10000111100101100
println!("Base 8 (octal):        {:o}", 69420); // 207454
println!("Base 16 (hexadecimal): {:x}", 69420); // 10f2c
println!("Base 16 (hexadecimal): {:X}", 69420); // 10F2C
// You can right-justify text with a specified width. This will
// output "    1". (Four white spaces and a "1", for a total width of 5.)
println!("{number:>5}", number=1);
// You can pad numbers with extra zeroes,
// and left-adjust by flipping the sign. This will output "10000".
println!("{number:0<5}", number=1);
// You can use named arguments in the format specifier by appending a `$`.
println!("{number:0>width$}", number=1, width=5);
\n\n

syntax

1
2
3
4
5
6
7
8
9
10
11
12
/// print to stderr
eprintln!("server error: {}", e);

struct MyTupleStruct<M>(M);
let myTupleStruct =  MyTupleStruct::<String>(String::from("hello"));

vec.iter().position()
vec.iter().find()
vec.iter().any()


static mut A: u32 = 0;
\n\n

attributes

1
2
3
4
5
6
#![allow(warnings)]
#[allow(dead_code)]
#![allow(unused)]
// Suppress all warnings from casts which overflow.
#![allow(overflowing_literals)]
#![allow(unreachable_code)]
\n\n

memory

The compiler will not rearrange the memory layout

\n
1
2
3
4
5
6
#[repr(C)]
struct A {
    a:u8,
    b:u32,
    c:u16
}
\n\n

ptr

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
NonNull::new_unchecked()

#![feature(new_uninit)]
let mut five = Box::<u32>::new_uninit();
let five = unsafe {
    // Deferred initialization:
    five.as_mut_ptr().write(5);
    five.assume_init()
};
assert_eq!(*five, 5)

let zero = Box::<u32>::new_zeroed();
let zero = unsafe { zero.assume_init() };
assert_eq!(*zero, 0)

use std::alloc::{alloc, Layout};
unsafe {
    let ptr = alloc(Layout::new::<i32>()) as *mut i32;
    ptr.write(5);
    let x = Box::from_raw(ptr);
}
","site":{"data":{}},"excerpt":"","more":"

formatted print

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
// Positional arguments can be used
println!("{0}, this is {1}. {1}, this is {0}", "Alice", "Bob");
// As can named arguments.
println!("{subject} {verb} {object}",
            object="the lazy dog",
            subject="the quick brown fox",
            verb="jumps over");
// Different formatting can be invoked by specifying the format character
// after a `:`.
println!("Base 10:               {}",   69420); // 69420
println!("Base 2 (binary):       {:b}", 69420); // 10000111100101100
println!("Base 8 (octal):        {:o}", 69420); // 207454
println!("Base 16 (hexadecimal): {:x}", 69420); // 10f2c
println!("Base 16 (hexadecimal): {:X}", 69420); // 10F2C
// You can right-justify text with a specified width. This will
// output "    1". (Four white spaces and a "1", for a total width of 5.)
println!("{number:>5}", number=1);
// You can pad numbers with extra zeroes,
// and left-adjust by flipping the sign. This will output "10000".
println!("{number:0<5}", number=1);
// You can use named arguments in the format specifier by appending a `$`.
println!("{number:0>width$}", number=1, width=5);
\n\n

syntax

1
2
3
4
5
6
7
8
9
10
11
12
/// print to stderr
eprintln!("server error: {}", e);

struct MyTupleStruct<M>(M);
let myTupleStruct =  MyTupleStruct::<String>(String::from("hello"));

vec.iter().position()
vec.iter().find()
vec.iter().any()


static mut A: u32 = 0;
\n\n

attributes

1
2
3
4
5
6
#![allow(warnings)]
#[allow(dead_code)]
#![allow(unused)]
// Suppress all warnings from casts which overflow.
#![allow(overflowing_literals)]
#![allow(unreachable_code)]
\n\n

memory

The compiler will not rearrange the memory layout

\n
1
2
3
4
5
6
#[repr(C)]
struct A {
    a:u8,
    b:u32,
    c:u16
}
\n\n

ptr

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
NonNull::new_unchecked()

#![feature(new_uninit)]
let mut five = Box::<u32>::new_uninit();
let five = unsafe {
    // Deferred initialization:
    five.as_mut_ptr().write(5);
    five.assume_init()
};
assert_eq!(*five, 5)

let zero = Box::<u32>::new_zeroed();
let zero = unsafe { zero.assume_init() };
assert_eq!(*zero, 0)

use std::alloc::{alloc, Layout};
unsafe {
    let ptr = alloc(Layout::new::<i32>()) as *mut i32;
    ptr.write(5);
    let x = Box::from_raw(ptr);
}
"},{"title":"rust tools","date":"2022-11-20T09:14:05.000Z","_content":"\n## tools\n- cargo-edit\nThis tool extends Cargo to allow you to add, remove, and upgrade dependencies by modifying your Cargo.toml file from the command line\n\n- cargo whatfeatures ${crate}\n eg: `cargo whatfeatures hyper`\n","source":"_posts/rust/rust-tools.md","raw":"---\ntitle: rust tools\ndate: 2022-11-20 17:14:05\ntags: [rust]\n---\n\n## tools\n- cargo-edit\nThis tool extends Cargo to allow you to add, remove, and upgrade dependencies by modifying your Cargo.toml file from the command line\n\n- cargo whatfeatures ${crate}\n eg: `cargo whatfeatures hyper`\n","slug":"rust/rust-tools","published":1,"updated":"2023-11-05T04:21:13.735Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8du001rqwsj7syrcry3","content":"

tools

    \n

  • cargo-edit
    This tool extends Cargo to allow you to add, remove, and upgrade dependencies by modifying your Cargo.toml file from the command line

    \n
    • \n

  • cargo whatfeatures ${crate}
    eg: cargo whatfeatures hyper

    \n
    • \n
\n","site":{"data":{}},"excerpt":"","more":"

tools

    \n

  • cargo-edit
    This tool extends Cargo to allow you to add, remove, and upgrade dependencies by modifying your Cargo.toml file from the command line

    \n
    • \n

  • cargo whatfeatures ${crate}
    eg: cargo whatfeatures hyper

    \n
    • \n
\n"},{"title":"markdown & latex","date":"2023-10-01T01:22:35.000Z","_content":"\n\n\n\n## markdown by example\n_italic_\ncolor\n\n\n\n## latex by example\n## symbol\n\\\\( \\mu \\\\)\n\\\\( \\omega \\\\)\n\\\\( \\sigma \\\\)\n\\\\( \\epsilon \\\\)\n\\\\( \\zeta \\\\)\n\\\\( \\eta \\\\)\n\\\\( \\theta \\\\)\n\\\\( \\kappa \\\\)\n\\\\( \\nu \\\\)\n\\\\( \\omicron \\\\)\n\\\\( \\rho \\\\)\n\\\\( \\delta \\\\)\n\\\\( \\tau \\\\)\n\\\\( \\upsilon \\\\)\n\\\\( \\phi \\\\)\n\\\\( \\chi \\\\)\n\\\\( \\psi \\\\)\n\\\\(\\mathcal O\\\\)\n\n## font & effects\n\\\\( \\mathbb{G_{1}}\\\\)\n\\\\( \\boldsymbol{F}\\\\)\n\\\\( \\hat{h} \\\\)\n\n## equation\n\\\\[ \\tag{1.1} X = F \\cdot x^{T}\\\\]\n## blob\n\\\\[\n\\begin{bmatrix}\n\\displaylines{\n \\omega_{N}^{0\\cdot0} & \\omega_{N}^{0\\cdot1} & \\dots & \\omega_{N}^{0\\cdot (N-1)}\\\\\\\\\n \\omega_{N}^{1\\cdot0} & \\omega_{N}^{1\\cdot1} & \\dots & \\omega_{N}^{2\\cdot (N-1)}\\\\\\\\\n \\vdots & \\vdots & \\ddots & \\vdots\\\\\\\\\n \\omega_{N}^{N-1\\cdot0} & \\omega_{N}^{N-1\\cdot1} & \\dots & \\omega_{N}^{N-1\\cdot (N-1)}\n}\n\\end{bmatrix}\n\\\\]","source":"_posts/tools/markdown_and_latex.md","raw":"---\ntitle: markdown & latex\ndate: 2023-10-01 09:22:35\ntags: [tool]\n---\n\n\n\n\n## markdown by example\n_italic_\ncolor\n\n\n\n## latex by example\n## symbol\n\\\\( \\mu \\\\)\n\\\\( \\omega \\\\)\n\\\\( \\sigma \\\\)\n\\\\( \\epsilon \\\\)\n\\\\( \\zeta \\\\)\n\\\\( \\eta \\\\)\n\\\\( \\theta \\\\)\n\\\\( \\kappa \\\\)\n\\\\( \\nu \\\\)\n\\\\( \\omicron \\\\)\n\\\\( \\rho \\\\)\n\\\\( \\delta \\\\)\n\\\\( \\tau \\\\)\n\\\\( \\upsilon \\\\)\n\\\\( \\phi \\\\)\n\\\\( \\chi \\\\)\n\\\\( \\psi \\\\)\n\\\\(\\mathcal O\\\\)\n\n## font & effects\n\\\\( \\mathbb{G_{1}}\\\\)\n\\\\( \\boldsymbol{F}\\\\)\n\\\\( \\hat{h} \\\\)\n\n## equation\n\\\\[ \\tag{1.1} X = F \\cdot x^{T}\\\\]\n## blob\n\\\\[\n\\begin{bmatrix}\n\\displaylines{\n \\omega_{N}^{0\\cdot0} & \\omega_{N}^{0\\cdot1} & \\dots & \\omega_{N}^{0\\cdot (N-1)}\\\\\\\\\n \\omega_{N}^{1\\cdot0} & \\omega_{N}^{1\\cdot1} & \\dots & \\omega_{N}^{2\\cdot (N-1)}\\\\\\\\\n \\vdots & \\vdots & \\ddots & \\vdots\\\\\\\\\n \\omega_{N}^{N-1\\cdot0} & \\omega_{N}^{N-1\\cdot1} & \\dots & \\omega_{N}^{N-1\\cdot (N-1)}\n}\n\\end{bmatrix}\n\\\\]","slug":"tools/markdown_and_latex","published":1,"updated":"2023-12-01T06:06:41.564Z","_id":"clokyy8dv001zqwsj9hdfghdd","comments":1,"layout":"post","photos":[],"link":"","content":"\n\n\n

markdown by example

italic
color

\n

latex by example

symbol

\\( \\mu \\)
\\( \\omega \\)
\\( \\sigma \\)
\\( \\epsilon \\)
\\( \\zeta \\)
\\( \\eta \\)
\\( \\theta \\)
\\( \\kappa \\)
\\( \\nu \\)
\\( \\omicron \\)
\\( \\rho \\)
\\( \\delta \\)
\\( \\tau \\)
\\( \\upsilon \\)
\\( \\phi \\)
\\( \\chi \\)
\\( \\psi \\)
\\(\\mathcal O\\)

\n

font & effects

\\( \\mathbb{G_{1}}\\)
\\( \\boldsymbol{F}\\)
\\( \\hat{h} \\)

\n

equation

\\[ \\tag{1.1} X = F \\cdot x^{T}\\]

\n

blob

\\[
\\begin{bmatrix}
\\displaylines{
\\omega_{N}^{0\\cdot0} & \\omega_{N}^{0\\cdot1} & \\dots & \\omega_{N}^{0\\cdot (N-1)}\\\\
\\omega_{N}^{1\\cdot0} & \\omega_{N}^{1\\cdot1} & \\dots & \\omega_{N}^{2\\cdot (N-1)}\\\\
\\vdots & \\vdots & \\ddots & \\vdots\\\\
\\omega_{N}^{N-1\\cdot0} & \\omega_{N}^{N-1\\cdot1} & \\dots & \\omega_{N}^{N-1\\cdot (N-1)}
}
\\end{bmatrix}
\\]

\n","site":{"data":{}},"excerpt":"","more":"\n\n\n

markdown by example

italic
color

\n

latex by example

symbol

\\( \\mu \\)
\\( \\omega \\)
\\( \\sigma \\)
\\( \\epsilon \\)
\\( \\zeta \\)
\\( \\eta \\)
\\( \\theta \\)
\\( \\kappa \\)
\\( \\nu \\)
\\( \\omicron \\)
\\( \\rho \\)
\\( \\delta \\)
\\( \\tau \\)
\\( \\upsilon \\)
\\( \\phi \\)
\\( \\chi \\)
\\( \\psi \\)
\\(\\mathcal O\\)

\n

font & effects

\\( \\mathbb{G_{1}}\\)
\\( \\boldsymbol{F}\\)
\\( \\hat{h} \\)

\n

equation

\\[ \\tag{1.1} X = F \\cdot x^{T}\\]

\n

blob

\\[
\\begin{bmatrix}
\\displaylines{
\\omega_{N}^{0\\cdot0} & \\omega_{N}^{0\\cdot1} & \\dots & \\omega_{N}^{0\\cdot (N-1)}\\\\
\\omega_{N}^{1\\cdot0} & \\omega_{N}^{1\\cdot1} & \\dots & \\omega_{N}^{2\\cdot (N-1)}\\\\
\\vdots & \\vdots & \\ddots & \\vdots\\\\
\\omega_{N}^{N-1\\cdot0} & \\omega_{N}^{N-1\\cdot1} & \\dots & \\omega_{N}^{N-1\\cdot (N-1)}
}
\\end{bmatrix}
\\]

\n"},{"title":"fourier series, DFT and FFT","date":"2023-10-12T03:13:17.000Z","_content":"\n\n\n\n## Fourier Series\n### innter product of two functions\n\\\\[ = \\int_{a}^{b} f(x) \\bar{g}(x)dx\\\\]\n\\\\(\\bar{g}\\\\) is the congugate of g on complex number system\n\n### Fourier series definition\nlet \\\\(f(x)\\\\) be a periodic function over a period of \\\\([-\\pi, +\\pi]\\\\)\n![f(x)](/images/math/fourier_series/f_x.png)\n\n\\\\( f(x) \\\\) can be transformed into below\n\\\\[ f(x) = \\frac{A_{0}}{2} + \\sum_{k=1}^{\\infty}A_{k}cos(kx) + B_{k}sin(kx)\\\\],\nwhere\n\\\\[ A_{k} = \\frac{1}{\\pi} \\int_{-\\pi}^{+\\pi} f(x)cos(kx)dx = \\frac{1}{||cos(kx)||^2} \\\\]\n\\\\[ B_{k} = \\frac{1}{\\pi} \\int_{-\\pi}^{+\\pi} f(x)sin(kx)dx = \\frac{1}{||sin(kx)||^2} \\\\]\nwhere \\\\(A_{k}, B_{k}\\\\) is just the **innver product** of \\\\(f(x)\\\\), and \\\\( cos(kx) \\\\), \\\\(sin(kx)\\\\) respectively. intuitively, it is the **projection** of \\\\(f(x)\\\\) over \\\\(sin(kx)\\\\), or \\\\(cos(kx)\\\\). \n\n### if period is L\n\\\\[ f(x) = \\frac{A_{0}}{2} + \\sum_{k=1}^{\\infty}A_{k}cos(\\frac{2\\pi kx}{L}) + B_{k}sin(\\frac{2\\pi kx}{L})\\\\],\nwhere \n\\\\[ A_{k} = \\frac{2}{L} \\int_{0}^{L} f(x)cos(\\frac{2\\pi kx}{L})dx\\\\]\n\\\\[ B_{k} = \\frac{2}{L} \\int_{0}^{L} f(x)sin(\\frac{2\\pi kx}{L})dx\\\\]\n\n## Complex Fourier Series\nlet's define \\\\[ \\psi_k = e^{ikx} = cos(kx) + i \\cdot sin(kx)\\\\]\nwe can prove that \\\\(\\psi_k, and \\psi_j\\\\) are **orthogonal** as their inner product is zero\n**proof**\n\\\\[ <\\psi_j, \\psi_k> = \\int_{-\\pi}^{+\\pi} e^{jkx}e^{-ikx}dx = \\int_{-\\pi}^{+\\pi} e^{j(j-k)x}dx = \\frac{1}{i(j-k)}[e^{i(j-k)x}]_{-\\pi}^{\\pi}\\\\]\nit is 0 if \\\\(j \\ne k\\\\) , \\\\(2\\pi \\\\) if \\\\(j=k\\\\)\n\n**note** conjugate of \\\\(\\psi_k\\\\) is \\\\(e^{-ikx}\\\\)\n\n\\\\(f(x) = \\frac{1}{2\\pi} \\sum_{k=-\\infty}^{\\infty} c_k \\psi_k\\\\), where \\\\(c_k= = \\int_{-\\pi}^{+\\pi} f(x)e^{-ikx}dx\\\\)\n\n## Discrete Fourier Transform (DFT)\nThe discrete Fourier transform transforms a sequence of N complex numbers \n\\\\[\\lbrace x_{n} \\rbrace := x_0, x_1,..., x_{N-1} \\\\] into another sequence of complex numbers, \n\\\\[\\lbrace X_{k} \\rbrace := X_0, X_1,..., X_{N-1} \\\\] which is defined by\n\\\\[X_{k} = \\sum_{n=0}^{N-1} x_{n} \\cdot e^{-\\frac{i2\\pi}{N}kn}\\\\]\n\nlet the n-th root of unity be \\\\(\\omega_N = e^{-\\frac{2\\pi i}{N}}\\\\), we have\n\\\\[ \\tag{1.1} X_{k} = \\sum_{n=0}^{N-1} x_{n} \\cdot \\omega_N^{kn}\\\\]\n\n### inverse DFT\nThe inverse transform is given by\n\\\\[ \\tag{1.2} x_{n} = \\frac{1}{N}\\sum_{k=0}^{N-1} X_{k} \\cdot \\omega_N^{-kn}\\\\]\n\n\n### the unitary DFT\nAnother way of looking at the DFT is to note that the DFT can be expressed as the DFT matrix, a Vandermonde matrix, introduced by Sylvester in 1867.\n\\\\[\n \\boldsymbol{F} =\n\\begin{bmatrix}\n\\displaylines{\n \\omega_{N}^{0\\cdot0} & \\omega_{N}^{0\\cdot1} & \\dots & \\omega_{N}^{0\\cdot (N-1)}\\\\\\\\\n \\omega_{N}^{1\\cdot0} & \\omega_{N}^{1\\cdot1} & \\dots & \\omega_{N}^{2\\cdot (N-1)}\\\\\\\\\n \\vdots & \\vdots & \\ddots & \\vdots\\\\\\\\\n \\omega_{N}^{N-1\\cdot0} & \\omega_{N}^{N-1\\cdot1} & \\dots & \\omega_{N}^{N-1\\cdot (N-1)}\n}\n\\end{bmatrix}\n\\\\]\nwhere \\\\( \\omega_{N} = e^{-\\frac{i2\\pi}{N}}\\\\) is a primitive Nth root of unity. (any complex number that yields 1 when raised to some positive integer power n)\n\n\\\\[ X = F \\cdot x^{T}\\\\]\n\\\\(x\\\\) is a column vector\n\n\n## Polynomial & DFT\n### Polynomial evaluation & DFT\nlet a polynomial be\n$$f(x)=\\sum_{n=0}^{N-1}x_nx^n$$\nthen,\n$$f(\\omega_N^{k})=\\sum_{n=0}^{N-1}x_n\\omega_N^{kn},\\quad k=0,1,2,...,N-1$$\ncompare this equation with E.q(1.1), we can know that DFT is the evaluations of a polynomial at N points of Nth root of unity, \\\\(\\omega_N^{0},\\omega_N^1,\\omega_N^2,...,\\omega_N^{N-1}\\\\)\n\n$$(\\omega_N^{0},X_0),(\\omega_N^1,X_1),(\\omega_N^2,X_2),...,(\\omega_N^{N-1},X_{N-1})$$\n\n### Polynomial interpolation & IDFT\nAccording to E.q(1.2), IDFT is the polynomial interpolation (getting original polynomial coefficients).\n\n## Fast Fourier Transform (FFT) [3]\nThe problem is to calculate the below \n\\\\[ \\tag{2.1} X_{k} = \\sum_{n=0}^{N-1} x_{n} \\cdot \\omega_N^{kn}, \\quad k= 0,1,...,N-1\\\\]\nA straightforward calculation using would require \\\\(N^2\\\\) operations where \"operation\" means, as it will throughout this note, a complex multiplication followed by a complex addition.\nThe FFT algorithm described here iterates on the array and yields the result in less than \\\\( 2N log_2N \\\\)\n\nTo derive the algorithm, suppose \\\\(N\\\\) is a composite, i.e., \\\\(N = r_1\\cdot r_2\\\\). Then let the indices in (2.1) be expressed\n\\\\[ k = k_1r_1 + k_0, \\quad k_0 = 0,1,...,r_1-1, \\quad k_1 = 0,1,..., r_2 -1 \\\\]\n\\\\[ n = n_1r_2 + n_0, \\quad n_0 = 0,1,...,r_2-1, \\quad n_1 = 0,1,..., r_1 -1 \\\\]\n\nSince \\\\( n = n_1r_2 + n_0\\\\), we can write\n\n\\\\[ \\tag{2.2} X(k_1, k_0) = \\sum_{n_0} \\sum_{n_1}x(n_1,n_0)\\omega_N^{kn} = \\sum_{n_0} \\sum_{n_1}x(n_1,n_0)\\omega_N^{kn_1r_2}\\omega_N^{kn_0} \\\\]\n**note** \\\\( x(n_1,n_0)\\\\) is same to \\\\(x(n)\\\\), just the indexing is transformed form 1D to 2D. You can think original 1D array of size N is resized to a matrix of size \\\\( (r_2, r_1) \\\\), \\\\(r_1\\\\) is number of cols, while \\\\(r_2\\\\) is number of rows\n\nSince \\\\(k = k_1r_1 + k_0\\\\), we have\n\\\\[ \\omega_N^{kn_1r_2} =\\omega_N^{(k_1r_1+k_0)n_1r_2} = \\omega_N^{(k_1r_1)n_1r_2} \\cdot \\omega_N^{k_0n_1r_2}\\\\]\naccording to the property of Nth root of unity, \\\\( \\omega_N^{N} = \\omega_N^{r_1r_2} =1 \\\\), \\\\( \\omega_N^{(k_1r_1)n_1r_2}\\\\) is also 1.\nthen we have \n\\\\[ \\omega_N^{kn_1r_2} = \\omega_N^{k_0n_1r_2}\\\\]\nsubstitute it into E.q 2.2, we get \n\\\\[ \\tag{2.3} X(k_1, k_0) = \\sum_{n_0} \\sum_{n_1}x(n_1,n_0)\\omega_N^{kn} = \\sum_{n_0} \\sum_{n_1}x(n_1,n_0)\\omega_N^{k_0n_1r_2}\\omega_N^{kn_0} \\\\]\nTherefore, the inner sum, over \\\\(n_1\\\\), depends only on \\\\(k_0\\\\) and \\\\(n_0\\\\), and can be defined as a new array,\n\\\\[ \\tag{2.4} x_1(k_0, n_0) = \\sum_{n_1} x(n_1, n_0) \\cdot \\omega_N^{k_0n_1r_2}\\\\]\nE.q(2.3) can be writtern as\n\\\\[ \\tag{2.5} X(k_1, k_0) = \\sum_{n_0} x_1(k_0,n_0)\\omega_N^{kn_0} = \\sum_{n_0} x_1(k_0,n_0)\\omega_N^{(k_1r_1 + k_0)n_0} \\\\]\n\nThere are N elements in the matrix \\\\(x_1\\\\), each requiring \\\\(r_1\\\\) operations, giving a total of \\\\(Nr_1\\\\) operations. Similarly, it takes \\\\(Nr_2\\\\) operations to calculate \\\\(X\\\\) from \\\\(x_1\\\\). Therefore, the two-step algorithm, given by Eq(2.4) and Eq(2.5) requires a total of\n\\\\[ \\tag{2.6} T = N(r_1+r_2) \\\\]\n\nit is easy to see how successive applications of the above procedure (recursively), starting with its appliation to Eq(2.4) give an m-step algorihtm requiring \n\\\\[ \\tag{2.7} T = N(r_1+r_2 + ... + r_m) \\\\]\nwhere \n\\\\[ N = \\prod_{j=1}^{j=m}r_j\\\\]\nif all \\\\(r_j\\\\) are equal to \\\\(r\\\\), i.e \\\\(N = r^m \\\\), which gives \\\\( m = log_rN \\\\)\nE.q(2.7) becomes\n\\\\[ \\tag{2.8} T = N(m \\cdot r) = rNm = rN(log_rN) \\\\]\n\n## radix-2 FFT\nThe algorithm with \\\\(r=2\\\\) is derived by expressing the indices in the form\n\n\\\\[\\tag{3.1} k=k_{m-1} \\cdot 2^{m-1} + ... + k_1 \\cdot 2 + k_0 \\\\]\n\\\\[\\tag{3.2} n=n_{m-1} \\cdot 2^{m-1} + ... + n_1 \\cdot 2 + n_0 \\\\]\nwhere \\\\( k_v \\in [0,1] \\\\), for \\\\( v = 0,...,m-1 \\\\), and \\\\( n_v \\in [0,1] \\\\), for \\\\( v = 0,...,m-1 \\\\)\n**\\\\(k_v\\\\) and \\\\(n_v\\\\) are the contents of the respective bit positions in the binary representation of \\\\(k\\\\) and \\\\(n\\\\)**\n\nAll arrays will now be written as functions of the bits of their indices. With this convention E.q(2.1) is written as \n\\\\[ \\tag{3.3} \n\\displaylines{\n X(k_{m-1}, ..., k_0) = \\sum_{n_0}\\sum_{n_1} ... \\sum_{n_{m-1}} x(n_{m-1}, ...,n_1, n_0) \\cdot \\omega_N^{kn} \\\\\\\\\n = \\sum_{n_0}\\sum_{n_1} ... \\sum_{n_{m-1}} x(n_{m-1}, ...,n_1, n_0) \\cdot \\omega_N^{k(n_{m-1} \\cdot 2^{m-1} + ... + n_1 \\cdot 2 + n_0)} \\\\\\\\\n = \\sum_{n_0}\\sum_{n_1} ... \\sum_{n_{m-1}} x(n_{m-1}, ...,n_1, n_0) \\cdot \\omega_N^{kn_{m-1} \\cdot 2^{m-1} + ... + kn_1 \\cdot 2 + kn_0}\n} \n\\\\]\nwhere the sums are over \\\\(n_v \\in [0,1]\\\\), for \\\\(v = 0,1,...,m-1\\\\)\n\nSince \\\\[ \\displaylines{\n \\omega_N^{kn_{m-1}\\cdot 2^{m-1}} = \\omega_N^{(k_{m-1} \\cdot 2^{m-1} + ... + k_1 \\cdot 2 + k_0)n_{m-1}\\cdot 2^{m-1}} \\\\\\\\\n = \\omega_N^{k_0 n_{m-1} \\cdot 2^{m-1}}\n}\n \\\\]\n(it is easy to show that all other terms are 1 as \\\\( \\omega_N^{2^m} = 1 \\\\), so only \\\\( k_0\\\\) is kept)\n\nthe innermost sum of E.q(3.3) over \\\\(n_{m-1} \\\\), depends only on \\\\( k_0, n_{m-2}, ..., n_0\\\\) and can be writtern\n\\\\[ \\displaylines{\nx_1(k_0, n_{m-2}, ..., n_1, n_0) = \\sum_{n_{m-1}} x(n_{m-1}, ..., n_0) \\cdot \\omega_N^{k_0 n_{m-1} \\cdot 2^{m-1}}\n}\n \\\\]\n\nproceeding to the next innermost sum, over \\\\( n_{m-2} \\\\), and so on, and using \n\\\\[ \\displaylines{\n \\omega_N^{kn_{m-l}\\cdot 2^{m-l}} = \\omega_N^{(k_{m-1} \\cdot 2^{m-1} + ... + k_1 \\cdot 2 + k_0)n_{m-l}\\cdot 2^{m-l}} \\\\\\\\\n = \\omega_N^{(k_{l-1}\\cdot 2^{l-1} + ... + k_0) n_{m-l} \\cdot 2^{m-1}}\n}\n \\\\]\none obtains successive arrays\n\\\\[\\displaylines{\nx_l(k_0, ..., k_{l-1}, n_{m-l-1}, ... , n_0) \\\\\\\\\n= \\sum_{n_{m-l}}x_{l-1}(k_0, ..., k_{l-2}, n_{m-l}, ..., n_0 ) \\cdot \\omega_N^{(k_{l-1}\\cdot 2^{l-1} + ...+ k_0) \\cdot n_{m-l} \\cdot 2^{m-l}}\n}\\\\]\nfor \\\\(l = 1,2,...,m \\\\)\n\nwriting out the sum this appears as \n\\\\[ \\tag{3.4} \\displaylines{\nx_l(k_0, ..., k_{l-1}, n_{m-l-1}, ... , n_0) \\\\\\\\\n= x_{l-1}(k_0, ..., k_{l-2}, 0, n_{m-l -1}, ..., n_0 ) \\cdot \\omega_N^{(k_{l-1}\\cdot 2^{l-1} + ...+ k_0) \\cdot 0 \\cdot 2^{m-l}} \\\\\\\\\n\\+ x_{l-1}(k_0, ..., k_{l-2}, 1, n_{m-l -1}, ..., n_0 ) \\cdot \\omega_N^{(k_{l-1}\\cdot 2^{l-1} + ...+ k_0) \\cdot 1 \\cdot 2^{m-l}} \\\\\\\\\n= x_{l-1}(k_0, ..., k_{l-2}, 0, n_{m-l -1}, ..., n_0 ) \\\\\\\\\n\\+ x_{l-1}(k_0, ..., k_{l-2}, 1, n_{m-l-1}, ..., n_0 ) \\cdot \\omega_N^{(k_{l-1}\\cdot 2^{l-1} + k_{l-2}\\cdot 2^{l-2 }+k_{l-3}\\cdot 2^{l-3} + ...+ k_0) \\cdot 2^{m-l}} \\\\\\\\\n= x_{l-1}(k_0, ..., k_{l-2}, 0, n_{m-l -1}, ..., n_0 ) \\\\\\\\\n\\+ \\omega_N^{k_{l-1}\\cdot 2^{l-1} \\cdot 2^{m-l} } \\cdot \\omega_N^{k_{l-2}\\cdot 2^{l-2 } \\cdot 2^{m-l}} \\cdot x_{l-1}(k_0, ..., k_{l-2}, 1, n_{m-l-1}, ..., n_0 ) \\cdot \\omega_N^{(k_{l-3}\\cdot 2^{l-3} + ...+ k_0) \\cdot 2^{m-l}} \\\\\\\\\n= x_{l-1}(k_0, ..., k_{l-2}, 0, n_{m-l -1}, ..., n_0 ) \\\\\\\\\n\\+ \\omega_N^{k_{l-1}\\cdot 2^{m-1} } \\cdot \\omega_N^{k_{l-2}\\cdot 2^{m-2}} \\cdot x_{l-1}(k_0, ..., k_{l-2}, 1, n_{m-l-1}, ..., n_0 ) \\cdot \\omega_N^{(k_{l-3}\\cdot 2^{l-3} + ...+ k_0) \\cdot 2^{m-l}} \\\\\\\\\n= x_{l-1}(k_0, ..., k_{l-2}, 0, n_{m-l -1}, ..., n_0 ) \\\\\\\\\n\\+ (-1)^{k_{l-1} } \\cdot i^{k_{l-2}} \\cdot x_{l-1}(k_0, ..., k_{l-2}, 1, n_{m-l-1}, ..., n_0 ) \\cdot \\omega_N^{(k_{l-3}\\cdot 2^{l-3} + ...+ k_0) \\cdot 2^{m-l}} \n}\\\\]\naccording to the indexing convension, this is stored in a location whose index is \n\\\\[ k_0 \\cdot 2^{m-1} + ... + k_{l-1} \\cdot 2 ^{m-l} + n_{m-l-1} \\cdot 2^{m-l-1} + ... + n_0 \\\\]\n\nIt can be seen in E.q(3.4) that only the two storage locations with indices having 0 and 1 in the \\\\(2^{m-l}\\\\) bit position are involved in the computation. Parallel computation is permitted since the operation described by E.q(3.4) can be carried out with all values of \\\\(k_0, ..., k_{l-2} \\\\), and \\\\( n_0, ..., n_{m-l-1}\\\\) simultaneously. In some applications it is convenient to use E.q(3.4) to express \\\\(x_l\\\\) in terms of \\\\(x_{l-2}\\\\), giving what is equivalent to an algorithm with \\\\(r = 4\\\\).\nthe last array calculated gives the desired Fourier sums,\n\n\\\\[\\tag{3.5}\nX(k_{m-1}, ..., k_0) = x_{m}(k_0, ..., k_{m-1})\n\\\\]\nin such an order that the index of an X must have its binary bits put in reverse order to yield its index in the array \\\\(x_m\\\\)\n## references\n- [1] [youtube video by Steve Brunton, Fourier Series](https://www.youtube.com/watch?v=MB6XGQWLV04)\n- [2] [DFT wikipedia](https://en.wikipedia.org/wiki/Discrete_Fourier_transform)\n- [3] [Cooley–Tukey FFT algorithm wikipedia](https://en.wikipedia.org/wiki/Cooley%E2%80%93Tukey_FFT_algorithm)\n- [4] [1965, Cooley & Turkey: an algorithm for the machine calculation of complex fourier series](https://www.ams.org/journals/mcom/1965-19-090/S0025-5718-1965-0178586-1/S0025-5718-1965-0178586-1.pdf)\n- [5] [Split-radix FFT algorithm wikipedia](https://en.wikipedia.org/wiki/Split-radix_FFT_algorithm)\n- [6] [csdn blog of FFT notes](https://blog.csdn.net/weixin_43870101/article/details/106095644?utm_medium=distribute.pc_relevant.none-task-blog-2~default~baidujs_utm_term~default-2-106095644-blog-132357995.235^v38^pc_relevant_sort_base1&spm=1001.2101.3001.4242.2&utm_relevant_index=5)\n\n\n","source":"_posts/math/fourier-series.md","raw":"---\ntitle: fourier series, DFT and FFT\ndate: 2023-10-12 11:13:17\ntags: [math]\n---\n\n\n\n\n## Fourier Series\n### innter product of two functions\n\\\\[ = \\int_{a}^{b} f(x) \\bar{g}(x)dx\\\\]\n\\\\(\\bar{g}\\\\) is the congugate of g on complex number system\n\n### Fourier series definition\nlet \\\\(f(x)\\\\) be a periodic function over a period of \\\\([-\\pi, +\\pi]\\\\)\n![f(x)](/images/math/fourier_series/f_x.png)\n\n\\\\( f(x) \\\\) can be transformed into below\n\\\\[ f(x) = \\frac{A_{0}}{2} + \\sum_{k=1}^{\\infty}A_{k}cos(kx) + B_{k}sin(kx)\\\\],\nwhere\n\\\\[ A_{k} = \\frac{1}{\\pi} \\int_{-\\pi}^{+\\pi} f(x)cos(kx)dx = \\frac{1}{||cos(kx)||^2} \\\\]\n\\\\[ B_{k} = \\frac{1}{\\pi} \\int_{-\\pi}^{+\\pi} f(x)sin(kx)dx = \\frac{1}{||sin(kx)||^2} \\\\]\nwhere \\\\(A_{k}, B_{k}\\\\) is just the **innver product** of \\\\(f(x)\\\\), and \\\\( cos(kx) \\\\), \\\\(sin(kx)\\\\) respectively. intuitively, it is the **projection** of \\\\(f(x)\\\\) over \\\\(sin(kx)\\\\), or \\\\(cos(kx)\\\\). \n\n### if period is L\n\\\\[ f(x) = \\frac{A_{0}}{2} + \\sum_{k=1}^{\\infty}A_{k}cos(\\frac{2\\pi kx}{L}) + B_{k}sin(\\frac{2\\pi kx}{L})\\\\],\nwhere \n\\\\[ A_{k} = \\frac{2}{L} \\int_{0}^{L} f(x)cos(\\frac{2\\pi kx}{L})dx\\\\]\n\\\\[ B_{k} = \\frac{2}{L} \\int_{0}^{L} f(x)sin(\\frac{2\\pi kx}{L})dx\\\\]\n\n## Complex Fourier Series\nlet's define \\\\[ \\psi_k = e^{ikx} = cos(kx) + i \\cdot sin(kx)\\\\]\nwe can prove that \\\\(\\psi_k, and \\psi_j\\\\) are **orthogonal** as their inner product is zero\n**proof**\n\\\\[ <\\psi_j, \\psi_k> = \\int_{-\\pi}^{+\\pi} e^{jkx}e^{-ikx}dx = \\int_{-\\pi}^{+\\pi} e^{j(j-k)x}dx = \\frac{1}{i(j-k)}[e^{i(j-k)x}]_{-\\pi}^{\\pi}\\\\]\nit is 0 if \\\\(j \\ne k\\\\) , \\\\(2\\pi \\\\) if \\\\(j=k\\\\)\n\n**note** conjugate of \\\\(\\psi_k\\\\) is \\\\(e^{-ikx}\\\\)\n\n\\\\(f(x) = \\frac{1}{2\\pi} \\sum_{k=-\\infty}^{\\infty} c_k \\psi_k\\\\), where \\\\(c_k= = \\int_{-\\pi}^{+\\pi} f(x)e^{-ikx}dx\\\\)\n\n## Discrete Fourier Transform (DFT)\nThe discrete Fourier transform transforms a sequence of N complex numbers \n\\\\[\\lbrace x_{n} \\rbrace := x_0, x_1,..., x_{N-1} \\\\] into another sequence of complex numbers, \n\\\\[\\lbrace X_{k} \\rbrace := X_0, X_1,..., X_{N-1} \\\\] which is defined by\n\\\\[X_{k} = \\sum_{n=0}^{N-1} x_{n} \\cdot e^{-\\frac{i2\\pi}{N}kn}\\\\]\n\nlet the n-th root of unity be \\\\(\\omega_N = e^{-\\frac{2\\pi i}{N}}\\\\), we have\n\\\\[ \\tag{1.1} X_{k} = \\sum_{n=0}^{N-1} x_{n} \\cdot \\omega_N^{kn}\\\\]\n\n### inverse DFT\nThe inverse transform is given by\n\\\\[ \\tag{1.2} x_{n} = \\frac{1}{N}\\sum_{k=0}^{N-1} X_{k} \\cdot \\omega_N^{-kn}\\\\]\n\n\n### the unitary DFT\nAnother way of looking at the DFT is to note that the DFT can be expressed as the DFT matrix, a Vandermonde matrix, introduced by Sylvester in 1867.\n\\\\[\n \\boldsymbol{F} =\n\\begin{bmatrix}\n\\displaylines{\n \\omega_{N}^{0\\cdot0} & \\omega_{N}^{0\\cdot1} & \\dots & \\omega_{N}^{0\\cdot (N-1)}\\\\\\\\\n \\omega_{N}^{1\\cdot0} & \\omega_{N}^{1\\cdot1} & \\dots & \\omega_{N}^{2\\cdot (N-1)}\\\\\\\\\n \\vdots & \\vdots & \\ddots & \\vdots\\\\\\\\\n \\omega_{N}^{N-1\\cdot0} & \\omega_{N}^{N-1\\cdot1} & \\dots & \\omega_{N}^{N-1\\cdot (N-1)}\n}\n\\end{bmatrix}\n\\\\]\nwhere \\\\( \\omega_{N} = e^{-\\frac{i2\\pi}{N}}\\\\) is a primitive Nth root of unity. (any complex number that yields 1 when raised to some positive integer power n)\n\n\\\\[ X = F \\cdot x^{T}\\\\]\n\\\\(x\\\\) is a column vector\n\n\n## Polynomial & DFT\n### Polynomial evaluation & DFT\nlet a polynomial be\n$$f(x)=\\sum_{n=0}^{N-1}x_nx^n$$\nthen,\n$$f(\\omega_N^{k})=\\sum_{n=0}^{N-1}x_n\\omega_N^{kn},\\quad k=0,1,2,...,N-1$$\ncompare this equation with E.q(1.1), we can know that DFT is the evaluations of a polynomial at N points of Nth root of unity, \\\\(\\omega_N^{0},\\omega_N^1,\\omega_N^2,...,\\omega_N^{N-1}\\\\)\n\n$$(\\omega_N^{0},X_0),(\\omega_N^1,X_1),(\\omega_N^2,X_2),...,(\\omega_N^{N-1},X_{N-1})$$\n\n### Polynomial interpolation & IDFT\nAccording to E.q(1.2), IDFT is the polynomial interpolation (getting original polynomial coefficients).\n\n## Fast Fourier Transform (FFT) [3]\nThe problem is to calculate the below \n\\\\[ \\tag{2.1} X_{k} = \\sum_{n=0}^{N-1} x_{n} \\cdot \\omega_N^{kn}, \\quad k= 0,1,...,N-1\\\\]\nA straightforward calculation using would require \\\\(N^2\\\\) operations where \"operation\" means, as it will throughout this note, a complex multiplication followed by a complex addition.\nThe FFT algorithm described here iterates on the array and yields the result in less than \\\\( 2N log_2N \\\\)\n\nTo derive the algorithm, suppose \\\\(N\\\\) is a composite, i.e., \\\\(N = r_1\\cdot r_2\\\\). Then let the indices in (2.1) be expressed\n\\\\[ k = k_1r_1 + k_0, \\quad k_0 = 0,1,...,r_1-1, \\quad k_1 = 0,1,..., r_2 -1 \\\\]\n\\\\[ n = n_1r_2 + n_0, \\quad n_0 = 0,1,...,r_2-1, \\quad n_1 = 0,1,..., r_1 -1 \\\\]\n\nSince \\\\( n = n_1r_2 + n_0\\\\), we can write\n\n\\\\[ \\tag{2.2} X(k_1, k_0) = \\sum_{n_0} \\sum_{n_1}x(n_1,n_0)\\omega_N^{kn} = \\sum_{n_0} \\sum_{n_1}x(n_1,n_0)\\omega_N^{kn_1r_2}\\omega_N^{kn_0} \\\\]\n**note** \\\\( x(n_1,n_0)\\\\) is same to \\\\(x(n)\\\\), just the indexing is transformed form 1D to 2D. You can think original 1D array of size N is resized to a matrix of size \\\\( (r_2, r_1) \\\\), \\\\(r_1\\\\) is number of cols, while \\\\(r_2\\\\) is number of rows\n\nSince \\\\(k = k_1r_1 + k_0\\\\), we have\n\\\\[ \\omega_N^{kn_1r_2} =\\omega_N^{(k_1r_1+k_0)n_1r_2} = \\omega_N^{(k_1r_1)n_1r_2} \\cdot \\omega_N^{k_0n_1r_2}\\\\]\naccording to the property of Nth root of unity, \\\\( \\omega_N^{N} = \\omega_N^{r_1r_2} =1 \\\\), \\\\( \\omega_N^{(k_1r_1)n_1r_2}\\\\) is also 1.\nthen we have \n\\\\[ \\omega_N^{kn_1r_2} = \\omega_N^{k_0n_1r_2}\\\\]\nsubstitute it into E.q 2.2, we get \n\\\\[ \\tag{2.3} X(k_1, k_0) = \\sum_{n_0} \\sum_{n_1}x(n_1,n_0)\\omega_N^{kn} = \\sum_{n_0} \\sum_{n_1}x(n_1,n_0)\\omega_N^{k_0n_1r_2}\\omega_N^{kn_0} \\\\]\nTherefore, the inner sum, over \\\\(n_1\\\\), depends only on \\\\(k_0\\\\) and \\\\(n_0\\\\), and can be defined as a new array,\n\\\\[ \\tag{2.4} x_1(k_0, n_0) = \\sum_{n_1} x(n_1, n_0) \\cdot \\omega_N^{k_0n_1r_2}\\\\]\nE.q(2.3) can be writtern as\n\\\\[ \\tag{2.5} X(k_1, k_0) = \\sum_{n_0} x_1(k_0,n_0)\\omega_N^{kn_0} = \\sum_{n_0} x_1(k_0,n_0)\\omega_N^{(k_1r_1 + k_0)n_0} \\\\]\n\nThere are N elements in the matrix \\\\(x_1\\\\), each requiring \\\\(r_1\\\\) operations, giving a total of \\\\(Nr_1\\\\) operations. Similarly, it takes \\\\(Nr_2\\\\) operations to calculate \\\\(X\\\\) from \\\\(x_1\\\\). Therefore, the two-step algorithm, given by Eq(2.4) and Eq(2.5) requires a total of\n\\\\[ \\tag{2.6} T = N(r_1+r_2) \\\\]\n\nit is easy to see how successive applications of the above procedure (recursively), starting with its appliation to Eq(2.4) give an m-step algorihtm requiring \n\\\\[ \\tag{2.7} T = N(r_1+r_2 + ... + r_m) \\\\]\nwhere \n\\\\[ N = \\prod_{j=1}^{j=m}r_j\\\\]\nif all \\\\(r_j\\\\) are equal to \\\\(r\\\\), i.e \\\\(N = r^m \\\\), which gives \\\\( m = log_rN \\\\)\nE.q(2.7) becomes\n\\\\[ \\tag{2.8} T = N(m \\cdot r) = rNm = rN(log_rN) \\\\]\n\n## radix-2 FFT\nThe algorithm with \\\\(r=2\\\\) is derived by expressing the indices in the form\n\n\\\\[\\tag{3.1} k=k_{m-1} \\cdot 2^{m-1} + ... + k_1 \\cdot 2 + k_0 \\\\]\n\\\\[\\tag{3.2} n=n_{m-1} \\cdot 2^{m-1} + ... + n_1 \\cdot 2 + n_0 \\\\]\nwhere \\\\( k_v \\in [0,1] \\\\), for \\\\( v = 0,...,m-1 \\\\), and \\\\( n_v \\in [0,1] \\\\), for \\\\( v = 0,...,m-1 \\\\)\n**\\\\(k_v\\\\) and \\\\(n_v\\\\) are the contents of the respective bit positions in the binary representation of \\\\(k\\\\) and \\\\(n\\\\)**\n\nAll arrays will now be written as functions of the bits of their indices. With this convention E.q(2.1) is written as \n\\\\[ \\tag{3.3} \n\\displaylines{\n X(k_{m-1}, ..., k_0) = \\sum_{n_0}\\sum_{n_1} ... \\sum_{n_{m-1}} x(n_{m-1}, ...,n_1, n_0) \\cdot \\omega_N^{kn} \\\\\\\\\n = \\sum_{n_0}\\sum_{n_1} ... \\sum_{n_{m-1}} x(n_{m-1}, ...,n_1, n_0) \\cdot \\omega_N^{k(n_{m-1} \\cdot 2^{m-1} + ... + n_1 \\cdot 2 + n_0)} \\\\\\\\\n = \\sum_{n_0}\\sum_{n_1} ... \\sum_{n_{m-1}} x(n_{m-1}, ...,n_1, n_0) \\cdot \\omega_N^{kn_{m-1} \\cdot 2^{m-1} + ... + kn_1 \\cdot 2 + kn_0}\n} \n\\\\]\nwhere the sums are over \\\\(n_v \\in [0,1]\\\\), for \\\\(v = 0,1,...,m-1\\\\)\n\nSince \\\\[ \\displaylines{\n \\omega_N^{kn_{m-1}\\cdot 2^{m-1}} = \\omega_N^{(k_{m-1} \\cdot 2^{m-1} + ... + k_1 \\cdot 2 + k_0)n_{m-1}\\cdot 2^{m-1}} \\\\\\\\\n = \\omega_N^{k_0 n_{m-1} \\cdot 2^{m-1}}\n}\n \\\\]\n(it is easy to show that all other terms are 1 as \\\\( \\omega_N^{2^m} = 1 \\\\), so only \\\\( k_0\\\\) is kept)\n\nthe innermost sum of E.q(3.3) over \\\\(n_{m-1} \\\\), depends only on \\\\( k_0, n_{m-2}, ..., n_0\\\\) and can be writtern\n\\\\[ \\displaylines{\nx_1(k_0, n_{m-2}, ..., n_1, n_0) = \\sum_{n_{m-1}} x(n_{m-1}, ..., n_0) \\cdot \\omega_N^{k_0 n_{m-1} \\cdot 2^{m-1}}\n}\n \\\\]\n\nproceeding to the next innermost sum, over \\\\( n_{m-2} \\\\), and so on, and using \n\\\\[ \\displaylines{\n \\omega_N^{kn_{m-l}\\cdot 2^{m-l}} = \\omega_N^{(k_{m-1} \\cdot 2^{m-1} + ... + k_1 \\cdot 2 + k_0)n_{m-l}\\cdot 2^{m-l}} \\\\\\\\\n = \\omega_N^{(k_{l-1}\\cdot 2^{l-1} + ... + k_0) n_{m-l} \\cdot 2^{m-1}}\n}\n \\\\]\none obtains successive arrays\n\\\\[\\displaylines{\nx_l(k_0, ..., k_{l-1}, n_{m-l-1}, ... , n_0) \\\\\\\\\n= \\sum_{n_{m-l}}x_{l-1}(k_0, ..., k_{l-2}, n_{m-l}, ..., n_0 ) \\cdot \\omega_N^{(k_{l-1}\\cdot 2^{l-1} + ...+ k_0) \\cdot n_{m-l} \\cdot 2^{m-l}}\n}\\\\]\nfor \\\\(l = 1,2,...,m \\\\)\n\nwriting out the sum this appears as \n\\\\[ \\tag{3.4} \\displaylines{\nx_l(k_0, ..., k_{l-1}, n_{m-l-1}, ... , n_0) \\\\\\\\\n= x_{l-1}(k_0, ..., k_{l-2}, 0, n_{m-l -1}, ..., n_0 ) \\cdot \\omega_N^{(k_{l-1}\\cdot 2^{l-1} + ...+ k_0) \\cdot 0 \\cdot 2^{m-l}} \\\\\\\\\n\\+ x_{l-1}(k_0, ..., k_{l-2}, 1, n_{m-l -1}, ..., n_0 ) \\cdot \\omega_N^{(k_{l-1}\\cdot 2^{l-1} + ...+ k_0) \\cdot 1 \\cdot 2^{m-l}} \\\\\\\\\n= x_{l-1}(k_0, ..., k_{l-2}, 0, n_{m-l -1}, ..., n_0 ) \\\\\\\\\n\\+ x_{l-1}(k_0, ..., k_{l-2}, 1, n_{m-l-1}, ..., n_0 ) \\cdot \\omega_N^{(k_{l-1}\\cdot 2^{l-1} + k_{l-2}\\cdot 2^{l-2 }+k_{l-3}\\cdot 2^{l-3} + ...+ k_0) \\cdot 2^{m-l}} \\\\\\\\\n= x_{l-1}(k_0, ..., k_{l-2}, 0, n_{m-l -1}, ..., n_0 ) \\\\\\\\\n\\+ \\omega_N^{k_{l-1}\\cdot 2^{l-1} \\cdot 2^{m-l} } \\cdot \\omega_N^{k_{l-2}\\cdot 2^{l-2 } \\cdot 2^{m-l}} \\cdot x_{l-1}(k_0, ..., k_{l-2}, 1, n_{m-l-1}, ..., n_0 ) \\cdot \\omega_N^{(k_{l-3}\\cdot 2^{l-3} + ...+ k_0) \\cdot 2^{m-l}} \\\\\\\\\n= x_{l-1}(k_0, ..., k_{l-2}, 0, n_{m-l -1}, ..., n_0 ) \\\\\\\\\n\\+ \\omega_N^{k_{l-1}\\cdot 2^{m-1} } \\cdot \\omega_N^{k_{l-2}\\cdot 2^{m-2}} \\cdot x_{l-1}(k_0, ..., k_{l-2}, 1, n_{m-l-1}, ..., n_0 ) \\cdot \\omega_N^{(k_{l-3}\\cdot 2^{l-3} + ...+ k_0) \\cdot 2^{m-l}} \\\\\\\\\n= x_{l-1}(k_0, ..., k_{l-2}, 0, n_{m-l -1}, ..., n_0 ) \\\\\\\\\n\\+ (-1)^{k_{l-1} } \\cdot i^{k_{l-2}} \\cdot x_{l-1}(k_0, ..., k_{l-2}, 1, n_{m-l-1}, ..., n_0 ) \\cdot \\omega_N^{(k_{l-3}\\cdot 2^{l-3} + ...+ k_0) \\cdot 2^{m-l}} \n}\\\\]\naccording to the indexing convension, this is stored in a location whose index is \n\\\\[ k_0 \\cdot 2^{m-1} + ... + k_{l-1} \\cdot 2 ^{m-l} + n_{m-l-1} \\cdot 2^{m-l-1} + ... + n_0 \\\\]\n\nIt can be seen in E.q(3.4) that only the two storage locations with indices having 0 and 1 in the \\\\(2^{m-l}\\\\) bit position are involved in the computation. Parallel computation is permitted since the operation described by E.q(3.4) can be carried out with all values of \\\\(k_0, ..., k_{l-2} \\\\), and \\\\( n_0, ..., n_{m-l-1}\\\\) simultaneously. In some applications it is convenient to use E.q(3.4) to express \\\\(x_l\\\\) in terms of \\\\(x_{l-2}\\\\), giving what is equivalent to an algorithm with \\\\(r = 4\\\\).\nthe last array calculated gives the desired Fourier sums,\n\n\\\\[\\tag{3.5}\nX(k_{m-1}, ..., k_0) = x_{m}(k_0, ..., k_{m-1})\n\\\\]\nin such an order that the index of an X must have its binary bits put in reverse order to yield its index in the array \\\\(x_m\\\\)\n## references\n- [1] [youtube video by Steve Brunton, Fourier Series](https://www.youtube.com/watch?v=MB6XGQWLV04)\n- [2] [DFT wikipedia](https://en.wikipedia.org/wiki/Discrete_Fourier_transform)\n- [3] [Cooley–Tukey FFT algorithm wikipedia](https://en.wikipedia.org/wiki/Cooley%E2%80%93Tukey_FFT_algorithm)\n- [4] [1965, Cooley & Turkey: an algorithm for the machine calculation of complex fourier series](https://www.ams.org/journals/mcom/1965-19-090/S0025-5718-1965-0178586-1/S0025-5718-1965-0178586-1.pdf)\n- [5] [Split-radix FFT algorithm wikipedia](https://en.wikipedia.org/wiki/Split-radix_FFT_algorithm)\n- [6] [csdn blog of FFT notes](https://blog.csdn.net/weixin_43870101/article/details/106095644?utm_medium=distribute.pc_relevant.none-task-blog-2~default~baidujs_utm_term~default-2-106095644-blog-132357995.235^v38^pc_relevant_sort_base1&spm=1001.2101.3001.4242.2&utm_relevant_index=5)\n\n\n","slug":"math/fourier-series","published":1,"updated":"2023-11-26T15:51:47.630Z","_id":"clokyy8dv0021qwsjhghcad7b","comments":1,"layout":"post","photos":[],"link":"","content":"\n\n\n

Fourier Series

innter product of two functions

\\[ <f(x), g(x)> = \\int_{a}^{b} f(x) \\bar{g}(x)dx\\]
\\(\\bar{g}\\) is the congugate of g on complex number system

\n

Fourier series definition

let \\(f(x)\\) be a periodic function over a period of \\([-\\pi, +\\pi]\\)
\"f(x)\"

\n

\\( f(x) \\) can be transformed into below
\\[ f(x) = \\frac{A_{0}}{2} + \\sum_{k=1}^{\\infty}A_{k}cos(kx) + B_{k}sin(kx)\\],
where
\\[ A_{k} = \\frac{1}{\\pi} \\int_{-\\pi}^{+\\pi} f(x)cos(kx)dx = \\frac{1}{||cos(kx)||^2}<f(x),cos(kx)> \\]
\\[ B_{k} = \\frac{1}{\\pi} \\int_{-\\pi}^{+\\pi} f(x)sin(kx)dx = \\frac{1}{||sin(kx)||^2}<f(x),sin(kx)> \\]
where \\(A_{k}, B_{k}\\) is just the innver product of \\(f(x)\\), and \\( cos(kx) \\), \\(sin(kx)\\) respectively. intuitively, it is the projection of \\(f(x)\\) over \\(sin(kx)\\), or \\(cos(kx)\\).

\n

if period is L

\\[ f(x) = \\frac{A_{0}}{2} + \\sum_{k=1}^{\\infty}A_{k}cos(\\frac{2\\pi kx}{L}) + B_{k}sin(\\frac{2\\pi kx}{L})\\],
where
\\[ A_{k} = \\frac{2}{L} \\int_{0}^{L} f(x)cos(\\frac{2\\pi kx}{L})dx\\]
\\[ B_{k} = \\frac{2}{L} \\int_{0}^{L} f(x)sin(\\frac{2\\pi kx}{L})dx\\]

\n

Complex Fourier Series

let’s define \\[ \\psi_k = e^{ikx} = cos(kx) + i \\cdot sin(kx)\\]
we can prove that \\(\\psi_k, and \\psi_j\\) are orthogonal as their inner product is zero
proof
\\[ <\\psi_j, \\psi_k> = \\int_{-\\pi}^{+\\pi} e^{jkx}e^{-ikx}dx = \\int_{-\\pi}^{+\\pi} e^{j(j-k)x}dx = \\frac{1}{i(j-k)}[e^{i(j-k)x}]_{-\\pi}^{\\pi}\\]
it is 0 if \\(j \\ne k\\) , \\(2\\pi \\) if \\(j=k\\)

\n

note conjugate of \\(\\psi_k\\) is \\(e^{-ikx}\\)

\n

\\(f(x) = \\frac{1}{2\\pi} \\sum_{k=-\\infty}^{\\infty} c_k \\psi_k\\), where \\(c_k=<f(x), \\psi_k> = \\int_{-\\pi}^{+\\pi} f(x)e^{-ikx}dx\\)

\n

Discrete Fourier Transform (DFT)

The discrete Fourier transform transforms a sequence of N complex numbers
\\[\\lbrace x_{n} \\rbrace := x_0, x_1,…, x_{N-1} \\] into another sequence of complex numbers,
\\[\\lbrace X_{k} \\rbrace := X_0, X_1,…, X_{N-1} \\] which is defined by
\\[X_{k} = \\sum_{n=0}^{N-1} x_{n} \\cdot e^{-\\frac{i2\\pi}{N}kn}\\]

\n

let the n-th root of unity be \\(\\omega_N = e^{-\\frac{2\\pi i}{N}}\\), we have
\\[ \\tag{1.1} X_{k} = \\sum_{n=0}^{N-1} x_{n} \\cdot \\omega_N^{kn}\\]

\n

inverse DFT

The inverse transform is given by
\\[ \\tag{1.2} x_{n} = \\frac{1}{N}\\sum_{k=0}^{N-1} X_{k} \\cdot \\omega_N^{-kn}\\]

\n

the unitary DFT

Another way of looking at the DFT is to note that the DFT can be expressed as the DFT matrix, a Vandermonde matrix, introduced by Sylvester in 1867.
\\[
\\boldsymbol{F} =
\\begin{bmatrix}
\\displaylines{
\\omega_{N}^{0\\cdot0} & \\omega_{N}^{0\\cdot1} & \\dots & \\omega_{N}^{0\\cdot (N-1)}\\\\
\\omega_{N}^{1\\cdot0} & \\omega_{N}^{1\\cdot1} & \\dots & \\omega_{N}^{2\\cdot (N-1)}\\\\
\\vdots & \\vdots & \\ddots & \\vdots\\\\
\\omega_{N}^{N-1\\cdot0} & \\omega_{N}^{N-1\\cdot1} & \\dots & \\omega_{N}^{N-1\\cdot (N-1)}
}
\\end{bmatrix}
\\]
where \\( \\omega_{N} = e^{-\\frac{i2\\pi}{N}}\\) is a primitive Nth root of unity. (any complex number that yields 1 when raised to some positive integer power n)

\n

\\[ X = F \\cdot x^{T}\\]
\\(x\\) is a column vector

\n

Polynomial & DFT

Polynomial evaluation & DFT

let a polynomial be
$$f(x)=\\sum_{n=0}^{N-1}x_nx^n$$
then,
$$f(\\omega_N^{k})=\\sum_{n=0}^{N-1}x_n\\omega_N^{kn},\\quad k=0,1,2,…,N-1$$
compare this equation with E.q(1.1), we can know that DFT is the evaluations of a polynomial at N points of Nth root of unity, \\(\\omega_N^{0},\\omega_N^1,\\omega_N^2,…,\\omega_N^{N-1}\\)

\n

$$(\\omega_N^{0},X_0),(\\omega_N^1,X_1),(\\omega_N^2,X_2),…,(\\omega_N^{N-1},X_{N-1})$$

\n

Polynomial interpolation & IDFT

According to E.q(1.2), IDFT is the polynomial interpolation (getting original polynomial coefficients).

\n

Fast Fourier Transform (FFT) [3]

The problem is to calculate the below
\\[ \\tag{2.1} X_{k} = \\sum_{n=0}^{N-1} x_{n} \\cdot \\omega_N^{kn}, \\quad k= 0,1,…,N-1\\]
A straightforward calculation using would require \\(N^2\\) operations where “operation” means, as it will throughout this note, a complex multiplication followed by a complex addition.
The FFT algorithm described here iterates on the array and yields the result in less than \\( 2N log_2N \\)

\n

To derive the algorithm, suppose \\(N\\) is a composite, i.e., \\(N = r_1\\cdot r_2\\). Then let the indices in (2.1) be expressed
\\[ k = k_1r_1 + k_0, \\quad k_0 = 0,1,…,r_1-1, \\quad k_1 = 0,1,…, r_2 -1 \\]
\\[ n = n_1r_2 + n_0, \\quad n_0 = 0,1,…,r_2-1, \\quad n_1 = 0,1,…, r_1 -1 \\]

\n

Since \\( n = n_1r_2 + n_0\\), we can write

\n

\\[ \\tag{2.2} X(k_1, k_0) = \\sum_{n_0} \\sum_{n_1}x(n_1,n_0)\\omega_N^{kn} = \\sum_{n_0} \\sum_{n_1}x(n_1,n_0)\\omega_N^{kn_1r_2}\\omega_N^{kn_0} \\]
note \\( x(n_1,n_0)\\) is same to \\(x(n)\\), just the indexing is transformed form 1D to 2D. You can think original 1D array of size N is resized to a matrix of size \\( (r_2, r_1) \\), \\(r_1\\) is number of cols, while \\(r_2\\) is number of rows

\n

Since \\(k = k_1r_1 + k_0\\), we have
\\[ \\omega_N^{kn_1r_2} =\\omega_N^{(k_1r_1+k_0)n_1r_2} = \\omega_N^{(k_1r_1)n_1r_2} \\cdot \\omega_N^{k_0n_1r_2}\\]
according to the property of Nth root of unity, \\( \\omega_N^{N} = \\omega_N^{r_1r_2} =1 \\), \\( \\omega_N^{(k_1r_1)n_1r_2}\\) is also 1.
then we have
\\[ \\omega_N^{kn_1r_2} = \\omega_N^{k_0n_1r_2}\\]
substitute it into E.q 2.2, we get
\\[ \\tag{2.3} X(k_1, k_0) = \\sum_{n_0} \\sum_{n_1}x(n_1,n_0)\\omega_N^{kn} = \\sum_{n_0} \\sum_{n_1}x(n_1,n_0)\\omega_N^{k_0n_1r_2}\\omega_N^{kn_0} \\]
Therefore, the inner sum, over \\(n_1\\), depends only on \\(k_0\\) and \\(n_0\\), and can be defined as a new array,
\\[ \\tag{2.4} x_1(k_0, n_0) = \\sum_{n_1} x(n_1, n_0) \\cdot \\omega_N^{k_0n_1r_2}\\]
E.q(2.3) can be writtern as
\\[ \\tag{2.5} X(k_1, k_0) = \\sum_{n_0} x_1(k_0,n_0)\\omega_N^{kn_0} = \\sum_{n_0} x_1(k_0,n_0)\\omega_N^{(k_1r_1 + k_0)n_0} \\]

\n

There are N elements in the matrix \\(x_1\\), each requiring \\(r_1\\) operations, giving a total of \\(Nr_1\\) operations. Similarly, it takes \\(Nr_2\\) operations to calculate \\(X\\) from \\(x_1\\). Therefore, the two-step algorithm, given by Eq(2.4) and Eq(2.5) requires a total of
\\[ \\tag{2.6} T = N(r_1+r_2) \\]

\n

it is easy to see how successive applications of the above procedure (recursively), starting with its appliation to Eq(2.4) give an m-step algorihtm requiring
\\[ \\tag{2.7} T = N(r_1+r_2 + … + r_m) \\]
where
\\[ N = \\prod_{j=1}^{j=m}r_j\\]
if all \\(r_j\\) are equal to \\(r\\), i.e \\(N = r^m \\), which gives \\( m = log_rN \\)
E.q(2.7) becomes
\\[ \\tag{2.8} T = N(m \\cdot r) = rNm = rN(log_rN) \\]

\n

radix-2 FFT

The algorithm with \\(r=2\\) is derived by expressing the indices in the form

\n

\\[\\tag{3.1} k=k_{m-1} \\cdot 2^{m-1} + … + k_1 \\cdot 2 + k_0 \\]
\\[\\tag{3.2} n=n_{m-1} \\cdot 2^{m-1} + … + n_1 \\cdot 2 + n_0 \\]
where \\( k_v \\in [0,1] \\), for \\( v = 0,…,m-1 \\), and \\( n_v \\in [0,1] \\), for \\( v = 0,…,m-1 \\)
\\(k_v\\) and \\(n_v\\) are the contents of the respective bit positions in the binary representation of \\(k\\) and \\(n\\)

\n

All arrays will now be written as functions of the bits of their indices. With this convention E.q(2.1) is written as
\\[ \\tag{3.3}
\\displaylines{
X(k_{m-1}, …, k_0) = \\sum_{n_0}\\sum_{n_1} … \\sum_{n_{m-1}} x(n_{m-1}, …,n_1, n_0) \\cdot \\omega_N^{kn} \\\\
= \\sum_{n_0}\\sum_{n_1} … \\sum_{n_{m-1}} x(n_{m-1}, …,n_1, n_0) \\cdot \\omega_N^{k(n_{m-1} \\cdot 2^{m-1} + … + n_1 \\cdot 2 + n_0)} \\\\
= \\sum_{n_0}\\sum_{n_1} … \\sum_{n_{m-1}} x(n_{m-1}, …,n_1, n_0) \\cdot \\omega_N^{kn_{m-1} \\cdot 2^{m-1} + … + kn_1 \\cdot 2 + kn_0}
}
\\]
where the sums are over \\(n_v \\in [0,1]\\), for \\(v = 0,1,…,m-1\\)

\n

Since \\[ \\displaylines{
\\omega_N^{kn_{m-1}\\cdot 2^{m-1}} = \\omega_N^{(k_{m-1} \\cdot 2^{m-1} + … + k_1 \\cdot 2 + k_0)n_{m-1}\\cdot 2^{m-1}} \\\\
= \\omega_N^{k_0 n_{m-1} \\cdot 2^{m-1}}
}
\\]
(it is easy to show that all other terms are 1 as \\( \\omega_N^{2^m} = 1 \\), so only \\( k_0\\) is kept)

\n

the innermost sum of E.q(3.3) over \\(n_{m-1} \\), depends only on \\( k_0, n_{m-2}, …, n_0\\) and can be writtern
\\[ \\displaylines{
x_1(k_0, n_{m-2}, …, n_1, n_0) = \\sum_{n_{m-1}} x(n_{m-1}, …, n_0) \\cdot \\omega_N^{k_0 n_{m-1} \\cdot 2^{m-1}}
}
\\]

\n

proceeding to the next innermost sum, over \\( n_{m-2} \\), and so on, and using
\\[ \\displaylines{
\\omega_N^{kn_{m-l}\\cdot 2^{m-l}} = \\omega_N^{(k_{m-1} \\cdot 2^{m-1} + … + k_1 \\cdot 2 + k_0)n_{m-l}\\cdot 2^{m-l}} \\\\
= \\omega_N^{(k_{l-1}\\cdot 2^{l-1} + … + k_0) n_{m-l} \\cdot 2^{m-1}}
}
\\]
one obtains successive arrays
\\[\\displaylines{
x_l(k_0, …, k_{l-1}, n_{m-l-1}, … , n_0) \\\\
= \\sum_{n_{m-l}}x_{l-1}(k_0, …, k_{l-2}, n_{m-l}, …, n_0 ) \\cdot \\omega_N^{(k_{l-1}\\cdot 2^{l-1} + …+ k_0) \\cdot n_{m-l} \\cdot 2^{m-l}}
}\\]
for \\(l = 1,2,…,m \\)

\n

writing out the sum this appears as
\\[ \\tag{3.4} \\displaylines{
x_l(k_0, …, k_{l-1}, n_{m-l-1}, … , n_0) \\\\
= x_{l-1}(k_0, …, k_{l-2}, 0, n_{m-l -1}, …, n_0 ) \\cdot \\omega_N^{(k_{l-1}\\cdot 2^{l-1} + …+ k_0) \\cdot 0 \\cdot 2^{m-l}} \\\\
+ x_{l-1}(k_0, …, k_{l-2}, 1, n_{m-l -1}, …, n_0 ) \\cdot \\omega_N^{(k_{l-1}\\cdot 2^{l-1} + …+ k_0) \\cdot 1 \\cdot 2^{m-l}} \\\\
= x_{l-1}(k_0, …, k_{l-2}, 0, n_{m-l -1}, …, n_0 ) \\\\
+ x_{l-1}(k_0, …, k_{l-2}, 1, n_{m-l-1}, …, n_0 ) \\cdot \\omega_N^{(k_{l-1}\\cdot 2^{l-1} + k_{l-2}\\cdot 2^{l-2 }+k_{l-3}\\cdot 2^{l-3} + …+ k_0) \\cdot 2^{m-l}} \\\\
= x_{l-1}(k_0, …, k_{l-2}, 0, n_{m-l -1}, …, n_0 ) \\\\
+ \\omega_N^{k_{l-1}\\cdot 2^{l-1} \\cdot 2^{m-l} } \\cdot \\omega_N^{k_{l-2}\\cdot 2^{l-2 } \\cdot 2^{m-l}} \\cdot x_{l-1}(k_0, …, k_{l-2}, 1, n_{m-l-1}, …, n_0 ) \\cdot \\omega_N^{(k_{l-3}\\cdot 2^{l-3} + …+ k_0) \\cdot 2^{m-l}} \\\\
= x_{l-1}(k_0, …, k_{l-2}, 0, n_{m-l -1}, …, n_0 ) \\\\
+ \\omega_N^{k_{l-1}\\cdot 2^{m-1} } \\cdot \\omega_N^{k_{l-2}\\cdot 2^{m-2}} \\cdot x_{l-1}(k_0, …, k_{l-2}, 1, n_{m-l-1}, …, n_0 ) \\cdot \\omega_N^{(k_{l-3}\\cdot 2^{l-3} + …+ k_0) \\cdot 2^{m-l}} \\\\
= x_{l-1}(k_0, …, k_{l-2}, 0, n_{m-l -1}, …, n_0 ) \\\\
+ (-1)^{k_{l-1} } \\cdot i^{k_{l-2}} \\cdot x_{l-1}(k_0, …, k_{l-2}, 1, n_{m-l-1}, …, n_0 ) \\cdot \\omega_N^{(k_{l-3}\\cdot 2^{l-3} + …+ k_0) \\cdot 2^{m-l}}
}\\]
according to the indexing convension, this is stored in a location whose index is
\\[ k_0 \\cdot 2^{m-1} + … + k_{l-1} \\cdot 2 ^{m-l} + n_{m-l-1} \\cdot 2^{m-l-1} + … + n_0 \\]

\n

It can be seen in E.q(3.4) that only the two storage locations with indices having 0 and 1 in the \\(2^{m-l}\\) bit position are involved in the computation. Parallel computation is permitted since the operation described by E.q(3.4) can be carried out with all values of \\(k_0, …, k_{l-2} \\), and \\( n_0, …, n_{m-l-1}\\) simultaneously. In some applications it is convenient to use E.q(3.4) to express \\(x_l\\) in terms of \\(x_{l-2}\\), giving what is equivalent to an algorithm with \\(r = 4\\).
the last array calculated gives the desired Fourier sums,

\n

\\[\\tag{3.5}
X(k_{m-1}, …, k_0) = x_{m}(k_0, …, k_{m-1})
\\]
in such an order that the index of an X must have its binary bits put in reverse order to yield its index in the array \\(x_m\\)

\n

references

\n","site":{"data":{}},"excerpt":"","more":"\n\n\n

Fourier Series

innter product of two functions

\\[ <f(x), g(x)> = \\int_{a}^{b} f(x) \\bar{g}(x)dx\\]
\\(\\bar{g}\\) is the congugate of g on complex number system

\n

Fourier series definition

let \\(f(x)\\) be a periodic function over a period of \\([-\\pi, +\\pi]\\)
\"f(x)\"

\n

\\( f(x) \\) can be transformed into below
\\[ f(x) = \\frac{A_{0}}{2} + \\sum_{k=1}^{\\infty}A_{k}cos(kx) + B_{k}sin(kx)\\],
where
\\[ A_{k} = \\frac{1}{\\pi} \\int_{-\\pi}^{+\\pi} f(x)cos(kx)dx = \\frac{1}{||cos(kx)||^2}<f(x),cos(kx)> \\]
\\[ B_{k} = \\frac{1}{\\pi} \\int_{-\\pi}^{+\\pi} f(x)sin(kx)dx = \\frac{1}{||sin(kx)||^2}<f(x),sin(kx)> \\]
where \\(A_{k}, B_{k}\\) is just the innver product of \\(f(x)\\), and \\( cos(kx) \\), \\(sin(kx)\\) respectively. intuitively, it is the projection of \\(f(x)\\) over \\(sin(kx)\\), or \\(cos(kx)\\).

\n

if period is L

\\[ f(x) = \\frac{A_{0}}{2} + \\sum_{k=1}^{\\infty}A_{k}cos(\\frac{2\\pi kx}{L}) + B_{k}sin(\\frac{2\\pi kx}{L})\\],
where
\\[ A_{k} = \\frac{2}{L} \\int_{0}^{L} f(x)cos(\\frac{2\\pi kx}{L})dx\\]
\\[ B_{k} = \\frac{2}{L} \\int_{0}^{L} f(x)sin(\\frac{2\\pi kx}{L})dx\\]

\n

Complex Fourier Series

let’s define \\[ \\psi_k = e^{ikx} = cos(kx) + i \\cdot sin(kx)\\]
we can prove that \\(\\psi_k, and \\psi_j\\) are orthogonal as their inner product is zero
proof
\\[ <\\psi_j, \\psi_k> = \\int_{-\\pi}^{+\\pi} e^{jkx}e^{-ikx}dx = \\int_{-\\pi}^{+\\pi} e^{j(j-k)x}dx = \\frac{1}{i(j-k)}[e^{i(j-k)x}]_{-\\pi}^{\\pi}\\]
it is 0 if \\(j \\ne k\\) , \\(2\\pi \\) if \\(j=k\\)

\n

note conjugate of \\(\\psi_k\\) is \\(e^{-ikx}\\)

\n

\\(f(x) = \\frac{1}{2\\pi} \\sum_{k=-\\infty}^{\\infty} c_k \\psi_k\\), where \\(c_k=<f(x), \\psi_k> = \\int_{-\\pi}^{+\\pi} f(x)e^{-ikx}dx\\)

\n

Discrete Fourier Transform (DFT)

The discrete Fourier transform transforms a sequence of N complex numbers
\\[\\lbrace x_{n} \\rbrace := x_0, x_1,…, x_{N-1} \\] into another sequence of complex numbers,
\\[\\lbrace X_{k} \\rbrace := X_0, X_1,…, X_{N-1} \\] which is defined by
\\[X_{k} = \\sum_{n=0}^{N-1} x_{n} \\cdot e^{-\\frac{i2\\pi}{N}kn}\\]

\n

let the n-th root of unity be \\(\\omega_N = e^{-\\frac{2\\pi i}{N}}\\), we have
\\[ \\tag{1.1} X_{k} = \\sum_{n=0}^{N-1} x_{n} \\cdot \\omega_N^{kn}\\]

\n

inverse DFT

The inverse transform is given by
\\[ \\tag{1.2} x_{n} = \\frac{1}{N}\\sum_{k=0}^{N-1} X_{k} \\cdot \\omega_N^{-kn}\\]

\n

the unitary DFT

Another way of looking at the DFT is to note that the DFT can be expressed as the DFT matrix, a Vandermonde matrix, introduced by Sylvester in 1867.
\\[
\\boldsymbol{F} =
\\begin{bmatrix}
\\displaylines{
\\omega_{N}^{0\\cdot0} & \\omega_{N}^{0\\cdot1} & \\dots & \\omega_{N}^{0\\cdot (N-1)}\\\\
\\omega_{N}^{1\\cdot0} & \\omega_{N}^{1\\cdot1} & \\dots & \\omega_{N}^{2\\cdot (N-1)}\\\\
\\vdots & \\vdots & \\ddots & \\vdots\\\\
\\omega_{N}^{N-1\\cdot0} & \\omega_{N}^{N-1\\cdot1} & \\dots & \\omega_{N}^{N-1\\cdot (N-1)}
}
\\end{bmatrix}
\\]
where \\( \\omega_{N} = e^{-\\frac{i2\\pi}{N}}\\) is a primitive Nth root of unity. (any complex number that yields 1 when raised to some positive integer power n)

\n

\\[ X = F \\cdot x^{T}\\]
\\(x\\) is a column vector

\n

Polynomial & DFT

Polynomial evaluation & DFT

let a polynomial be
$$f(x)=\\sum_{n=0}^{N-1}x_nx^n$$
then,
$$f(\\omega_N^{k})=\\sum_{n=0}^{N-1}x_n\\omega_N^{kn},\\quad k=0,1,2,…,N-1$$
compare this equation with E.q(1.1), we can know that DFT is the evaluations of a polynomial at N points of Nth root of unity, \\(\\omega_N^{0},\\omega_N^1,\\omega_N^2,…,\\omega_N^{N-1}\\)

\n

$$(\\omega_N^{0},X_0),(\\omega_N^1,X_1),(\\omega_N^2,X_2),…,(\\omega_N^{N-1},X_{N-1})$$

\n

Polynomial interpolation & IDFT

According to E.q(1.2), IDFT is the polynomial interpolation (getting original polynomial coefficients).

\n

Fast Fourier Transform (FFT) [3]

The problem is to calculate the below
\\[ \\tag{2.1} X_{k} = \\sum_{n=0}^{N-1} x_{n} \\cdot \\omega_N^{kn}, \\quad k= 0,1,…,N-1\\]
A straightforward calculation using would require \\(N^2\\) operations where “operation” means, as it will throughout this note, a complex multiplication followed by a complex addition.
The FFT algorithm described here iterates on the array and yields the result in less than \\( 2N log_2N \\)

\n

To derive the algorithm, suppose \\(N\\) is a composite, i.e., \\(N = r_1\\cdot r_2\\). Then let the indices in (2.1) be expressed
\\[ k = k_1r_1 + k_0, \\quad k_0 = 0,1,…,r_1-1, \\quad k_1 = 0,1,…, r_2 -1 \\]
\\[ n = n_1r_2 + n_0, \\quad n_0 = 0,1,…,r_2-1, \\quad n_1 = 0,1,…, r_1 -1 \\]

\n

Since \\( n = n_1r_2 + n_0\\), we can write

\n

\\[ \\tag{2.2} X(k_1, k_0) = \\sum_{n_0} \\sum_{n_1}x(n_1,n_0)\\omega_N^{kn} = \\sum_{n_0} \\sum_{n_1}x(n_1,n_0)\\omega_N^{kn_1r_2}\\omega_N^{kn_0} \\]
note \\( x(n_1,n_0)\\) is same to \\(x(n)\\), just the indexing is transformed form 1D to 2D. You can think original 1D array of size N is resized to a matrix of size \\( (r_2, r_1) \\), \\(r_1\\) is number of cols, while \\(r_2\\) is number of rows

\n

Since \\(k = k_1r_1 + k_0\\), we have
\\[ \\omega_N^{kn_1r_2} =\\omega_N^{(k_1r_1+k_0)n_1r_2} = \\omega_N^{(k_1r_1)n_1r_2} \\cdot \\omega_N^{k_0n_1r_2}\\]
according to the property of Nth root of unity, \\( \\omega_N^{N} = \\omega_N^{r_1r_2} =1 \\), \\( \\omega_N^{(k_1r_1)n_1r_2}\\) is also 1.
then we have
\\[ \\omega_N^{kn_1r_2} = \\omega_N^{k_0n_1r_2}\\]
substitute it into E.q 2.2, we get
\\[ \\tag{2.3} X(k_1, k_0) = \\sum_{n_0} \\sum_{n_1}x(n_1,n_0)\\omega_N^{kn} = \\sum_{n_0} \\sum_{n_1}x(n_1,n_0)\\omega_N^{k_0n_1r_2}\\omega_N^{kn_0} \\]
Therefore, the inner sum, over \\(n_1\\), depends only on \\(k_0\\) and \\(n_0\\), and can be defined as a new array,
\\[ \\tag{2.4} x_1(k_0, n_0) = \\sum_{n_1} x(n_1, n_0) \\cdot \\omega_N^{k_0n_1r_2}\\]
E.q(2.3) can be writtern as
\\[ \\tag{2.5} X(k_1, k_0) = \\sum_{n_0} x_1(k_0,n_0)\\omega_N^{kn_0} = \\sum_{n_0} x_1(k_0,n_0)\\omega_N^{(k_1r_1 + k_0)n_0} \\]

\n

There are N elements in the matrix \\(x_1\\), each requiring \\(r_1\\) operations, giving a total of \\(Nr_1\\) operations. Similarly, it takes \\(Nr_2\\) operations to calculate \\(X\\) from \\(x_1\\). Therefore, the two-step algorithm, given by Eq(2.4) and Eq(2.5) requires a total of
\\[ \\tag{2.6} T = N(r_1+r_2) \\]

\n

it is easy to see how successive applications of the above procedure (recursively), starting with its appliation to Eq(2.4) give an m-step algorihtm requiring
\\[ \\tag{2.7} T = N(r_1+r_2 + … + r_m) \\]
where
\\[ N = \\prod_{j=1}^{j=m}r_j\\]
if all \\(r_j\\) are equal to \\(r\\), i.e \\(N = r^m \\), which gives \\( m = log_rN \\)
E.q(2.7) becomes
\\[ \\tag{2.8} T = N(m \\cdot r) = rNm = rN(log_rN) \\]

\n

radix-2 FFT

The algorithm with \\(r=2\\) is derived by expressing the indices in the form

\n

\\[\\tag{3.1} k=k_{m-1} \\cdot 2^{m-1} + … + k_1 \\cdot 2 + k_0 \\]
\\[\\tag{3.2} n=n_{m-1} \\cdot 2^{m-1} + … + n_1 \\cdot 2 + n_0 \\]
where \\( k_v \\in [0,1] \\), for \\( v = 0,…,m-1 \\), and \\( n_v \\in [0,1] \\), for \\( v = 0,…,m-1 \\)
\\(k_v\\) and \\(n_v\\) are the contents of the respective bit positions in the binary representation of \\(k\\) and \\(n\\)

\n

All arrays will now be written as functions of the bits of their indices. With this convention E.q(2.1) is written as
\\[ \\tag{3.3}
\\displaylines{
X(k_{m-1}, …, k_0) = \\sum_{n_0}\\sum_{n_1} … \\sum_{n_{m-1}} x(n_{m-1}, …,n_1, n_0) \\cdot \\omega_N^{kn} \\\\
= \\sum_{n_0}\\sum_{n_1} … \\sum_{n_{m-1}} x(n_{m-1}, …,n_1, n_0) \\cdot \\omega_N^{k(n_{m-1} \\cdot 2^{m-1} + … + n_1 \\cdot 2 + n_0)} \\\\
= \\sum_{n_0}\\sum_{n_1} … \\sum_{n_{m-1}} x(n_{m-1}, …,n_1, n_0) \\cdot \\omega_N^{kn_{m-1} \\cdot 2^{m-1} + … + kn_1 \\cdot 2 + kn_0}
}
\\]
where the sums are over \\(n_v \\in [0,1]\\), for \\(v = 0,1,…,m-1\\)

\n

Since \\[ \\displaylines{
\\omega_N^{kn_{m-1}\\cdot 2^{m-1}} = \\omega_N^{(k_{m-1} \\cdot 2^{m-1} + … + k_1 \\cdot 2 + k_0)n_{m-1}\\cdot 2^{m-1}} \\\\
= \\omega_N^{k_0 n_{m-1} \\cdot 2^{m-1}}
}
\\]
(it is easy to show that all other terms are 1 as \\( \\omega_N^{2^m} = 1 \\), so only \\( k_0\\) is kept)

\n

the innermost sum of E.q(3.3) over \\(n_{m-1} \\), depends only on \\( k_0, n_{m-2}, …, n_0\\) and can be writtern
\\[ \\displaylines{
x_1(k_0, n_{m-2}, …, n_1, n_0) = \\sum_{n_{m-1}} x(n_{m-1}, …, n_0) \\cdot \\omega_N^{k_0 n_{m-1} \\cdot 2^{m-1}}
}
\\]

\n

proceeding to the next innermost sum, over \\( n_{m-2} \\), and so on, and using
\\[ \\displaylines{
\\omega_N^{kn_{m-l}\\cdot 2^{m-l}} = \\omega_N^{(k_{m-1} \\cdot 2^{m-1} + … + k_1 \\cdot 2 + k_0)n_{m-l}\\cdot 2^{m-l}} \\\\
= \\omega_N^{(k_{l-1}\\cdot 2^{l-1} + … + k_0) n_{m-l} \\cdot 2^{m-1}}
}
\\]
one obtains successive arrays
\\[\\displaylines{
x_l(k_0, …, k_{l-1}, n_{m-l-1}, … , n_0) \\\\
= \\sum_{n_{m-l}}x_{l-1}(k_0, …, k_{l-2}, n_{m-l}, …, n_0 ) \\cdot \\omega_N^{(k_{l-1}\\cdot 2^{l-1} + …+ k_0) \\cdot n_{m-l} \\cdot 2^{m-l}}
}\\]
for \\(l = 1,2,…,m \\)

\n

writing out the sum this appears as
\\[ \\tag{3.4} \\displaylines{
x_l(k_0, …, k_{l-1}, n_{m-l-1}, … , n_0) \\\\
= x_{l-1}(k_0, …, k_{l-2}, 0, n_{m-l -1}, …, n_0 ) \\cdot \\omega_N^{(k_{l-1}\\cdot 2^{l-1} + …+ k_0) \\cdot 0 \\cdot 2^{m-l}} \\\\
+ x_{l-1}(k_0, …, k_{l-2}, 1, n_{m-l -1}, …, n_0 ) \\cdot \\omega_N^{(k_{l-1}\\cdot 2^{l-1} + …+ k_0) \\cdot 1 \\cdot 2^{m-l}} \\\\
= x_{l-1}(k_0, …, k_{l-2}, 0, n_{m-l -1}, …, n_0 ) \\\\
+ x_{l-1}(k_0, …, k_{l-2}, 1, n_{m-l-1}, …, n_0 ) \\cdot \\omega_N^{(k_{l-1}\\cdot 2^{l-1} + k_{l-2}\\cdot 2^{l-2 }+k_{l-3}\\cdot 2^{l-3} + …+ k_0) \\cdot 2^{m-l}} \\\\
= x_{l-1}(k_0, …, k_{l-2}, 0, n_{m-l -1}, …, n_0 ) \\\\
+ \\omega_N^{k_{l-1}\\cdot 2^{l-1} \\cdot 2^{m-l} } \\cdot \\omega_N^{k_{l-2}\\cdot 2^{l-2 } \\cdot 2^{m-l}} \\cdot x_{l-1}(k_0, …, k_{l-2}, 1, n_{m-l-1}, …, n_0 ) \\cdot \\omega_N^{(k_{l-3}\\cdot 2^{l-3} + …+ k_0) \\cdot 2^{m-l}} \\\\
= x_{l-1}(k_0, …, k_{l-2}, 0, n_{m-l -1}, …, n_0 ) \\\\
+ \\omega_N^{k_{l-1}\\cdot 2^{m-1} } \\cdot \\omega_N^{k_{l-2}\\cdot 2^{m-2}} \\cdot x_{l-1}(k_0, …, k_{l-2}, 1, n_{m-l-1}, …, n_0 ) \\cdot \\omega_N^{(k_{l-3}\\cdot 2^{l-3} + …+ k_0) \\cdot 2^{m-l}} \\\\
= x_{l-1}(k_0, …, k_{l-2}, 0, n_{m-l -1}, …, n_0 ) \\\\
+ (-1)^{k_{l-1} } \\cdot i^{k_{l-2}} \\cdot x_{l-1}(k_0, …, k_{l-2}, 1, n_{m-l-1}, …, n_0 ) \\cdot \\omega_N^{(k_{l-3}\\cdot 2^{l-3} + …+ k_0) \\cdot 2^{m-l}}
}\\]
according to the indexing convension, this is stored in a location whose index is
\\[ k_0 \\cdot 2^{m-1} + … + k_{l-1} \\cdot 2 ^{m-l} + n_{m-l-1} \\cdot 2^{m-l-1} + … + n_0 \\]

\n

It can be seen in E.q(3.4) that only the two storage locations with indices having 0 and 1 in the \\(2^{m-l}\\) bit position are involved in the computation. Parallel computation is permitted since the operation described by E.q(3.4) can be carried out with all values of \\(k_0, …, k_{l-2} \\), and \\( n_0, …, n_{m-l-1}\\) simultaneously. In some applications it is convenient to use E.q(3.4) to express \\(x_l\\) in terms of \\(x_{l-2}\\), giving what is equivalent to an algorithm with \\(r = 4\\).
the last array calculated gives the desired Fourier sums,

\n

\\[\\tag{3.5}
X(k_{m-1}, …, k_0) = x_{m}(k_0, …, k_{m-1})
\\]
in such an order that the index of an X must have its binary bits put in reverse order to yield its index in the array \\(x_m\\)

\n

references

\n"},{"title":"geth start","date":"2022-11-01T10:15:12.000Z","_content":"\n## build from source\n```\ngit clone https://github.com/ethereum/go-ethereum.git\ncd go-ethereum\nmake geth\n```\n\n## understanding geth config\ngeth config type is defined in /cmd/geth/config.go\n```go\ntype gethConfig struct {\n\tEth ethconfig.Config\n\tNode node.Config\n\tEthstats ethstatsConfig\n\tMetrics metrics.Config\n}\n```\n- **ethconfig** (eth/ethconfig/config.go)\ncontains configuration options for of the ETH and LES(light node) protocols, such as NetworkId, SyncMode, txpool.Config, database options\n- **nodeConfig** (node/config.go)\nrepresents a small collection of configuration values to fine tune the P2P network layer of a protocol stack. These values can be further extended by all registered services. such as p2p.Config, DataDir, KeyStoreDir, HTTPHost, HTTPModules(eth,net,web3), WSHost\n- **metrics.Config** (metrics/config.go)\ncontains the configuration for the metric collection, such as InfluxDBEndpoint, etc\n- **ethstatsConfig**\nonly one URL entry\n\ngeth provides default config in the above files. user config file path is given by the below flag\n```go\nconfigFileFlag = &cli.StringFlag{\n\t\tName: \"config\",\n\t\tUsage: \"TOML configuration file\",\n\t\tCategory: flags.EthCategory,\n\t}\n```\n\nThe config file should be a .toml file. A convenient way to create a config file is to get Geth to create one for you and use it as a template. To do this, use the dumpconfig command, saving the result to a .toml file. Note that you also need to explicitly provide the network_id on the command line for the public testnets such as Sepolia or Geoerli:\n```\n./geth --sepolia dumpconfig > geth-config.toml\n```\nto specify path to config file\n```\ngeth --sepolia --config geth-config.toml\n```\n\n## key configs\n- [Eth].TxLookupLimit \nNumber of recent blocks to maintain transactions index for (default = about one year, 0 = entire chain), default: 2350000\n- [Node].BootstrapNodes\nused to establish connectivity with the rest of the network.\ngeth provides default bootstrapNodes in file `params/bootnodes.go`\n- [Metrics_AND_STATS].ethstats\nReporting URL of a ethstats service (nodename:secret@host:port), [more detail](https://geth.ethereum.org/docs/monitoring/ethstats)\n- SyncMode\n- TrieDirtyCache\n- NoPruning\n- TrieCleanCacheJournal e.g triecache\n## how geth starts\n\n![geth starts](/images/geth_starts.drawio.png)\nthe main func is in `cmd/geth/main.go`\n```go\nfunc main() {\n\tif err := app.Run(os.Args); err != nil {\n\t\tfmt.Fprintln(os.Stderr, err)\n\t\tos.Exit(1)\n\t}\n}\n```\nthe main() function is very short, and its main function is to start a tool for parsing command line commands: `gopkg.in/urfave/cli.v1`. Going deeper, we will find that `app.Action = geth` will be called when the cli app is initialized to call the geth() function\n```go\nfunc init() {\n\t// Initialize the CLI app and start Geth\n\tapp.Action = geth\n // ....\n}\n```\ngeth is the main entry point into the system if no special subcommand is run. It creates a default node based on the command line arguments and runs it in blocking mode, waiting for it to be shut down.\n```go\nfunc geth(ctx *cli.Context) error {\n\tif args := ctx.Args().Slice(); len(args) > 0 {\n\t\treturn fmt.Errorf(\"invalid command: %q\", args[0])\n\t}\n\n\tprepare(ctx)\n\tstack, backend := makeFullNode(ctx)\n\tdefer stack.Close()\n\n\tstartNode(ctx, stack, backend, false)\n\tstack.Wait()\n\treturn nil\n}\n```\nIn the geth() function, there are three important function calls, namely: `prepare()`, `makeFullNode()`, and `startNode()`.\n\n### prepare\nThe implementation of the prepare() function is in the current main.go file. It is mainly used to set some configurations required for node initialization.\n\n### makeFullNode\nThe implementation of the `makeFullNode()` function is located in the `cmd/geth/config.go` file. It will load the context of the command and apply user given configuration; and generate instances of `stack` and `backend`. Among them, `stack` is an instance of `Node` type (Node is the top-level instance in the life cycle of geth. It is responsible for managing high-level abstractions such as P2P Server, Http Server, and Database in the node. The definition of the Node type is located in the `node/node.go` file), which is initialized by calling `makeConfigNode()` function through `makeFullNode()` function. inside `makeFullNode`, it calls `node.New(&cfg.Node)` to initiate a node. During instantiating of node, it invokes `rpc.NewServer()` to create a new rpc server and put in the field `inprocHandler`. it registers `rpc` api namespace by default.\n\nThe `backend` here is an interface of `ethapi.Backend` type, which provides the basic functions needed to obtain the runtime of the Ethereum execution layer. Its definition is located in `internal/ethapi/backend.go`. Since there are many functions in this interface, we have selected some of the key functions as below for a glimpse of its functionality. `backend` is created by calling `backend, eth := utils.RegisterEthService(stack, &cfg.Eth)`. Inside, it calls `eth.New(stack, cfg)` to create `backend` instance. During `backend` initiating, it opens database (`chainDb, err := stack.OpenDatabaseWithFreezer(\"chaindata\", config.DatabaseCache, config.DatabaseHandles, config.DatabaseFreezer, \"eth/db/chaindata/\", false)`). Further, it creates consensus engine, `engine := ethconfig.CreateConsensusEngine(stack, ðashConfig, cliqueConfig, config.Miner.Notify, config.Miner.Noverify, chainDb)`. goerli testnet use POA consensus (clique). \n```go\ntype Backend interface {\n\tSyncProgress() ethereum.SyncProgress\n\tSuggestGasTipCap(ctx context.Context) (*big.Int, error)\n\tChainDb() ethdb.Database\n\tAccountManager() *accounts.Manager\n\tExtRPCEnabled() bool\n\tRPCGasCap() uint64 // global gas cap for eth_call over rpc: DoS protection\n\tRPCEVMTimeout() time.Duration // global timeout for eth_call over rpc: DoS protection\n\tRPCTxFeeCap() float64 // global tx fee cap for all transaction related APIs\n\tUnprotectedAllowed() bool // allows only for EIP155 transactions.\n\tSetHead(number uint64)\n\tHeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Header, error)\n\tHeaderByHash(ctx context.Context, hash common.Hash) (*types.Header, error)\n\tHeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*types.Header, error)\n\tCurrentHeader() *types.Header\n\tCurrentBlock() *types.Header\n\tBlockByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Block, error)\n\tBlockByHash(ctx context.Context, hash common.Hash) (*types.Block, error)\n\tBlockByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*types.Block, error)\n\tStateAndHeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*state.StateDB, *types.Header, error)\n\tStateAndHeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*state.StateDB, *types.Header, error)\n\tPendingBlockAndReceipts() (*types.Block, types.Receipts)\n\tGetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error)\n\tGetTd(ctx context.Context, hash common.Hash) *big.Int\n\tGetEVM(ctx context.Context, msg *core.Message, state *state.StateDB, header *types.Header, vmConfig *vm.Config) (*vm.EVM, func() error, error)\n\tSubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription\n\tSubscribeChainHeadEvent(ch chan<- core.ChainHeadEvent) event.Subscription\n\tSubscribeChainSideEvent(ch chan<- core.ChainSideEvent) event.Subscription\n\tSendTx(ctx context.Context, signedTx *types.Transaction) error\n\tGetTransaction(ctx context.Context, txHash common.Hash) (*types.Transaction, common.Hash, uint64, uint64, error)\n\tGetPoolTransactions() (types.Transactions, error)\n\tGetPoolTransaction(txHash common.Hash) *types.Transaction\n\tGetPoolNonce(ctx context.Context, addr common.Address) (uint64, error)\n\tStats() (pending int, queued int)\n\tTxPoolContent() (map[common.Address]types.Transactions, map[common.Address]types.Transactions)\n\tTxPoolContentFrom(addr common.Address) (types.Transactions, types.Transactions)\n\tSubscribeNewTxsEvent(chan<- core.NewTxsEvent) event.Subscription\n\tChainConfig() *params.ChainConfig\n\tEngine() consensus.Engine\n\tGetBody(ctx context.Context, hash common.Hash, number rpc.BlockNumber) (*types.Body, error)\n\tGetLogs(ctx context.Context, blockHash common.Hash, number uint64) ([][]*types.Log, error)\n\tSubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription\n\tSubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription\n\tSubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription\n\tBloomStatus() (uint64, uint64)\n\tServiceFilter(ctx context.Context, session *bloombits.MatcherSession)\n}\n```\n\nIf readers want to customize some new RPC APIs, they can define functions in the /internal/ethapi.Backend interface and add specific implementations to EthAPIBackend\n\n### startNode\nThe last key function, `startNode()`, is to officially start an Ethereum execution layer node. It starts the Stack instance (Node) by calling the utils.StartNode() function which triggers the Node.Start() function. In the Node.Start() function, it traverses the backend instances registered in `Node.lifecycles` and starts them. In addition, in the startNode() function, the unlockAccounts() function is still called, and the unlocked wallet is registered in the stack, and the RPClient module that interacts with local Geth is created through the stack.Attach() function\n\nAt the end of the geth() function, the function executes `stack.Wait()`, so that the main thread enters the blocking state, and the services of other functional modules are distributed to other sub-coroutines for maintenance\n\n## Node\nAs we mentioned earlier, the Node type belongs to the top-level instance in the life cycle of geth, and it is responsible for being the administrator of the high-level abstract module communicating with the outside world, such as managing rpc server, http server, Web Socket, and P2P Server external interface . At the same time, Node maintains the back-end instances and services (lifecycles []Lifecycle) required for node operation, such as the Ethereum type we mentioned above that is responsible for the specific Service\n```go\ntype Node struct {\n\teventmux *event.TypeMux\n\tconfig *Config\n\taccman *accounts.Manager\n\tlog log.Logger\n\tkeyDir string // key store directory\n\tkeyDirTemp bool // If true, key directory will be removed by Stop\n\tdirLock *flock.Flock // prevents concurrent use of instance directory\n\tstop chan struct{} // Channel to wait for termination notifications\n\tserver *p2p.Server // Currently running P2P networking layer\n\tstartStopLock sync.Mutex // Start/Stop are protected by an additional lock\n\tstate int // Tracks state of node lifecycle\n\n\tlock sync.Mutex\n\tlifecycles []Lifecycle // All registered backends, services, and auxiliary services that have a lifecycle\n\trpcAPIs []rpc.API // List of APIs currently provided by the node\n\thttp *httpServer //\n\tws *httpServer //\n\thttpAuth *httpServer //\n\twsAuth *httpServer //\n\tipc *ipcServer // Stores information about the ipc http server\n\tinprocHandler *rpc.Server // In-process RPC request handler to process the API requests\n\n\tdatabases map[*closeTrackingDB]struct{} // All open databases\n}\n```\n\n### close node\nAs mentioned earlier, the main thread of the entire program is blocked because of calling `stack.Wait()`. We can see that a channel called `stop` is declared in the Node structure. Since this Channel has not been assigned a value, the main process of the entire geth enters the blocking state, and continues to execute other business coroutines concurrently\n```go\n// Wait blocks until the node is closed.\nfunc (n *Node) Wait() {\n <-n.stop\n}\n```\nWhen the Channel n.stop is assigned a value, the geth main function will stop the current blocking state and start to perform a series of corresponding resource release operations.\nIt is worth noting that in the current codebase of go-ethereum, the blocking state of the main process is not ended directly by assigning a value to the stop channel, but a more concise and rude way is used: call the close() function directly Close the Channel. We can find the related implementation in node.doClose(). close() is a native function of go language, used when closing Channel.\n```go\n// doClose releases resources acquired by New(), collecting errors.\nfunc (n *Node) doClose(errs []error) error {\n // Close databases. This needs the lock because it needs to\n // synchronize with OpenDatabase*.\n n.lock.Lock()\n n.state = closedState\n errs = append(errs, n.closeDatabases()...)\n n.lock.Unlock()\n\n if err := n.accman.Close(); err != nil {\n errs = append(errs, err)\n }\n if n.keyDirTemp {\n if err := os.RemoveAll(n.keyDir); err != nil {\n errs = append(errs, err)\n }\n }\n\n // Release instance directory lock.\n n.closeDataDir()\n\n // Unblock n.Wait.\n close(n.stop)\n\n // Report any errors that might have occurred.\n switch len(errs) {\n case 0:\n return nil\n case 1:\n return errs[0]\n default:\n return fmt.Errorf(\"%v\", errs)\n }\n}\n```\n\n## Ethereum Backend\nWe can find the definition of the Ethereum structure in eth/backend.go. The member variables and receiving methods contained in this structure implement all the functions and data structures required by an Ethereum full node. We can see in the following code definition that the Ethereum structure contains several core data structures such as TxPool, Blockchain, consensus.Engine, and miner as member variables.\n```go\ntype Ethereum struct {\n\tconfig *ethconfig.Config\n\n\t// Handlers\n\ttxPool *txpool.TxPool\n\tblockchain *core.BlockChain\n\thandler *handler\n\tethDialCandidates enode.Iterator\n\tsnapDialCandidates enode.Iterator\n\tmerger *consensus.Merger\n\n\t// DB interfaces\n\tchainDb ethdb.Database // Block chain database\n\n\teventMux *event.TypeMux\n\tengine consensus.Engine\n\taccountManager *accounts.Manager\n\n\tbloomRequests chan chan *bloombits.Retrieval // Channel receiving bloom data retrieval requests\n\tbloomIndexer *core.ChainIndexer // Bloom indexer operating during block imports\n\tcloseBloomHandler chan struct{}\n\n\tAPIBackend *EthAPIBackend\n\n\tminer *miner.Miner\n\tgasPrice *big.Int\n\tetherbase common.Address\n\n\tnetworkID uint64\n\tnetRPCService *ethapi.NetAPI\n\n\tp2pServer *p2p.Server\n\n\tlock sync.RWMutex // Protects the variadic fields (e.g. gas price and etherbase)\n\n\tshutdownTracker *shutdowncheck.ShutdownTracker // Tracks if and when the node has shutdown ungracefully\n}\n```\nNodes start and stop Mining by calling `Ethereum.StartMining()` and `Ethereum.StopMining()`. Setting the profit account of Mining is achieved by calling `Ethereum.SetEtherbase()`\nHere we pay extra attention to the member variable `handler`. The definition of `handler` is in `eth/handler.go`.\nFrom a macro point of view, the main workflow of a node needs to: 1. Obtain/synchronize Transaction and Block data from the network 2. Add the Block obtained from the network to the Blockchain. The handler is responsible for providing the function of synchronizing blocks and transaction data, for example, `downloader.Downloader` is responsible for synchronizing Block from the network, and `fetcher.TxFetcher` is responsible for synchronizing transactions from the network","source":"_posts/geth/code_analysis/geth.0.get.start.md","raw":"---\ntitle: geth start\ndate: 2022-11-01 18:15:12\ntags: [blockchain,geth]\n---\n\n## build from source\n```\ngit clone https://github.com/ethereum/go-ethereum.git\ncd go-ethereum\nmake geth\n```\n\n## understanding geth config\ngeth config type is defined in /cmd/geth/config.go\n```go\ntype gethConfig struct {\n\tEth ethconfig.Config\n\tNode node.Config\n\tEthstats ethstatsConfig\n\tMetrics metrics.Config\n}\n```\n- **ethconfig** (eth/ethconfig/config.go)\ncontains configuration options for of the ETH and LES(light node) protocols, such as NetworkId, SyncMode, txpool.Config, database options\n- **nodeConfig** (node/config.go)\nrepresents a small collection of configuration values to fine tune the P2P network layer of a protocol stack. These values can be further extended by all registered services. such as p2p.Config, DataDir, KeyStoreDir, HTTPHost, HTTPModules(eth,net,web3), WSHost\n- **metrics.Config** (metrics/config.go)\ncontains the configuration for the metric collection, such as InfluxDBEndpoint, etc\n- **ethstatsConfig**\nonly one URL entry\n\ngeth provides default config in the above files. user config file path is given by the below flag\n```go\nconfigFileFlag = &cli.StringFlag{\n\t\tName: \"config\",\n\t\tUsage: \"TOML configuration file\",\n\t\tCategory: flags.EthCategory,\n\t}\n```\n\nThe config file should be a .toml file. A convenient way to create a config file is to get Geth to create one for you and use it as a template. To do this, use the dumpconfig command, saving the result to a .toml file. Note that you also need to explicitly provide the network_id on the command line for the public testnets such as Sepolia or Geoerli:\n```\n./geth --sepolia dumpconfig > geth-config.toml\n```\nto specify path to config file\n```\ngeth --sepolia --config geth-config.toml\n```\n\n## key configs\n- [Eth].TxLookupLimit \nNumber of recent blocks to maintain transactions index for (default = about one year, 0 = entire chain), default: 2350000\n- [Node].BootstrapNodes\nused to establish connectivity with the rest of the network.\ngeth provides default bootstrapNodes in file `params/bootnodes.go`\n- [Metrics_AND_STATS].ethstats\nReporting URL of a ethstats service (nodename:secret@host:port), [more detail](https://geth.ethereum.org/docs/monitoring/ethstats)\n- SyncMode\n- TrieDirtyCache\n- NoPruning\n- TrieCleanCacheJournal e.g triecache\n## how geth starts\n\n![geth starts](/images/geth_starts.drawio.png)\nthe main func is in `cmd/geth/main.go`\n```go\nfunc main() {\n\tif err := app.Run(os.Args); err != nil {\n\t\tfmt.Fprintln(os.Stderr, err)\n\t\tos.Exit(1)\n\t}\n}\n```\nthe main() function is very short, and its main function is to start a tool for parsing command line commands: `gopkg.in/urfave/cli.v1`. Going deeper, we will find that `app.Action = geth` will be called when the cli app is initialized to call the geth() function\n```go\nfunc init() {\n\t// Initialize the CLI app and start Geth\n\tapp.Action = geth\n // ....\n}\n```\ngeth is the main entry point into the system if no special subcommand is run. It creates a default node based on the command line arguments and runs it in blocking mode, waiting for it to be shut down.\n```go\nfunc geth(ctx *cli.Context) error {\n\tif args := ctx.Args().Slice(); len(args) > 0 {\n\t\treturn fmt.Errorf(\"invalid command: %q\", args[0])\n\t}\n\n\tprepare(ctx)\n\tstack, backend := makeFullNode(ctx)\n\tdefer stack.Close()\n\n\tstartNode(ctx, stack, backend, false)\n\tstack.Wait()\n\treturn nil\n}\n```\nIn the geth() function, there are three important function calls, namely: `prepare()`, `makeFullNode()`, and `startNode()`.\n\n### prepare\nThe implementation of the prepare() function is in the current main.go file. It is mainly used to set some configurations required for node initialization.\n\n### makeFullNode\nThe implementation of the `makeFullNode()` function is located in the `cmd/geth/config.go` file. It will load the context of the command and apply user given configuration; and generate instances of `stack` and `backend`. Among them, `stack` is an instance of `Node` type (Node is the top-level instance in the life cycle of geth. It is responsible for managing high-level abstractions such as P2P Server, Http Server, and Database in the node. The definition of the Node type is located in the `node/node.go` file), which is initialized by calling `makeConfigNode()` function through `makeFullNode()` function. inside `makeFullNode`, it calls `node.New(&cfg.Node)` to initiate a node. During instantiating of node, it invokes `rpc.NewServer()` to create a new rpc server and put in the field `inprocHandler`. it registers `rpc` api namespace by default.\n\nThe `backend` here is an interface of `ethapi.Backend` type, which provides the basic functions needed to obtain the runtime of the Ethereum execution layer. Its definition is located in `internal/ethapi/backend.go`. Since there are many functions in this interface, we have selected some of the key functions as below for a glimpse of its functionality. `backend` is created by calling `backend, eth := utils.RegisterEthService(stack, &cfg.Eth)`. Inside, it calls `eth.New(stack, cfg)` to create `backend` instance. During `backend` initiating, it opens database (`chainDb, err := stack.OpenDatabaseWithFreezer(\"chaindata\", config.DatabaseCache, config.DatabaseHandles, config.DatabaseFreezer, \"eth/db/chaindata/\", false)`). Further, it creates consensus engine, `engine := ethconfig.CreateConsensusEngine(stack, ðashConfig, cliqueConfig, config.Miner.Notify, config.Miner.Noverify, chainDb)`. goerli testnet use POA consensus (clique). \n```go\ntype Backend interface {\n\tSyncProgress() ethereum.SyncProgress\n\tSuggestGasTipCap(ctx context.Context) (*big.Int, error)\n\tChainDb() ethdb.Database\n\tAccountManager() *accounts.Manager\n\tExtRPCEnabled() bool\n\tRPCGasCap() uint64 // global gas cap for eth_call over rpc: DoS protection\n\tRPCEVMTimeout() time.Duration // global timeout for eth_call over rpc: DoS protection\n\tRPCTxFeeCap() float64 // global tx fee cap for all transaction related APIs\n\tUnprotectedAllowed() bool // allows only for EIP155 transactions.\n\tSetHead(number uint64)\n\tHeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Header, error)\n\tHeaderByHash(ctx context.Context, hash common.Hash) (*types.Header, error)\n\tHeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*types.Header, error)\n\tCurrentHeader() *types.Header\n\tCurrentBlock() *types.Header\n\tBlockByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Block, error)\n\tBlockByHash(ctx context.Context, hash common.Hash) (*types.Block, error)\n\tBlockByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*types.Block, error)\n\tStateAndHeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*state.StateDB, *types.Header, error)\n\tStateAndHeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*state.StateDB, *types.Header, error)\n\tPendingBlockAndReceipts() (*types.Block, types.Receipts)\n\tGetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error)\n\tGetTd(ctx context.Context, hash common.Hash) *big.Int\n\tGetEVM(ctx context.Context, msg *core.Message, state *state.StateDB, header *types.Header, vmConfig *vm.Config) (*vm.EVM, func() error, error)\n\tSubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription\n\tSubscribeChainHeadEvent(ch chan<- core.ChainHeadEvent) event.Subscription\n\tSubscribeChainSideEvent(ch chan<- core.ChainSideEvent) event.Subscription\n\tSendTx(ctx context.Context, signedTx *types.Transaction) error\n\tGetTransaction(ctx context.Context, txHash common.Hash) (*types.Transaction, common.Hash, uint64, uint64, error)\n\tGetPoolTransactions() (types.Transactions, error)\n\tGetPoolTransaction(txHash common.Hash) *types.Transaction\n\tGetPoolNonce(ctx context.Context, addr common.Address) (uint64, error)\n\tStats() (pending int, queued int)\n\tTxPoolContent() (map[common.Address]types.Transactions, map[common.Address]types.Transactions)\n\tTxPoolContentFrom(addr common.Address) (types.Transactions, types.Transactions)\n\tSubscribeNewTxsEvent(chan<- core.NewTxsEvent) event.Subscription\n\tChainConfig() *params.ChainConfig\n\tEngine() consensus.Engine\n\tGetBody(ctx context.Context, hash common.Hash, number rpc.BlockNumber) (*types.Body, error)\n\tGetLogs(ctx context.Context, blockHash common.Hash, number uint64) ([][]*types.Log, error)\n\tSubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription\n\tSubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription\n\tSubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription\n\tBloomStatus() (uint64, uint64)\n\tServiceFilter(ctx context.Context, session *bloombits.MatcherSession)\n}\n```\n\nIf readers want to customize some new RPC APIs, they can define functions in the /internal/ethapi.Backend interface and add specific implementations to EthAPIBackend\n\n### startNode\nThe last key function, `startNode()`, is to officially start an Ethereum execution layer node. It starts the Stack instance (Node) by calling the utils.StartNode() function which triggers the Node.Start() function. In the Node.Start() function, it traverses the backend instances registered in `Node.lifecycles` and starts them. In addition, in the startNode() function, the unlockAccounts() function is still called, and the unlocked wallet is registered in the stack, and the RPClient module that interacts with local Geth is created through the stack.Attach() function\n\nAt the end of the geth() function, the function executes `stack.Wait()`, so that the main thread enters the blocking state, and the services of other functional modules are distributed to other sub-coroutines for maintenance\n\n## Node\nAs we mentioned earlier, the Node type belongs to the top-level instance in the life cycle of geth, and it is responsible for being the administrator of the high-level abstract module communicating with the outside world, such as managing rpc server, http server, Web Socket, and P2P Server external interface . At the same time, Node maintains the back-end instances and services (lifecycles []Lifecycle) required for node operation, such as the Ethereum type we mentioned above that is responsible for the specific Service\n```go\ntype Node struct {\n\teventmux *event.TypeMux\n\tconfig *Config\n\taccman *accounts.Manager\n\tlog log.Logger\n\tkeyDir string // key store directory\n\tkeyDirTemp bool // If true, key directory will be removed by Stop\n\tdirLock *flock.Flock // prevents concurrent use of instance directory\n\tstop chan struct{} // Channel to wait for termination notifications\n\tserver *p2p.Server // Currently running P2P networking layer\n\tstartStopLock sync.Mutex // Start/Stop are protected by an additional lock\n\tstate int // Tracks state of node lifecycle\n\n\tlock sync.Mutex\n\tlifecycles []Lifecycle // All registered backends, services, and auxiliary services that have a lifecycle\n\trpcAPIs []rpc.API // List of APIs currently provided by the node\n\thttp *httpServer //\n\tws *httpServer //\n\thttpAuth *httpServer //\n\twsAuth *httpServer //\n\tipc *ipcServer // Stores information about the ipc http server\n\tinprocHandler *rpc.Server // In-process RPC request handler to process the API requests\n\n\tdatabases map[*closeTrackingDB]struct{} // All open databases\n}\n```\n\n### close node\nAs mentioned earlier, the main thread of the entire program is blocked because of calling `stack.Wait()`. We can see that a channel called `stop` is declared in the Node structure. Since this Channel has not been assigned a value, the main process of the entire geth enters the blocking state, and continues to execute other business coroutines concurrently\n```go\n// Wait blocks until the node is closed.\nfunc (n *Node) Wait() {\n <-n.stop\n}\n```\nWhen the Channel n.stop is assigned a value, the geth main function will stop the current blocking state and start to perform a series of corresponding resource release operations.\nIt is worth noting that in the current codebase of go-ethereum, the blocking state of the main process is not ended directly by assigning a value to the stop channel, but a more concise and rude way is used: call the close() function directly Close the Channel. We can find the related implementation in node.doClose(). close() is a native function of go language, used when closing Channel.\n```go\n// doClose releases resources acquired by New(), collecting errors.\nfunc (n *Node) doClose(errs []error) error {\n // Close databases. This needs the lock because it needs to\n // synchronize with OpenDatabase*.\n n.lock.Lock()\n n.state = closedState\n errs = append(errs, n.closeDatabases()...)\n n.lock.Unlock()\n\n if err := n.accman.Close(); err != nil {\n errs = append(errs, err)\n }\n if n.keyDirTemp {\n if err := os.RemoveAll(n.keyDir); err != nil {\n errs = append(errs, err)\n }\n }\n\n // Release instance directory lock.\n n.closeDataDir()\n\n // Unblock n.Wait.\n close(n.stop)\n\n // Report any errors that might have occurred.\n switch len(errs) {\n case 0:\n return nil\n case 1:\n return errs[0]\n default:\n return fmt.Errorf(\"%v\", errs)\n }\n}\n```\n\n## Ethereum Backend\nWe can find the definition of the Ethereum structure in eth/backend.go. The member variables and receiving methods contained in this structure implement all the functions and data structures required by an Ethereum full node. We can see in the following code definition that the Ethereum structure contains several core data structures such as TxPool, Blockchain, consensus.Engine, and miner as member variables.\n```go\ntype Ethereum struct {\n\tconfig *ethconfig.Config\n\n\t// Handlers\n\ttxPool *txpool.TxPool\n\tblockchain *core.BlockChain\n\thandler *handler\n\tethDialCandidates enode.Iterator\n\tsnapDialCandidates enode.Iterator\n\tmerger *consensus.Merger\n\n\t// DB interfaces\n\tchainDb ethdb.Database // Block chain database\n\n\teventMux *event.TypeMux\n\tengine consensus.Engine\n\taccountManager *accounts.Manager\n\n\tbloomRequests chan chan *bloombits.Retrieval // Channel receiving bloom data retrieval requests\n\tbloomIndexer *core.ChainIndexer // Bloom indexer operating during block imports\n\tcloseBloomHandler chan struct{}\n\n\tAPIBackend *EthAPIBackend\n\n\tminer *miner.Miner\n\tgasPrice *big.Int\n\tetherbase common.Address\n\n\tnetworkID uint64\n\tnetRPCService *ethapi.NetAPI\n\n\tp2pServer *p2p.Server\n\n\tlock sync.RWMutex // Protects the variadic fields (e.g. gas price and etherbase)\n\n\tshutdownTracker *shutdowncheck.ShutdownTracker // Tracks if and when the node has shutdown ungracefully\n}\n```\nNodes start and stop Mining by calling `Ethereum.StartMining()` and `Ethereum.StopMining()`. Setting the profit account of Mining is achieved by calling `Ethereum.SetEtherbase()`\nHere we pay extra attention to the member variable `handler`. The definition of `handler` is in `eth/handler.go`.\nFrom a macro point of view, the main workflow of a node needs to: 1. Obtain/synchronize Transaction and Block data from the network 2. Add the Block obtained from the network to the Blockchain. The handler is responsible for providing the function of synchronizing blocks and transaction data, for example, `downloader.Downloader` is responsible for synchronizing Block from the network, and `fetcher.TxFetcher` is responsible for synchronizing transactions from the network","slug":"geth/code_analysis/geth.0.get.start","published":1,"updated":"2023-11-05T04:21:13.733Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8dv0023qwsjbe4aeis9","content":"

build from source

1
2
3
git clone https://github.com/ethereum/go-ethereum.git
cd go-ethereum
make geth
\n\n

understanding geth config

geth config type is defined in /cmd/geth/config.go

\n
1
2
3
4
5
6
type gethConfig struct {
\tEth      ethconfig.Config
\tNode     node.Config
\tEthstats ethstatsConfig
\tMetrics  metrics.Config
}
\n
    \n
  • ethconfig (eth/ethconfig/config.go)
    contains configuration options for of the ETH and LES(light node) protocols, such as NetworkId, SyncMode, txpool.Config, database options
    • \n
  • nodeConfig (node/config.go)
    represents a small collection of configuration values to fine tune the P2P network layer of a protocol stack. These values can be further extended by all registered services. such as p2p.Config, DataDir, KeyStoreDir, HTTPHost, HTTPModules(eth,net,web3), WSHost
    • \n
  • metrics.Config (metrics/config.go)
    contains the configuration for the metric collection, such as InfluxDBEndpoint, etc
    • \n
  • ethstatsConfig
    only one URL entry
    • \n
\n

geth provides default config in the above files. user config file path is given by the below flag

\n
1
2
3
4
5
configFileFlag = &cli.StringFlag{
\t\tName:     "config",
\t\tUsage:    "TOML configuration file",
\t\tCategory: flags.EthCategory,
\t}
\n\n

The config file should be a .toml file. A convenient way to create a config file is to get Geth to create one for you and use it as a template. To do this, use the dumpconfig command, saving the result to a .toml file. Note that you also need to explicitly provide the network_id on the command line for the public testnets such as Sepolia or Geoerli:

\n
1
./geth --sepolia dumpconfig > geth-config.toml
\n

to specify path to config file

\n
1
geth --sepolia --config geth-config.toml
\n\n

key configs

    \n
  • [Eth].TxLookupLimit
    Number of recent blocks to maintain transactions index for (default = about one year, 0 = entire chain), default: 2350000
    • \n
  • [Node].BootstrapNodes
    used to establish connectivity with the rest of the network.
    geth provides default bootstrapNodes in file params/bootnodes.go
    • \n
  • [Metrics_AND_STATS].ethstats
    Reporting URL of a ethstats service (nodename:secret@host:port), more detail
    • \n
  • SyncMode
    • \n
  • TrieDirtyCache
    • \n
  • NoPruning
    • \n
  • TrieCleanCacheJournal e.g triecache
    • \n
\n

how geth starts

\"geth
the main func is in cmd/geth/main.go

\n
1
2
3
4
5
6
func main() {
\tif err := app.Run(os.Args); err != nil {
\t\tfmt.Fprintln(os.Stderr, err)
\t\tos.Exit(1)
\t}
}
\n

the main() function is very short, and its main function is to start a tool for parsing command line commands: gopkg.in/urfave/cli.v1. Going deeper, we will find that app.Action = geth will be called when the cli app is initialized to call the geth() function

\n
1
2
3
4
5
func init() {
\t// Initialize the CLI app and start Geth
\tapp.Action = geth
    // ....
}
\n

geth is the main entry point into the system if no special subcommand is run. It creates a default node based on the command line arguments and runs it in blocking mode, waiting for it to be shut down.

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
func geth(ctx *cli.Context) error {
\tif args := ctx.Args().Slice(); len(args) > 0 {
\t\treturn fmt.Errorf("invalid command: %q", args[0])
\t}

\tprepare(ctx)
\tstack, backend := makeFullNode(ctx)
\tdefer stack.Close()

\tstartNode(ctx, stack, backend, false)
\tstack.Wait()
\treturn nil
}
\n

In the geth() function, there are three important function calls, namely: prepare(), makeFullNode(), and startNode().

\n

prepare

The implementation of the prepare() function is in the current main.go file. It is mainly used to set some configurations required for node initialization.

\n

makeFullNode

The implementation of the makeFullNode() function is located in the cmd/geth/config.go file. It will load the context of the command and apply user given configuration; and generate instances of stack and backend. Among them, stack is an instance of Node type (Node is the top-level instance in the life cycle of geth. It is responsible for managing high-level abstractions such as P2P Server, Http Server, and Database in the node. The definition of the Node type is located in the node/node.go file), which is initialized by calling makeConfigNode() function through makeFullNode() function. inside makeFullNode, it calls node.New(&cfg.Node) to initiate a node. During instantiating of node, it invokes rpc.NewServer() to create a new rpc server and put in the field inprocHandler. it registers rpc api namespace by default.

\n

The backend here is an interface of ethapi.Backend type, which provides the basic functions needed to obtain the runtime of the Ethereum execution layer. Its definition is located in internal/ethapi/backend.go. Since there are many functions in this interface, we have selected some of the key functions as below for a glimpse of its functionality. backend is created by calling backend, eth := utils.RegisterEthService(stack, &cfg.Eth). Inside, it calls eth.New(stack, cfg) to create backend instance. During backend initiating, it opens database (chainDb, err := stack.OpenDatabaseWithFreezer("chaindata", config.DatabaseCache, config.DatabaseHandles, config.DatabaseFreezer, "eth/db/chaindata/", false)). Further, it creates consensus engine, engine := ethconfig.CreateConsensusEngine(stack, &ethashConfig, cliqueConfig, config.Miner.Notify, config.Miner.Noverify, chainDb). goerli testnet use POA consensus (clique).

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
type Backend interface {
\tSyncProgress() ethereum.SyncProgress
\tSuggestGasTipCap(ctx context.Context) (*big.Int, error)
\tChainDb() ethdb.Database
\tAccountManager() *accounts.Manager
\tExtRPCEnabled() bool
\tRPCGasCap() uint64            // global gas cap for eth_call over rpc: DoS protection
\tRPCEVMTimeout() time.Duration // global timeout for eth_call over rpc: DoS protection
\tRPCTxFeeCap() float64         // global tx fee cap for all transaction related APIs
\tUnprotectedAllowed() bool     // allows only for EIP155 transactions.
\tSetHead(number uint64)
\tHeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Header, error)
\tHeaderByHash(ctx context.Context, hash common.Hash) (*types.Header, error)
\tHeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*types.Header, error)
\tCurrentHeader() *types.Header
\tCurrentBlock() *types.Header
\tBlockByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Block, error)
\tBlockByHash(ctx context.Context, hash common.Hash) (*types.Block, error)
\tBlockByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*types.Block, error)
\tStateAndHeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*state.StateDB, *types.Header, error)
\tStateAndHeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*state.StateDB, *types.Header, error)
\tPendingBlockAndReceipts() (*types.Block, types.Receipts)
\tGetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error)
\tGetTd(ctx context.Context, hash common.Hash) *big.Int
\tGetEVM(ctx context.Context, msg *core.Message, state *state.StateDB, header *types.Header, vmConfig *vm.Config) (*vm.EVM, func() error, error)
\tSubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription
\tSubscribeChainHeadEvent(ch chan<- core.ChainHeadEvent) event.Subscription
\tSubscribeChainSideEvent(ch chan<- core.ChainSideEvent) event.Subscription
\tSendTx(ctx context.Context, signedTx *types.Transaction) error
\tGetTransaction(ctx context.Context, txHash common.Hash) (*types.Transaction, common.Hash, uint64, uint64, error)
\tGetPoolTransactions() (types.Transactions, error)
\tGetPoolTransaction(txHash common.Hash) *types.Transaction
\tGetPoolNonce(ctx context.Context, addr common.Address) (uint64, error)
\tStats() (pending int, queued int)
\tTxPoolContent() (map[common.Address]types.Transactions, map[common.Address]types.Transactions)
\tTxPoolContentFrom(addr common.Address) (types.Transactions, types.Transactions)
\tSubscribeNewTxsEvent(chan<- core.NewTxsEvent) event.Subscription
\tChainConfig() *params.ChainConfig
\tEngine() consensus.Engine
\tGetBody(ctx context.Context, hash common.Hash, number rpc.BlockNumber) (*types.Body, error)
\tGetLogs(ctx context.Context, blockHash common.Hash, number uint64) ([][]*types.Log, error)
\tSubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription
\tSubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription
\tSubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription
\tBloomStatus() (uint64, uint64)
\tServiceFilter(ctx context.Context, session *bloombits.MatcherSession)
}
\n\n

If readers want to customize some new RPC APIs, they can define functions in the /internal/ethapi.Backend interface and add specific implementations to EthAPIBackend

\n

startNode

The last key function, startNode(), is to officially start an Ethereum execution layer node. It starts the Stack instance (Node) by calling the utils.StartNode() function which triggers the Node.Start() function. In the Node.Start() function, it traverses the backend instances registered in Node.lifecycles and starts them. In addition, in the startNode() function, the unlockAccounts() function is still called, and the unlocked wallet is registered in the stack, and the RPClient module that interacts with local Geth is created through the stack.Attach() function

\n

At the end of the geth() function, the function executes stack.Wait(), so that the main thread enters the blocking state, and the services of other functional modules are distributed to other sub-coroutines for maintenance

\n

Node

As we mentioned earlier, the Node type belongs to the top-level instance in the life cycle of geth, and it is responsible for being the administrator of the high-level abstract module communicating with the outside world, such as managing rpc server, http server, Web Socket, and P2P Server external interface . At the same time, Node maintains the back-end instances and services (lifecycles []Lifecycle) required for node operation, such as the Ethereum type we mentioned above that is responsible for the specific Service

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
type Node struct {
\teventmux      *event.TypeMux
\tconfig        *Config
\taccman        *accounts.Manager
\tlog           log.Logger
\tkeyDir        string        // key store directory
\tkeyDirTemp    bool          // If true, key directory will be removed by Stop
\tdirLock       *flock.Flock  // prevents concurrent use of instance directory
\tstop          chan struct{} // Channel to wait for termination notifications
\tserver        *p2p.Server   // Currently running P2P networking layer
\tstartStopLock sync.Mutex    // Start/Stop are protected by an additional lock
\tstate         int           // Tracks state of node lifecycle

\tlock          sync.Mutex
\tlifecycles    []Lifecycle // All registered backends, services, and auxiliary services that have a lifecycle
\trpcAPIs       []rpc.API   // List of APIs currently provided by the node
\thttp          *httpServer //
\tws            *httpServer //
\thttpAuth      *httpServer //
\twsAuth        *httpServer //
\tipc           *ipcServer  // Stores information about the ipc http server
\tinprocHandler *rpc.Server // In-process RPC request handler to process the API requests

\tdatabases map[*closeTrackingDB]struct{} // All open databases
}
\n\n

close node

As mentioned earlier, the main thread of the entire program is blocked because of calling stack.Wait(). We can see that a channel called stop is declared in the Node structure. Since this Channel has not been assigned a value, the main process of the entire geth enters the blocking state, and continues to execute other business coroutines concurrently

\n
1
2
3
4
// Wait blocks until the node is closed.
func (n *Node) Wait() {
 <-n.stop
}
\n

When the Channel n.stop is assigned a value, the geth main function will stop the current blocking state and start to perform a series of corresponding resource release operations.
It is worth noting that in the current codebase of go-ethereum, the blocking state of the main process is not ended directly by assigning a value to the stop channel, but a more concise and rude way is used: call the close() function directly Close the Channel. We can find the related implementation in node.doClose(). close() is a native function of go language, used when closing Channel.

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
// doClose releases resources acquired by New(), collecting errors.
func (n *Node) doClose(errs []error) error {
 // Close databases. This needs the lock because it needs to
 // synchronize with OpenDatabase*.
 n.lock.Lock()
 n.state = closedState
 errs = append(errs, n.closeDatabases()...)
 n.lock.Unlock()

 if err := n.accman.Close(); err != nil {
  errs = append(errs, err)
 }
 if n.keyDirTemp {
  if err := os.RemoveAll(n.keyDir); err != nil {
   errs = append(errs, err)
  }
 }

 // Release instance directory lock.
 n.closeDataDir()

 // Unblock n.Wait.
 close(n.stop)

 // Report any errors that might have occurred.
 switch len(errs) {
 case 0:
  return nil
 case 1:
  return errs[0]
 default:
  return fmt.Errorf("%v", errs)
 }
}
\n\n

Ethereum Backend

We can find the definition of the Ethereum structure in eth/backend.go. The member variables and receiving methods contained in this structure implement all the functions and data structures required by an Ethereum full node. We can see in the following code definition that the Ethereum structure contains several core data structures such as TxPool, Blockchain, consensus.Engine, and miner as member variables.

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
type Ethereum struct {
\tconfig *ethconfig.Config

\t// Handlers
\ttxPool             *txpool.TxPool
\tblockchain         *core.BlockChain
\thandler            *handler
\tethDialCandidates  enode.Iterator
\tsnapDialCandidates enode.Iterator
\tmerger             *consensus.Merger

\t// DB interfaces
\tchainDb ethdb.Database // Block chain database

\teventMux       *event.TypeMux
\tengine         consensus.Engine
\taccountManager *accounts.Manager

\tbloomRequests     chan chan *bloombits.Retrieval // Channel receiving bloom data retrieval requests
\tbloomIndexer      *core.ChainIndexer             // Bloom indexer operating during block imports
\tcloseBloomHandler chan struct{}

\tAPIBackend *EthAPIBackend

\tminer     *miner.Miner
\tgasPrice  *big.Int
\tetherbase common.Address

\tnetworkID     uint64
\tnetRPCService *ethapi.NetAPI

\tp2pServer *p2p.Server

\tlock sync.RWMutex // Protects the variadic fields (e.g. gas price and etherbase)

\tshutdownTracker *shutdowncheck.ShutdownTracker // Tracks if and when the node has shutdown ungracefully
}
\n

Nodes start and stop Mining by calling Ethereum.StartMining() and Ethereum.StopMining(). Setting the profit account of Mining is achieved by calling Ethereum.SetEtherbase()
Here we pay extra attention to the member variable handler. The definition of handler is in eth/handler.go.
From a macro point of view, the main workflow of a node needs to: 1. Obtain/synchronize Transaction and Block data from the network 2. Add the Block obtained from the network to the Blockchain. The handler is responsible for providing the function of synchronizing blocks and transaction data, for example, downloader.Downloader is responsible for synchronizing Block from the network, and fetcher.TxFetcher is responsible for synchronizing transactions from the network

\n","site":{"data":{}},"excerpt":"","more":"

build from source

1
2
3
git clone https://github.com/ethereum/go-ethereum.git
cd go-ethereum
make geth
\n\n

understanding geth config

geth config type is defined in /cmd/geth/config.go

\n
1
2
3
4
5
6
type gethConfig struct {
\tEth      ethconfig.Config
\tNode     node.Config
\tEthstats ethstatsConfig
\tMetrics  metrics.Config
}
\n
    \n
  • ethconfig (eth/ethconfig/config.go)
    contains configuration options for of the ETH and LES(light node) protocols, such as NetworkId, SyncMode, txpool.Config, database options
    • \n
  • nodeConfig (node/config.go)
    represents a small collection of configuration values to fine tune the P2P network layer of a protocol stack. These values can be further extended by all registered services. such as p2p.Config, DataDir, KeyStoreDir, HTTPHost, HTTPModules(eth,net,web3), WSHost
    • \n
  • metrics.Config (metrics/config.go)
    contains the configuration for the metric collection, such as InfluxDBEndpoint, etc
    • \n
  • ethstatsConfig
    only one URL entry
    • \n
\n

geth provides default config in the above files. user config file path is given by the below flag

\n
1
2
3
4
5
configFileFlag = &cli.StringFlag{
\t\tName:     "config",
\t\tUsage:    "TOML configuration file",
\t\tCategory: flags.EthCategory,
\t}
\n\n

The config file should be a .toml file. A convenient way to create a config file is to get Geth to create one for you and use it as a template. To do this, use the dumpconfig command, saving the result to a .toml file. Note that you also need to explicitly provide the network_id on the command line for the public testnets such as Sepolia or Geoerli:

\n
1
./geth --sepolia dumpconfig > geth-config.toml
\n

to specify path to config file

\n
1
geth --sepolia --config geth-config.toml
\n\n

key configs

    \n
  • [Eth].TxLookupLimit
    Number of recent blocks to maintain transactions index for (default = about one year, 0 = entire chain), default: 2350000
    • \n
  • [Node].BootstrapNodes
    used to establish connectivity with the rest of the network.
    geth provides default bootstrapNodes in file params/bootnodes.go
    • \n
  • [Metrics_AND_STATS].ethstats
    Reporting URL of a ethstats service (nodename:secret@host:port), more detail
    • \n
  • SyncMode
    • \n
  • TrieDirtyCache
    • \n
  • NoPruning
    • \n
  • TrieCleanCacheJournal e.g triecache
    • \n
\n

how geth starts

\"geth
the main func is in cmd/geth/main.go

\n
1
2
3
4
5
6
func main() {
\tif err := app.Run(os.Args); err != nil {
\t\tfmt.Fprintln(os.Stderr, err)
\t\tos.Exit(1)
\t}
}
\n

the main() function is very short, and its main function is to start a tool for parsing command line commands: gopkg.in/urfave/cli.v1. Going deeper, we will find that app.Action = geth will be called when the cli app is initialized to call the geth() function

\n
1
2
3
4
5
func init() {
\t// Initialize the CLI app and start Geth
\tapp.Action = geth
    // ....
}
\n

geth is the main entry point into the system if no special subcommand is run. It creates a default node based on the command line arguments and runs it in blocking mode, waiting for it to be shut down.

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
func geth(ctx *cli.Context) error {
\tif args := ctx.Args().Slice(); len(args) > 0 {
\t\treturn fmt.Errorf("invalid command: %q", args[0])
\t}

\tprepare(ctx)
\tstack, backend := makeFullNode(ctx)
\tdefer stack.Close()

\tstartNode(ctx, stack, backend, false)
\tstack.Wait()
\treturn nil
}
\n

In the geth() function, there are three important function calls, namely: prepare(), makeFullNode(), and startNode().

\n

prepare

The implementation of the prepare() function is in the current main.go file. It is mainly used to set some configurations required for node initialization.

\n

makeFullNode

The implementation of the makeFullNode() function is located in the cmd/geth/config.go file. It will load the context of the command and apply user given configuration; and generate instances of stack and backend. Among them, stack is an instance of Node type (Node is the top-level instance in the life cycle of geth. It is responsible for managing high-level abstractions such as P2P Server, Http Server, and Database in the node. The definition of the Node type is located in the node/node.go file), which is initialized by calling makeConfigNode() function through makeFullNode() function. inside makeFullNode, it calls node.New(&cfg.Node) to initiate a node. During instantiating of node, it invokes rpc.NewServer() to create a new rpc server and put in the field inprocHandler. it registers rpc api namespace by default.

\n

The backend here is an interface of ethapi.Backend type, which provides the basic functions needed to obtain the runtime of the Ethereum execution layer. Its definition is located in internal/ethapi/backend.go. Since there are many functions in this interface, we have selected some of the key functions as below for a glimpse of its functionality. backend is created by calling backend, eth := utils.RegisterEthService(stack, &cfg.Eth). Inside, it calls eth.New(stack, cfg) to create backend instance. During backend initiating, it opens database (chainDb, err := stack.OpenDatabaseWithFreezer("chaindata", config.DatabaseCache, config.DatabaseHandles, config.DatabaseFreezer, "eth/db/chaindata/", false)). Further, it creates consensus engine, engine := ethconfig.CreateConsensusEngine(stack, &ethashConfig, cliqueConfig, config.Miner.Notify, config.Miner.Noverify, chainDb). goerli testnet use POA consensus (clique).

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
type Backend interface {
\tSyncProgress() ethereum.SyncProgress
\tSuggestGasTipCap(ctx context.Context) (*big.Int, error)
\tChainDb() ethdb.Database
\tAccountManager() *accounts.Manager
\tExtRPCEnabled() bool
\tRPCGasCap() uint64            // global gas cap for eth_call over rpc: DoS protection
\tRPCEVMTimeout() time.Duration // global timeout for eth_call over rpc: DoS protection
\tRPCTxFeeCap() float64         // global tx fee cap for all transaction related APIs
\tUnprotectedAllowed() bool     // allows only for EIP155 transactions.
\tSetHead(number uint64)
\tHeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Header, error)
\tHeaderByHash(ctx context.Context, hash common.Hash) (*types.Header, error)
\tHeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*types.Header, error)
\tCurrentHeader() *types.Header
\tCurrentBlock() *types.Header
\tBlockByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Block, error)
\tBlockByHash(ctx context.Context, hash common.Hash) (*types.Block, error)
\tBlockByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*types.Block, error)
\tStateAndHeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*state.StateDB, *types.Header, error)
\tStateAndHeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*state.StateDB, *types.Header, error)
\tPendingBlockAndReceipts() (*types.Block, types.Receipts)
\tGetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error)
\tGetTd(ctx context.Context, hash common.Hash) *big.Int
\tGetEVM(ctx context.Context, msg *core.Message, state *state.StateDB, header *types.Header, vmConfig *vm.Config) (*vm.EVM, func() error, error)
\tSubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription
\tSubscribeChainHeadEvent(ch chan<- core.ChainHeadEvent) event.Subscription
\tSubscribeChainSideEvent(ch chan<- core.ChainSideEvent) event.Subscription
\tSendTx(ctx context.Context, signedTx *types.Transaction) error
\tGetTransaction(ctx context.Context, txHash common.Hash) (*types.Transaction, common.Hash, uint64, uint64, error)
\tGetPoolTransactions() (types.Transactions, error)
\tGetPoolTransaction(txHash common.Hash) *types.Transaction
\tGetPoolNonce(ctx context.Context, addr common.Address) (uint64, error)
\tStats() (pending int, queued int)
\tTxPoolContent() (map[common.Address]types.Transactions, map[common.Address]types.Transactions)
\tTxPoolContentFrom(addr common.Address) (types.Transactions, types.Transactions)
\tSubscribeNewTxsEvent(chan<- core.NewTxsEvent) event.Subscription
\tChainConfig() *params.ChainConfig
\tEngine() consensus.Engine
\tGetBody(ctx context.Context, hash common.Hash, number rpc.BlockNumber) (*types.Body, error)
\tGetLogs(ctx context.Context, blockHash common.Hash, number uint64) ([][]*types.Log, error)
\tSubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription
\tSubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription
\tSubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription
\tBloomStatus() (uint64, uint64)
\tServiceFilter(ctx context.Context, session *bloombits.MatcherSession)
}
\n\n

If readers want to customize some new RPC APIs, they can define functions in the /internal/ethapi.Backend interface and add specific implementations to EthAPIBackend

\n

startNode

The last key function, startNode(), is to officially start an Ethereum execution layer node. It starts the Stack instance (Node) by calling the utils.StartNode() function which triggers the Node.Start() function. In the Node.Start() function, it traverses the backend instances registered in Node.lifecycles and starts them. In addition, in the startNode() function, the unlockAccounts() function is still called, and the unlocked wallet is registered in the stack, and the RPClient module that interacts with local Geth is created through the stack.Attach() function

\n

At the end of the geth() function, the function executes stack.Wait(), so that the main thread enters the blocking state, and the services of other functional modules are distributed to other sub-coroutines for maintenance

\n

Node

As we mentioned earlier, the Node type belongs to the top-level instance in the life cycle of geth, and it is responsible for being the administrator of the high-level abstract module communicating with the outside world, such as managing rpc server, http server, Web Socket, and P2P Server external interface . At the same time, Node maintains the back-end instances and services (lifecycles []Lifecycle) required for node operation, such as the Ethereum type we mentioned above that is responsible for the specific Service

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
type Node struct {
\teventmux      *event.TypeMux
\tconfig        *Config
\taccman        *accounts.Manager
\tlog           log.Logger
\tkeyDir        string        // key store directory
\tkeyDirTemp    bool          // If true, key directory will be removed by Stop
\tdirLock       *flock.Flock  // prevents concurrent use of instance directory
\tstop          chan struct{} // Channel to wait for termination notifications
\tserver        *p2p.Server   // Currently running P2P networking layer
\tstartStopLock sync.Mutex    // Start/Stop are protected by an additional lock
\tstate         int           // Tracks state of node lifecycle

\tlock          sync.Mutex
\tlifecycles    []Lifecycle // All registered backends, services, and auxiliary services that have a lifecycle
\trpcAPIs       []rpc.API   // List of APIs currently provided by the node
\thttp          *httpServer //
\tws            *httpServer //
\thttpAuth      *httpServer //
\twsAuth        *httpServer //
\tipc           *ipcServer  // Stores information about the ipc http server
\tinprocHandler *rpc.Server // In-process RPC request handler to process the API requests

\tdatabases map[*closeTrackingDB]struct{} // All open databases
}
\n\n

close node

As mentioned earlier, the main thread of the entire program is blocked because of calling stack.Wait(). We can see that a channel called stop is declared in the Node structure. Since this Channel has not been assigned a value, the main process of the entire geth enters the blocking state, and continues to execute other business coroutines concurrently

\n
1
2
3
4
// Wait blocks until the node is closed.
func (n *Node) Wait() {
 <-n.stop
}
\n

When the Channel n.stop is assigned a value, the geth main function will stop the current blocking state and start to perform a series of corresponding resource release operations.
It is worth noting that in the current codebase of go-ethereum, the blocking state of the main process is not ended directly by assigning a value to the stop channel, but a more concise and rude way is used: call the close() function directly Close the Channel. We can find the related implementation in node.doClose(). close() is a native function of go language, used when closing Channel.

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
// doClose releases resources acquired by New(), collecting errors.
func (n *Node) doClose(errs []error) error {
 // Close databases. This needs the lock because it needs to
 // synchronize with OpenDatabase*.
 n.lock.Lock()
 n.state = closedState
 errs = append(errs, n.closeDatabases()...)
 n.lock.Unlock()

 if err := n.accman.Close(); err != nil {
  errs = append(errs, err)
 }
 if n.keyDirTemp {
  if err := os.RemoveAll(n.keyDir); err != nil {
   errs = append(errs, err)
  }
 }

 // Release instance directory lock.
 n.closeDataDir()

 // Unblock n.Wait.
 close(n.stop)

 // Report any errors that might have occurred.
 switch len(errs) {
 case 0:
  return nil
 case 1:
  return errs[0]
 default:
  return fmt.Errorf("%v", errs)
 }
}
\n\n

Ethereum Backend

We can find the definition of the Ethereum structure in eth/backend.go. The member variables and receiving methods contained in this structure implement all the functions and data structures required by an Ethereum full node. We can see in the following code definition that the Ethereum structure contains several core data structures such as TxPool, Blockchain, consensus.Engine, and miner as member variables.

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
type Ethereum struct {
\tconfig *ethconfig.Config

\t// Handlers
\ttxPool             *txpool.TxPool
\tblockchain         *core.BlockChain
\thandler            *handler
\tethDialCandidates  enode.Iterator
\tsnapDialCandidates enode.Iterator
\tmerger             *consensus.Merger

\t// DB interfaces
\tchainDb ethdb.Database // Block chain database

\teventMux       *event.TypeMux
\tengine         consensus.Engine
\taccountManager *accounts.Manager

\tbloomRequests     chan chan *bloombits.Retrieval // Channel receiving bloom data retrieval requests
\tbloomIndexer      *core.ChainIndexer             // Bloom indexer operating during block imports
\tcloseBloomHandler chan struct{}

\tAPIBackend *EthAPIBackend

\tminer     *miner.Miner
\tgasPrice  *big.Int
\tetherbase common.Address

\tnetworkID     uint64
\tnetRPCService *ethapi.NetAPI

\tp2pServer *p2p.Server

\tlock sync.RWMutex // Protects the variadic fields (e.g. gas price and etherbase)

\tshutdownTracker *shutdowncheck.ShutdownTracker // Tracks if and when the node has shutdown ungracefully
}
\n

Nodes start and stop Mining by calling Ethereum.StartMining() and Ethereum.StopMining(). Setting the profit account of Mining is achieved by calling Ethereum.SetEtherbase()
Here we pay extra attention to the member variable handler. The definition of handler is in eth/handler.go.
From a macro point of view, the main workflow of a node needs to: 1. Obtain/synchronize Transaction and Block data from the network 2. Add the Block obtained from the network to the Blockchain. The handler is responsible for providing the function of synchronizing blocks and transaction data, for example, downloader.Downloader is responsible for synchronizing Block from the network, and fetcher.TxFetcher is responsible for synchronizing transactions from the network

\n"},{"title":"rpc","date":"2022-11-08T06:23:08.000Z","_content":"\n\n## overview\npackage rpc implements bi-directional JSON-RPC 2.0 on multiple transports (http, ws, ipc). After creating a server or client instance, objects can be registered to make them visible as 'services'. Exported methods that follow specific conventions can be called remotely. It also has support for the publish/subscribe pattern.\n\n## methods\n### rpc endpoints (callback)\nMethods that satisfy the following criteria are made available for remote access:\n - method must be exported\n - method returns 0, 1 (response or error) or 2 (response and error) values\n\nThe server offers the ServeCodec method which accepts a ServerCodec instance. It will read requests from the codec, process the request and sends the response back to the client using the codec. The server can execute requests concurrently. Responses can be sent back to the client out of order.\n\nAn example server which uses the JSON codec:\n```go\ntype CalculatorService struct {}\n\nfunc (s *CalculatorService) Add(a, b int) int {\n return a + b\n}\n\nfunc (s *CalculatorService) Div(a, b int) (int, error) {\n if b == 0 {\n return 0, errors.New(\"divide by zero\")\n }\n return a/b, nil\n}\n\ncalculator := new(CalculatorService)\nserver := NewServer()\nserver.RegisterName(\"calculator\", calculator)\nl, _ := net.ListenUnix(\"unix\", &net.UnixAddr{Net: \"unix\", Name: \"/tmp/calculator.sock\"})\nserver.ServeListener(l)\n```\n\n### subscriptions\nThe package also supports the publish subscribe pattern through the use of subscriptions.\nA method that is considered eligible for notifications must satisfy the following\ncriteria:\n - method must be exported\n - first method argument type must be context.Context\n - method must have return types (rpc.Subscription, error)\n\nAn example method:\n```go\nfunc (s *BlockChainService) NewBlocks(ctx context.Context) (rpc.Subscription, error) {\n\t\t...\n\t}\n```\n\n### Reverse Calls\nIn any method handler, an instance of rpc.Client can be accessed through the `ClientFromContext` method. Using this client instance, server-to-client method calls can be performed on the RPC connection.\n\n## server\nto start rpc service, the invoking chain is as below\n```\nnode/node.go[func (n *Node) Start()] -> node/node.go[func (n *Node) openEndpoints()] -> node/node.go[func (n *Node) startRPC()]\n```\n\n### API registration\n","source":"_posts/geth/code_analysis/geth.1.rpc.md","raw":"---\ntitle: rpc\ndate: 2022-11-08 14:23:08\ntags: [blockchain, geth]\n---\n\n\n## overview\npackage rpc implements bi-directional JSON-RPC 2.0 on multiple transports (http, ws, ipc). After creating a server or client instance, objects can be registered to make them visible as 'services'. Exported methods that follow specific conventions can be called remotely. It also has support for the publish/subscribe pattern.\n\n## methods\n### rpc endpoints (callback)\nMethods that satisfy the following criteria are made available for remote access:\n - method must be exported\n - method returns 0, 1 (response or error) or 2 (response and error) values\n\nThe server offers the ServeCodec method which accepts a ServerCodec instance. It will read requests from the codec, process the request and sends the response back to the client using the codec. The server can execute requests concurrently. Responses can be sent back to the client out of order.\n\nAn example server which uses the JSON codec:\n```go\ntype CalculatorService struct {}\n\nfunc (s *CalculatorService) Add(a, b int) int {\n return a + b\n}\n\nfunc (s *CalculatorService) Div(a, b int) (int, error) {\n if b == 0 {\n return 0, errors.New(\"divide by zero\")\n }\n return a/b, nil\n}\n\ncalculator := new(CalculatorService)\nserver := NewServer()\nserver.RegisterName(\"calculator\", calculator)\nl, _ := net.ListenUnix(\"unix\", &net.UnixAddr{Net: \"unix\", Name: \"/tmp/calculator.sock\"})\nserver.ServeListener(l)\n```\n\n### subscriptions\nThe package also supports the publish subscribe pattern through the use of subscriptions.\nA method that is considered eligible for notifications must satisfy the following\ncriteria:\n - method must be exported\n - first method argument type must be context.Context\n - method must have return types (rpc.Subscription, error)\n\nAn example method:\n```go\nfunc (s *BlockChainService) NewBlocks(ctx context.Context) (rpc.Subscription, error) {\n\t\t...\n\t}\n```\n\n### Reverse Calls\nIn any method handler, an instance of rpc.Client can be accessed through the `ClientFromContext` method. Using this client instance, server-to-client method calls can be performed on the RPC connection.\n\n## server\nto start rpc service, the invoking chain is as below\n```\nnode/node.go[func (n *Node) Start()] -> node/node.go[func (n *Node) openEndpoints()] -> node/node.go[func (n *Node) startRPC()]\n```\n\n### API registration\n","slug":"geth/code_analysis/geth.1.rpc","published":1,"updated":"2023-11-05T04:21:13.733Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8dv0025qwsjg5vzbwcu","content":"

overview

package rpc implements bi-directional JSON-RPC 2.0 on multiple transports (http, ws, ipc). After creating a server or client instance, objects can be registered to make them visible as ‘services’. Exported methods that follow specific conventions can be called remotely. It also has support for the publish/subscribe pattern.

\n

methods

rpc endpoints (callback)

Methods that satisfy the following criteria are made available for remote access:

\n
    \n
  • method must be exported
    • \n
  • method returns 0, 1 (response or error) or 2 (response and error) values
    • \n
\n

The server offers the ServeCodec method which accepts a ServerCodec instance. It will read requests from the codec, process the request and sends the response back to the client using the codec. The server can execute requests concurrently. Responses can be sent back to the client out of order.

\n

An example server which uses the JSON codec:

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
type CalculatorService struct {}

func (s *CalculatorService) Add(a, b int) int {
    return a + b
}

func (s *CalculatorService) Div(a, b int) (int, error) {
    if b == 0 {
        return 0, errors.New("divide by zero")
    }
    return a/b, nil
}

calculator := new(CalculatorService)
server := NewServer()
server.RegisterName("calculator", calculator)
l, _ := net.ListenUnix("unix", &net.UnixAddr{Net: "unix", Name: "/tmp/calculator.sock"})
server.ServeListener(l)
\n\n

subscriptions

The package also supports the publish subscribe pattern through the use of subscriptions.
A method that is considered eligible for notifications must satisfy the following
criteria:

\n
    \n
  • method must be exported
    • \n
  • first method argument type must be context.Context
    • \n
  • method must have return types (rpc.Subscription, error)
    • \n
\n

An example method:

\n
1
2
3
func (s *BlockChainService) NewBlocks(ctx context.Context) (rpc.Subscription, error) {
\t\t...
\t}
\n\n

Reverse Calls

In any method handler, an instance of rpc.Client can be accessed through the ClientFromContext method. Using this client instance, server-to-client method calls can be performed on the RPC connection.

\n

server

to start rpc service, the invoking chain is as below

\n
1
node/node.go[func (n *Node) Start()] -> node/node.go[func (n *Node) openEndpoints()] -> node/node.go[func (n *Node) startRPC()]
\n\n

API registration

","site":{"data":{}},"excerpt":"","more":"

overview

package rpc implements bi-directional JSON-RPC 2.0 on multiple transports (http, ws, ipc). After creating a server or client instance, objects can be registered to make them visible as ‘services’. Exported methods that follow specific conventions can be called remotely. It also has support for the publish/subscribe pattern.

\n

methods

rpc endpoints (callback)

Methods that satisfy the following criteria are made available for remote access:

\n
    \n
  • method must be exported
    • \n
  • method returns 0, 1 (response or error) or 2 (response and error) values
    • \n
\n

The server offers the ServeCodec method which accepts a ServerCodec instance. It will read requests from the codec, process the request and sends the response back to the client using the codec. The server can execute requests concurrently. Responses can be sent back to the client out of order.

\n

An example server which uses the JSON codec:

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
type CalculatorService struct {}

func (s *CalculatorService) Add(a, b int) int {
    return a + b
}

func (s *CalculatorService) Div(a, b int) (int, error) {
    if b == 0 {
        return 0, errors.New("divide by zero")
    }
    return a/b, nil
}

calculator := new(CalculatorService)
server := NewServer()
server.RegisterName("calculator", calculator)
l, _ := net.ListenUnix("unix", &net.UnixAddr{Net: "unix", Name: "/tmp/calculator.sock"})
server.ServeListener(l)
\n\n

subscriptions

The package also supports the publish subscribe pattern through the use of subscriptions.
A method that is considered eligible for notifications must satisfy the following
criteria:

\n
    \n
  • method must be exported
    • \n
  • first method argument type must be context.Context
    • \n
  • method must have return types (rpc.Subscription, error)
    • \n
\n

An example method:

\n
1
2
3
func (s *BlockChainService) NewBlocks(ctx context.Context) (rpc.Subscription, error) {
\t\t...
\t}
\n\n

Reverse Calls

In any method handler, an instance of rpc.Client can be accessed through the ClientFromContext method. Using this client instance, server-to-client method calls can be performed on the RPC connection.

\n

server

to start rpc service, the invoking chain is as below

\n
1
node/node.go[func (n *Node) Start()] -> node/node.go[func (n *Node) openEndpoints()] -> node/node.go[func (n *Node) startRPC()]
\n\n

API registration

"},{"title":"geth evm source analysis","date":"2023-01-08T08:24:54.000Z","_content":"\n# overall\nthe code is under path `core/vm`\noverview of the whole evm module ![evm](/images/evm.drawio.google.png)\n\nthe core is `EVM` struct (in evm.go), with main function in creating or call contract. a new `EVM` object is created every time when processing a transaction. inside the EVM struct, the main items are `Interpreter`, and `StateDB` (for state persistence). `Interpreter` loops through contract call instructions.Before each instruction is executed, some checks are performed to ensure sufficient gas and stack space. actual instruction execution code is recorded in `JumpTable` (256 sized array of `operation`)\n\ndepending on the version of Ethereum, JumpTable may point to four different instruction sets: constantinopleInstructionSet, byzantiumInstructionSet, homesteadInstructionSet, frontierInstructionSet. Most of the instructions of these four sets of instruction sets are the same, but as the version is updated, the new version supports more instruction sets than the old version.\n\n# evm\nThe `EVM` object is the most important object exported by the evm module, which represents an Ethereum virtual machine\n\n## creating evm\nEvery time a transaction is processed, an EVM is created to execute the transaction. This is reflected in the function `ApplyTransaction` (core/state_processor.go)\n\n## creating contract\nIf the `to` of the transaction is empty, it means that this transaction is to create a contract, so call `EVM.Create` to perform related functions\n- CREATE\n```\ncontractAddr = crypto.CreateAddress(caller.Address(), evm.StateDB.GetNonce(caller.Address()))\n```\n- CREATE2\n```\ncodeAndHash := &codeAndHash{code: code}\n\tcontractAddr = crypto.CreateAddress2(caller.Address(), salt.Bytes32(), codeAndHash.Hash().Bytes())\n```\nduring create contract, an object `Contract` is created. A Contract object contains and maintains the necessary information during the execution of the contract, such as the contract creator, the address of the contract itself, the remaining gas of the contract, the contract code and the `jumpdests` record of the code.\n\nthen, it invokes below method to create contract\n```\nret, err := evm.interpreter.Run(contract, nil, false)\nevm.StateDB.SetCode(address, ret)\n```\nIf the operation is successful and the contract code does not exceed the length limit, call StateDB.SetCode to store the contract code in the contract account of the Ethereum state database. Of course, the storage needs to consume a certain amount of gas.\n\nYou may wonder why the stored contract code is the return code after the contract runs, not the data in the original transaction (ie Transaction.data.Payload). This is because when the contract source code is compiled into binary data, in addition to the original code of the contract, the compiler also inserts some codes to perform related functions. For creation, the compiler inserts code that executes the contract's \"constructor\" (that is, the contract object's constructor method). Therefore, when the binary compiled by the compiler is submitted to the Ethereum node to create a contract, the EVM executes this binary code, in fact, it mainly executes the constructor method of the contract, and then returns other codes of the contract, so there is a `ret` variable here Stored in the state database as the actual code of the contract\n\n## call contract\nThe EVM object has three methods to implement the call of the contract, they are:\n\n- EVM. Call\n- EVM. CallCode\n- EVM. DelegateCall\n- EVM.StaticCall\nThe basic contract call function implemented by EVM.Call is nothing special. The following three calling methods are the differences compared with EVM.Call. So here we only introduce the particularity of the last three calling methods\n\n### EVM.CallCode & EVM.DelegateCall\nThe existence of EVM.CallCode and EVM.DelegateCall is to realize the characteristics of the \"library\" of the contract. If the code written by solidity is to be called as a library, it must be deployed on the blockchain to obtain a fixed address like a normal contract. , other contracts can call the method provided by this \"library contract\". But the contract also involves some unique attributes, such as the caller of the contract, contract address, the amount of ether it owns, etc. If we directly call the code of the \"library contract\", these properties must be the properties of the \"library contract\" itself, but this may not be what we want\n\nas an example\n```\nA -> contractB - delegateCall -> libC\n```\n`EVM.DelegateCall` sets the caller (msg.sender) of the \"library contract\" (libC) to A, rather than contractB; sets the address of the \"library contract\" (libC) to contractB. \n`EVM.CallCode` is similar to `EVM.DelegateCall`. the only difference is that `EVM.CallCode` only change the address of the \"library contract\" (libC) to contractB, without chanding the caller to A.\n`EVM.StaticCall` is similar to `EVM.Call`, the only difference is that EVM.StaticCall does not allow execution of instructions that modify permanently stored data\n\nduring contract call, it first check whether it is precompiled contract. some precompiled contracts are\n- common.BytesToAddress([]byte{1}): &ecrecover{},\n- common.BytesToAddress([]byte{2}): &sha256hash{},\n- common.BytesToAddress([]byte{3}): &ripemd160hash{},\n- common.BytesToAddress([]byte{4}): &dataCopy{},\n\n# EVMInterpreter\nThe interpreter object EVMInterpreter is used to interpret and execute specified contract instructions. However, note that the actual instruction interpretation and execution is not really completed by the interpreter object, but by the operation object JumpTable. The interpreter object is only responsible for parsing instruction codes one by one, and then obtains the corresponding operation object, and check objects such as the stack before calling the operation.execute function that actually executre the instruction. It can also be said that the interpreter object is only responsible for the scheduling of interpretation.\n\n## execution layout\n![layout](/images/evm.layout.png)\n\n## intrinsic gas\nThe intrinsic gas for a transaction is the amount of gas that the transaction uses before any code runs. It is a constant transaction fee (currently 21000 gas) plus a fee for every byte of data supplied with the transaction (4 gas for a zero byte, 68 gas for non-zeros). These constants are all currently defined for geth in params/protocol_params.go.\n\n## gas cost\nthe gas cost of each instruction is stored in `JumpTable.operation.dynamicGas` or `JumpTable.operation.constantGas`. constantGas means the operation gas cost is a fixed constant. dynamicGas is a function which will return gas during runtime.\n\nIn fact, not only the interpretation and execution of the instruction itself consumes gas, but also consumes gas when using memory storage and StateDB permanent storage. For most instructions, the latter two are not used (memory & storage), but for some instructions (such as CODECOPY or SSTORE), their gasCost function will take memory and StateDB usage into account.\n\na method `memoryGasCost`is used to calculate the gas consumption of memory usage. only when the required space size exceeds the current space size, the excess part needs to consume gas.\n\n# JumpTable\njumptable is 256 sized array of `operation`\n\n## jump instruction\nAmong the instructions of the contract, there are two jump instructions (excluding CALL): JUMP and JUMPI. Their special feature is that the first instruction of the target address after the jump must be JUMPDEST\n```\nfunc opJump(pc *uint64, interpreter *EVMInterpreter, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) {\n pos := stack.pop()\n if !contract.validJumpdest(pos) {\n nop := contract.GetOp(pos.Uint64())\n return nil, fmt.Errorf(\"invalid jump destination (%v) %v\", nop, pos)\n }\n *pc = pos.Uint64()\n\n interpreter.intPool.put(pos)\n return nil, nil\n}\n```\nA function interprets and executes the JUMP instruction. The code first fetches a value from the stack as the jump destination. This value is actually an offset relative to field 0 of the contract code. Then the code will call Contract.validJumpdest to determine whether the first instruction of this destination is JUMPDEST, if it is not, an error will occur.\n\nTo judge whether the first instruction of the destination is JUMPDEST, two points must be guaranteed: first, its value is the value of the opcode of the JUMPDEST instruction; second, it is an instruction, not ordinary data.\n\nLet's introduce how Contract.validJumpdest works. In addition to comparing opcode (this is very simple), Contract will also create a bit vector object (ie bitvec, bit vector). This object will analyze the contract instructions from the beginning to the end. If the byte at a certain offset of the contract belongs to ordinary data, the \"bit\" corresponding to the offset value in bitvec is set to 1, and if it is an instruction, it is set to 0. In Contract.validJumpdest, it is judged whether this is a normal instruction by checking whether the \"bit\" of the offset value of the jump destination in this bit vector object is 0\n\n# references\n- [yangzhe_blog](https://yangzhe.me/2019/08/12/ethereum-evm/#%E8%A7%A3%E9%87%8A%E5%99%A8%E5%AF%B9%E8%B1%A1evminterpreter)\n- [op code manual](https://www.evm.codes/?fork=shanghai)","source":"_posts/geth/code_analysis/geth.evm.md","raw":"---\ntitle: geth evm source analysis\ndate: 2023-01-08 16:24:54\ntags: [blockchain,geth]\n---\n\n# overall\nthe code is under path `core/vm`\noverview of the whole evm module ![evm](/images/evm.drawio.google.png)\n\nthe core is `EVM` struct (in evm.go), with main function in creating or call contract. a new `EVM` object is created every time when processing a transaction. inside the EVM struct, the main items are `Interpreter`, and `StateDB` (for state persistence). `Interpreter` loops through contract call instructions.Before each instruction is executed, some checks are performed to ensure sufficient gas and stack space. actual instruction execution code is recorded in `JumpTable` (256 sized array of `operation`)\n\ndepending on the version of Ethereum, JumpTable may point to four different instruction sets: constantinopleInstructionSet, byzantiumInstructionSet, homesteadInstructionSet, frontierInstructionSet. Most of the instructions of these four sets of instruction sets are the same, but as the version is updated, the new version supports more instruction sets than the old version.\n\n# evm\nThe `EVM` object is the most important object exported by the evm module, which represents an Ethereum virtual machine\n\n## creating evm\nEvery time a transaction is processed, an EVM is created to execute the transaction. This is reflected in the function `ApplyTransaction` (core/state_processor.go)\n\n## creating contract\nIf the `to` of the transaction is empty, it means that this transaction is to create a contract, so call `EVM.Create` to perform related functions\n- CREATE\n```\ncontractAddr = crypto.CreateAddress(caller.Address(), evm.StateDB.GetNonce(caller.Address()))\n```\n- CREATE2\n```\ncodeAndHash := &codeAndHash{code: code}\n\tcontractAddr = crypto.CreateAddress2(caller.Address(), salt.Bytes32(), codeAndHash.Hash().Bytes())\n```\nduring create contract, an object `Contract` is created. A Contract object contains and maintains the necessary information during the execution of the contract, such as the contract creator, the address of the contract itself, the remaining gas of the contract, the contract code and the `jumpdests` record of the code.\n\nthen, it invokes below method to create contract\n```\nret, err := evm.interpreter.Run(contract, nil, false)\nevm.StateDB.SetCode(address, ret)\n```\nIf the operation is successful and the contract code does not exceed the length limit, call StateDB.SetCode to store the contract code in the contract account of the Ethereum state database. Of course, the storage needs to consume a certain amount of gas.\n\nYou may wonder why the stored contract code is the return code after the contract runs, not the data in the original transaction (ie Transaction.data.Payload). This is because when the contract source code is compiled into binary data, in addition to the original code of the contract, the compiler also inserts some codes to perform related functions. For creation, the compiler inserts code that executes the contract's \"constructor\" (that is, the contract object's constructor method). Therefore, when the binary compiled by the compiler is submitted to the Ethereum node to create a contract, the EVM executes this binary code, in fact, it mainly executes the constructor method of the contract, and then returns other codes of the contract, so there is a `ret` variable here Stored in the state database as the actual code of the contract\n\n## call contract\nThe EVM object has three methods to implement the call of the contract, they are:\n\n- EVM. Call\n- EVM. CallCode\n- EVM. DelegateCall\n- EVM.StaticCall\nThe basic contract call function implemented by EVM.Call is nothing special. The following three calling methods are the differences compared with EVM.Call. So here we only introduce the particularity of the last three calling methods\n\n### EVM.CallCode & EVM.DelegateCall\nThe existence of EVM.CallCode and EVM.DelegateCall is to realize the characteristics of the \"library\" of the contract. If the code written by solidity is to be called as a library, it must be deployed on the blockchain to obtain a fixed address like a normal contract. , other contracts can call the method provided by this \"library contract\". But the contract also involves some unique attributes, such as the caller of the contract, contract address, the amount of ether it owns, etc. If we directly call the code of the \"library contract\", these properties must be the properties of the \"library contract\" itself, but this may not be what we want\n\nas an example\n```\nA -> contractB - delegateCall -> libC\n```\n`EVM.DelegateCall` sets the caller (msg.sender) of the \"library contract\" (libC) to A, rather than contractB; sets the address of the \"library contract\" (libC) to contractB. \n`EVM.CallCode` is similar to `EVM.DelegateCall`. the only difference is that `EVM.CallCode` only change the address of the \"library contract\" (libC) to contractB, without chanding the caller to A.\n`EVM.StaticCall` is similar to `EVM.Call`, the only difference is that EVM.StaticCall does not allow execution of instructions that modify permanently stored data\n\nduring contract call, it first check whether it is precompiled contract. some precompiled contracts are\n- common.BytesToAddress([]byte{1}): &ecrecover{},\n- common.BytesToAddress([]byte{2}): &sha256hash{},\n- common.BytesToAddress([]byte{3}): &ripemd160hash{},\n- common.BytesToAddress([]byte{4}): &dataCopy{},\n\n# EVMInterpreter\nThe interpreter object EVMInterpreter is used to interpret and execute specified contract instructions. However, note that the actual instruction interpretation and execution is not really completed by the interpreter object, but by the operation object JumpTable. The interpreter object is only responsible for parsing instruction codes one by one, and then obtains the corresponding operation object, and check objects such as the stack before calling the operation.execute function that actually executre the instruction. It can also be said that the interpreter object is only responsible for the scheduling of interpretation.\n\n## execution layout\n![layout](/images/evm.layout.png)\n\n## intrinsic gas\nThe intrinsic gas for a transaction is the amount of gas that the transaction uses before any code runs. It is a constant transaction fee (currently 21000 gas) plus a fee for every byte of data supplied with the transaction (4 gas for a zero byte, 68 gas for non-zeros). These constants are all currently defined for geth in params/protocol_params.go.\n\n## gas cost\nthe gas cost of each instruction is stored in `JumpTable.operation.dynamicGas` or `JumpTable.operation.constantGas`. constantGas means the operation gas cost is a fixed constant. dynamicGas is a function which will return gas during runtime.\n\nIn fact, not only the interpretation and execution of the instruction itself consumes gas, but also consumes gas when using memory storage and StateDB permanent storage. For most instructions, the latter two are not used (memory & storage), but for some instructions (such as CODECOPY or SSTORE), their gasCost function will take memory and StateDB usage into account.\n\na method `memoryGasCost`is used to calculate the gas consumption of memory usage. only when the required space size exceeds the current space size, the excess part needs to consume gas.\n\n# JumpTable\njumptable is 256 sized array of `operation`\n\n## jump instruction\nAmong the instructions of the contract, there are two jump instructions (excluding CALL): JUMP and JUMPI. Their special feature is that the first instruction of the target address after the jump must be JUMPDEST\n```\nfunc opJump(pc *uint64, interpreter *EVMInterpreter, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) {\n pos := stack.pop()\n if !contract.validJumpdest(pos) {\n nop := contract.GetOp(pos.Uint64())\n return nil, fmt.Errorf(\"invalid jump destination (%v) %v\", nop, pos)\n }\n *pc = pos.Uint64()\n\n interpreter.intPool.put(pos)\n return nil, nil\n}\n```\nA function interprets and executes the JUMP instruction. The code first fetches a value from the stack as the jump destination. This value is actually an offset relative to field 0 of the contract code. Then the code will call Contract.validJumpdest to determine whether the first instruction of this destination is JUMPDEST, if it is not, an error will occur.\n\nTo judge whether the first instruction of the destination is JUMPDEST, two points must be guaranteed: first, its value is the value of the opcode of the JUMPDEST instruction; second, it is an instruction, not ordinary data.\n\nLet's introduce how Contract.validJumpdest works. In addition to comparing opcode (this is very simple), Contract will also create a bit vector object (ie bitvec, bit vector). This object will analyze the contract instructions from the beginning to the end. If the byte at a certain offset of the contract belongs to ordinary data, the \"bit\" corresponding to the offset value in bitvec is set to 1, and if it is an instruction, it is set to 0. In Contract.validJumpdest, it is judged whether this is a normal instruction by checking whether the \"bit\" of the offset value of the jump destination in this bit vector object is 0\n\n# references\n- [yangzhe_blog](https://yangzhe.me/2019/08/12/ethereum-evm/#%E8%A7%A3%E9%87%8A%E5%99%A8%E5%AF%B9%E8%B1%A1evminterpreter)\n- [op code manual](https://www.evm.codes/?fork=shanghai)","slug":"geth/code_analysis/geth.evm","published":1,"updated":"2023-11-05T04:21:13.733Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8dw0028qwsj2vul70i7","content":"

overall

the code is under path core/vm
overview of the whole evm module \"evm\"

\n

the core is EVM struct (in evm.go), with main function in creating or call contract. a new EVM object is created every time when processing a transaction. inside the EVM struct, the main items are Interpreter, and StateDB (for state persistence). Interpreter loops through contract call instructions.Before each instruction is executed, some checks are performed to ensure sufficient gas and stack space. actual instruction execution code is recorded in JumpTable (256 sized array of operation)

\n

depending on the version of Ethereum, JumpTable may point to four different instruction sets: constantinopleInstructionSet, byzantiumInstructionSet, homesteadInstructionSet, frontierInstructionSet. Most of the instructions of these four sets of instruction sets are the same, but as the version is updated, the new version supports more instruction sets than the old version.

\n

evm

The EVM object is the most important object exported by the evm module, which represents an Ethereum virtual machine

\n

creating evm

Every time a transaction is processed, an EVM is created to execute the transaction. This is reflected in the function ApplyTransaction (core/state_processor.go)

\n

creating contract

If the to of the transaction is empty, it means that this transaction is to create a contract, so call EVM.Create to perform related functions

\n
    \n
  • CREATE
    1
    contractAddr = crypto.CreateAddress(caller.Address(), evm.StateDB.GetNonce(caller.Address()))
    • \n
  • CREATE2
    1
    2
    codeAndHash := &codeAndHash{code: code}
    \tcontractAddr = crypto.CreateAddress2(caller.Address(), salt.Bytes32(), codeAndHash.Hash().Bytes())
    \nduring create contract, an object Contract is created. A Contract object contains and maintains the necessary information during the execution of the contract, such as the contract creator, the address of the contract itself, the remaining gas of the contract, the contract code and the jumpdests record of the code.
    • \n
\n

then, it invokes below method to create contract

\n
1
2
ret, err := evm.interpreter.Run(contract, nil, false)
evm.StateDB.SetCode(address, ret)
\n

If the operation is successful and the contract code does not exceed the length limit, call StateDB.SetCode to store the contract code in the contract account of the Ethereum state database. Of course, the storage needs to consume a certain amount of gas.

\n

You may wonder why the stored contract code is the return code after the contract runs, not the data in the original transaction (ie Transaction.data.Payload). This is because when the contract source code is compiled into binary data, in addition to the original code of the contract, the compiler also inserts some codes to perform related functions. For creation, the compiler inserts code that executes the contract’s “constructor” (that is, the contract object’s constructor method). Therefore, when the binary compiled by the compiler is submitted to the Ethereum node to create a contract, the EVM executes this binary code, in fact, it mainly executes the constructor method of the contract, and then returns other codes of the contract, so there is a ret variable here Stored in the state database as the actual code of the contract

\n

call contract

The EVM object has three methods to implement the call of the contract, they are:

\n
    \n
  • EVM. Call
    • \n
  • EVM. CallCode
    • \n
  • EVM. DelegateCall
    • \n
  • EVM.StaticCall
    The basic contract call function implemented by EVM.Call is nothing special. The following three calling methods are the differences compared with EVM.Call. So here we only introduce the particularity of the last three calling methods
    • \n
\n

EVM.CallCode & EVM.DelegateCall

The existence of EVM.CallCode and EVM.DelegateCall is to realize the characteristics of the “library” of the contract. If the code written by solidity is to be called as a library, it must be deployed on the blockchain to obtain a fixed address like a normal contract. , other contracts can call the method provided by this “library contract”. But the contract also involves some unique attributes, such as the caller of the contract, contract address, the amount of ether it owns, etc. If we directly call the code of the “library contract”, these properties must be the properties of the “library contract” itself, but this may not be what we want

\n

as an example

\n
1
A -> contractB - delegateCall -> libC
\n

EVM.DelegateCall sets the caller (msg.sender) of the “library contract” (libC) to A, rather than contractB; sets the address of the “library contract” (libC) to contractB.
EVM.CallCode is similar to EVM.DelegateCall. the only difference is that EVM.CallCode only change the address of the “library contract” (libC) to contractB, without chanding the caller to A.
EVM.StaticCall is similar to EVM.Call, the only difference is that EVM.StaticCall does not allow execution of instructions that modify permanently stored data

\n

during contract call, it first check whether it is precompiled contract. some precompiled contracts are

\n
    \n
  • common.BytesToAddress([]byte{1}): &ecrecover{},
    • \n
  • common.BytesToAddress([]byte{2}): &sha256hash{},
    • \n
  • common.BytesToAddress([]byte{3}): &ripemd160hash{},
    • \n
  • common.BytesToAddress([]byte{4}): &dataCopy{},
    • \n
\n

EVMInterpreter

The interpreter object EVMInterpreter is used to interpret and execute specified contract instructions. However, note that the actual instruction interpretation and execution is not really completed by the interpreter object, but by the operation object JumpTable. The interpreter object is only responsible for parsing instruction codes one by one, and then obtains the corresponding operation object, and check objects such as the stack before calling the operation.execute function that actually executre the instruction. It can also be said that the interpreter object is only responsible for the scheduling of interpretation.

\n

execution layout

\"layout\"

\n

intrinsic gas

The intrinsic gas for a transaction is the amount of gas that the transaction uses before any code runs. It is a constant transaction fee (currently 21000 gas) plus a fee for every byte of data supplied with the transaction (4 gas for a zero byte, 68 gas for non-zeros). These constants are all currently defined for geth in params/protocol_params.go.

\n

gas cost

the gas cost of each instruction is stored in JumpTable.operation.dynamicGas or JumpTable.operation.constantGas. constantGas means the operation gas cost is a fixed constant. dynamicGas is a function which will return gas during runtime.

\n

In fact, not only the interpretation and execution of the instruction itself consumes gas, but also consumes gas when using memory storage and StateDB permanent storage. For most instructions, the latter two are not used (memory & storage), but for some instructions (such as CODECOPY or SSTORE), their gasCost function will take memory and StateDB usage into account.

\n

a method memoryGasCostis used to calculate the gas consumption of memory usage. only when the required space size exceeds the current space size, the excess part needs to consume gas.

\n

JumpTable

jumptable is 256 sized array of operation

\n

jump instruction

Among the instructions of the contract, there are two jump instructions (excluding CALL): JUMP and JUMPI. Their special feature is that the first instruction of the target address after the jump must be JUMPDEST

\n
1
2
3
4
5
6
7
8
9
10
11
func opJump(pc *uint64, interpreter *EVMInterpreter, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) {
    pos := stack.pop()
    if !contract.validJumpdest(pos) {
        nop := contract.GetOp(pos.Uint64())
        return nil, fmt.Errorf("invalid jump destination (%v) %v", nop, pos)
    }
    *pc = pos.Uint64()

    interpreter.intPool.put(pos)
    return nil, nil
}
\n

A function interprets and executes the JUMP instruction. The code first fetches a value from the stack as the jump destination. This value is actually an offset relative to field 0 of the contract code. Then the code will call Contract.validJumpdest to determine whether the first instruction of this destination is JUMPDEST, if it is not, an error will occur.

\n

To judge whether the first instruction of the destination is JUMPDEST, two points must be guaranteed: first, its value is the value of the opcode of the JUMPDEST instruction; second, it is an instruction, not ordinary data.

\n

Let’s introduce how Contract.validJumpdest works. In addition to comparing opcode (this is very simple), Contract will also create a bit vector object (ie bitvec, bit vector). This object will analyze the contract instructions from the beginning to the end. If the byte at a certain offset of the contract belongs to ordinary data, the “bit” corresponding to the offset value in bitvec is set to 1, and if it is an instruction, it is set to 0. In Contract.validJumpdest, it is judged whether this is a normal instruction by checking whether the “bit” of the offset value of the jump destination in this bit vector object is 0

\n

references

\n","site":{"data":{}},"excerpt":"","more":"

overall

the code is under path core/vm
overview of the whole evm module \"evm\"

\n

the core is EVM struct (in evm.go), with main function in creating or call contract. a new EVM object is created every time when processing a transaction. inside the EVM struct, the main items are Interpreter, and StateDB (for state persistence). Interpreter loops through contract call instructions.Before each instruction is executed, some checks are performed to ensure sufficient gas and stack space. actual instruction execution code is recorded in JumpTable (256 sized array of operation)

\n

depending on the version of Ethereum, JumpTable may point to four different instruction sets: constantinopleInstructionSet, byzantiumInstructionSet, homesteadInstructionSet, frontierInstructionSet. Most of the instructions of these four sets of instruction sets are the same, but as the version is updated, the new version supports more instruction sets than the old version.

\n

evm

The EVM object is the most important object exported by the evm module, which represents an Ethereum virtual machine

\n

creating evm

Every time a transaction is processed, an EVM is created to execute the transaction. This is reflected in the function ApplyTransaction (core/state_processor.go)

\n

creating contract

If the to of the transaction is empty, it means that this transaction is to create a contract, so call EVM.Create to perform related functions

\n
    \n
  • CREATE
    1
    contractAddr = crypto.CreateAddress(caller.Address(), evm.StateDB.GetNonce(caller.Address()))
    • \n
  • CREATE2
    1
    2
    codeAndHash := &codeAndHash{code: code}
    \tcontractAddr = crypto.CreateAddress2(caller.Address(), salt.Bytes32(), codeAndHash.Hash().Bytes())
    \nduring create contract, an object Contract is created. A Contract object contains and maintains the necessary information during the execution of the contract, such as the contract creator, the address of the contract itself, the remaining gas of the contract, the contract code and the jumpdests record of the code.
    • \n
\n

then, it invokes below method to create contract

\n
1
2
ret, err := evm.interpreter.Run(contract, nil, false)
evm.StateDB.SetCode(address, ret)
\n

If the operation is successful and the contract code does not exceed the length limit, call StateDB.SetCode to store the contract code in the contract account of the Ethereum state database. Of course, the storage needs to consume a certain amount of gas.

\n

You may wonder why the stored contract code is the return code after the contract runs, not the data in the original transaction (ie Transaction.data.Payload). This is because when the contract source code is compiled into binary data, in addition to the original code of the contract, the compiler also inserts some codes to perform related functions. For creation, the compiler inserts code that executes the contract’s “constructor” (that is, the contract object’s constructor method). Therefore, when the binary compiled by the compiler is submitted to the Ethereum node to create a contract, the EVM executes this binary code, in fact, it mainly executes the constructor method of the contract, and then returns other codes of the contract, so there is a ret variable here Stored in the state database as the actual code of the contract

\n

call contract

The EVM object has three methods to implement the call of the contract, they are:

\n
    \n
  • EVM. Call
    • \n
  • EVM. CallCode
    • \n
  • EVM. DelegateCall
    • \n
  • EVM.StaticCall
    The basic contract call function implemented by EVM.Call is nothing special. The following three calling methods are the differences compared with EVM.Call. So here we only introduce the particularity of the last three calling methods
    • \n
\n

EVM.CallCode & EVM.DelegateCall

The existence of EVM.CallCode and EVM.DelegateCall is to realize the characteristics of the “library” of the contract. If the code written by solidity is to be called as a library, it must be deployed on the blockchain to obtain a fixed address like a normal contract. , other contracts can call the method provided by this “library contract”. But the contract also involves some unique attributes, such as the caller of the contract, contract address, the amount of ether it owns, etc. If we directly call the code of the “library contract”, these properties must be the properties of the “library contract” itself, but this may not be what we want

\n

as an example

\n
1
A -> contractB - delegateCall -> libC
\n

EVM.DelegateCall sets the caller (msg.sender) of the “library contract” (libC) to A, rather than contractB; sets the address of the “library contract” (libC) to contractB.
EVM.CallCode is similar to EVM.DelegateCall. the only difference is that EVM.CallCode only change the address of the “library contract” (libC) to contractB, without chanding the caller to A.
EVM.StaticCall is similar to EVM.Call, the only difference is that EVM.StaticCall does not allow execution of instructions that modify permanently stored data

\n

during contract call, it first check whether it is precompiled contract. some precompiled contracts are

\n
    \n
  • common.BytesToAddress([]byte{1}): &ecrecover{},
    • \n
  • common.BytesToAddress([]byte{2}): &sha256hash{},
    • \n
  • common.BytesToAddress([]byte{3}): &ripemd160hash{},
    • \n
  • common.BytesToAddress([]byte{4}): &dataCopy{},
    • \n
\n

EVMInterpreter

The interpreter object EVMInterpreter is used to interpret and execute specified contract instructions. However, note that the actual instruction interpretation and execution is not really completed by the interpreter object, but by the operation object JumpTable. The interpreter object is only responsible for parsing instruction codes one by one, and then obtains the corresponding operation object, and check objects such as the stack before calling the operation.execute function that actually executre the instruction. It can also be said that the interpreter object is only responsible for the scheduling of interpretation.

\n

execution layout

\"layout\"

\n

intrinsic gas

The intrinsic gas for a transaction is the amount of gas that the transaction uses before any code runs. It is a constant transaction fee (currently 21000 gas) plus a fee for every byte of data supplied with the transaction (4 gas for a zero byte, 68 gas for non-zeros). These constants are all currently defined for geth in params/protocol_params.go.

\n

gas cost

the gas cost of each instruction is stored in JumpTable.operation.dynamicGas or JumpTable.operation.constantGas. constantGas means the operation gas cost is a fixed constant. dynamicGas is a function which will return gas during runtime.

\n

In fact, not only the interpretation and execution of the instruction itself consumes gas, but also consumes gas when using memory storage and StateDB permanent storage. For most instructions, the latter two are not used (memory & storage), but for some instructions (such as CODECOPY or SSTORE), their gasCost function will take memory and StateDB usage into account.

\n

a method memoryGasCostis used to calculate the gas consumption of memory usage. only when the required space size exceeds the current space size, the excess part needs to consume gas.

\n

JumpTable

jumptable is 256 sized array of operation

\n

jump instruction

Among the instructions of the contract, there are two jump instructions (excluding CALL): JUMP and JUMPI. Their special feature is that the first instruction of the target address after the jump must be JUMPDEST

\n
1
2
3
4
5
6
7
8
9
10
11
func opJump(pc *uint64, interpreter *EVMInterpreter, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) {
    pos := stack.pop()
    if !contract.validJumpdest(pos) {
        nop := contract.GetOp(pos.Uint64())
        return nil, fmt.Errorf("invalid jump destination (%v) %v", nop, pos)
    }
    *pc = pos.Uint64()

    interpreter.intPool.put(pos)
    return nil, nil
}
\n

A function interprets and executes the JUMP instruction. The code first fetches a value from the stack as the jump destination. This value is actually an offset relative to field 0 of the contract code. Then the code will call Contract.validJumpdest to determine whether the first instruction of this destination is JUMPDEST, if it is not, an error will occur.

\n

To judge whether the first instruction of the destination is JUMPDEST, two points must be guaranteed: first, its value is the value of the opcode of the JUMPDEST instruction; second, it is an instruction, not ordinary data.

\n

Let’s introduce how Contract.validJumpdest works. In addition to comparing opcode (this is very simple), Contract will also create a bit vector object (ie bitvec, bit vector). This object will analyze the contract instructions from the beginning to the end. If the byte at a certain offset of the contract belongs to ordinary data, the “bit” corresponding to the offset value in bitvec is set to 1, and if it is an instruction, it is set to 0. In Contract.validJumpdest, it is judged whether this is a normal instruction by checking whether the “bit” of the offset value of the jump destination in this bit vector object is 0

\n

references

\n"},{"title":"geth state prune","date":"2023-03-25T11:29:43.000Z","_content":"\n> **_NOTE:_** Offline pruning is only for the hash-based state scheme. In future release, geth will have a path-based state scheme which enables the pruning by default. Once the hash-based state scheme is no longer supported, offline pruning will be deprecated.\n\n## introduction\nA snap-sync'd Geth node currently requires more than 650 GB of disk space to store the historic blockchain data. With default cache size the database grows by about 14 GB/week. Since Geth v1.10, users have been able to trigger a snapshot offline prune to bring the total storage back down to the original ~650 GB in about 4-5 hours.\n\n## how pruning works\nPruning uses snapshots of the state database as an indicator to determine which nodes in the state trie can be kept and which ones are stale and can be discarded. Geth identifies the target state trie based on a stored snapshot layer which has at least 128 block confirmations on top (for surviving reorgs) data that isn't part of the target state trie or genesis state.\n\nGeth prunes the database in three stages:\n\n1. Iterating state snapshot: Geth iterates the bottom-most snapshot layer and constructs a bloom filter set for identifying the target trie nodes.\n2. Pruning state data: Geth deletes stale trie nodes from the database which are not in the bloom filter set.\n3. Compacting database: Geth tidies up the new database to reclaim free space.\n\n## Pruning command\n```\ngeth snapshot prune-state\n```\n\n## references\n- [geth doc](https://geth.ethereum.org/docs/fundamentals/pruning)","source":"_posts/geth/tech_docs/geth.prune.md","raw":"---\ntitle: geth state prune\ndate: 2023-03-25 19:29:43\ntags: [geth]\n---\n\n> **_NOTE:_** Offline pruning is only for the hash-based state scheme. In future release, geth will have a path-based state scheme which enables the pruning by default. Once the hash-based state scheme is no longer supported, offline pruning will be deprecated.\n\n## introduction\nA snap-sync'd Geth node currently requires more than 650 GB of disk space to store the historic blockchain data. With default cache size the database grows by about 14 GB/week. Since Geth v1.10, users have been able to trigger a snapshot offline prune to bring the total storage back down to the original ~650 GB in about 4-5 hours.\n\n## how pruning works\nPruning uses snapshots of the state database as an indicator to determine which nodes in the state trie can be kept and which ones are stale and can be discarded. Geth identifies the target state trie based on a stored snapshot layer which has at least 128 block confirmations on top (for surviving reorgs) data that isn't part of the target state trie or genesis state.\n\nGeth prunes the database in three stages:\n\n1. Iterating state snapshot: Geth iterates the bottom-most snapshot layer and constructs a bloom filter set for identifying the target trie nodes.\n2. Pruning state data: Geth deletes stale trie nodes from the database which are not in the bloom filter set.\n3. Compacting database: Geth tidies up the new database to reclaim free space.\n\n## Pruning command\n```\ngeth snapshot prune-state\n```\n\n## references\n- [geth doc](https://geth.ethereum.org/docs/fundamentals/pruning)","slug":"geth/tech_docs/geth.prune","published":1,"updated":"2023-11-05T04:21:13.733Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8dx002aqwsjacvpcivw","content":"
\n

NOTE: Offline pruning is only for the hash-based state scheme. In future release, geth will have a path-based state scheme which enables the pruning by default. Once the hash-based state scheme is no longer supported, offline pruning will be deprecated.

\n
\n

introduction

A snap-sync’d Geth node currently requires more than 650 GB of disk space to store the historic blockchain data. With default cache size the database grows by about 14 GB/week. Since Geth v1.10, users have been able to trigger a snapshot offline prune to bring the total storage back down to the original ~650 GB in about 4-5 hours.

\n

how pruning works

Pruning uses snapshots of the state database as an indicator to determine which nodes in the state trie can be kept and which ones are stale and can be discarded. Geth identifies the target state trie based on a stored snapshot layer which has at least 128 block confirmations on top (for surviving reorgs) data that isn’t part of the target state trie or genesis state.

\n

Geth prunes the database in three stages:

\n
    \n
  1. Iterating state snapshot: Geth iterates the bottom-most snapshot layer and constructs a bloom filter set for identifying the target trie nodes.
    2. \n
  2. Pruning state data: Geth deletes stale trie nodes from the database which are not in the bloom filter set.
    3. \n
  3. Compacting database: Geth tidies up the new database to reclaim free space.
    4. \n
\n

Pruning command

1
geth snapshot prune-state
\n\n

references

\n","site":{"data":{}},"excerpt":"","more":"
\n

NOTE: Offline pruning is only for the hash-based state scheme. In future release, geth will have a path-based state scheme which enables the pruning by default. Once the hash-based state scheme is no longer supported, offline pruning will be deprecated.

\n
\n

introduction

A snap-sync’d Geth node currently requires more than 650 GB of disk space to store the historic blockchain data. With default cache size the database grows by about 14 GB/week. Since Geth v1.10, users have been able to trigger a snapshot offline prune to bring the total storage back down to the original ~650 GB in about 4-5 hours.

\n

how pruning works

Pruning uses snapshots of the state database as an indicator to determine which nodes in the state trie can be kept and which ones are stale and can be discarded. Geth identifies the target state trie based on a stored snapshot layer which has at least 128 block confirmations on top (for surviving reorgs) data that isn’t part of the target state trie or genesis state.

\n

Geth prunes the database in three stages:

\n
    \n
  1. Iterating state snapshot: Geth iterates the bottom-most snapshot layer and constructs a bloom filter set for identifying the target trie nodes.
    2. \n
  2. Pruning state data: Geth deletes stale trie nodes from the database which are not in the bloom filter set.
    3. \n
  3. Compacting database: Geth tidies up the new database to reclaim free space.
    4. \n
\n

Pruning command

1
geth snapshot prune-state
\n\n

references

\n"},{"title":"geth sync","date":"2023-03-18T08:29:43.000Z","_content":"\n## state \nEthereum maintains two different types of state: the set of accounts; and a set of storage slots for each contract account. Naively, storing these key-value pairs as flat data would be very efficient, however, verifying their correctness becomes computationally intractable. Every time a modification would be made, we'd need to hash all that data from scratch (which is not efficient).\n\nInstead of hashing the entire dataset all the time, eth uses MPT. The original useful data would be in the leaves, and each internal node would be a hash of everything below it. This would allow us to only recalculate a logarithmic number of hashes when something is modified, inserted, deleted and verified. A tiny extra is that keys are hashed before insertion to balance the tries (secured trie).\n\n## state storage\nMPT make every read/write of O(lnN) complexity. the depth of the trie is continuously growing; LevelDB also organizes its data into a maximum of 7 levels, so there's an extra multiplier from there. The net result is that a single state access is expected to amplify into **25-50** random disk accesses. \n\nOf course all client implementations try their best to minimize this overhead. Geth uses large memory areas for caching trie nodes; and also uses in-memory pruning to avoid writing to disk nodes that get deleted anyway after a few blocks.\n\n## Not all accesses are created equal\nThe Merkle Patricia tree is essential for writes (matain the capability to verify data), but it's an overhead for reads. \nAn Ethereum node accesses state in a few different places:\n- When importing a new block, EVM code execution does a more-or-less balanced number of state reads and writes. \n- When a node operator retrieves state (e.g. eth_call and family), EVM code execution only does reads (it can write too, but those get discarded at the end and are not persisted).\n- When a node is synchronizing, it is requesting state from remote nodes that need to dig it up and serve it over the network.\n\nif we can short circuit reads not to hit the state trie, a slew of node operations will become significantly faster. \n\n## snapshot\nGeth introduced its snapshot acceleration structure (not enabled by default). A snapshot is a complete view of the Ethereum state at a given block. Abstract implementation wise, it is a dump of all accounts and storage slots, represented by a flat key-value store.\nsnapshot is maintained as an extra to MPT. The snapshot essentially reduces reads from O(log n) to O(1) at the cost of increasing writes from O(log n) to O(1 + log n). \n\n## devil's in the details\nto maintain a snapshot, the naitve approach is to apply changes to current snapshot upon new block. If there's a mini reorg however (even a single block), we're in trouble, because there's no undo. \nTo overcome this limitation, Geth's snapshot is composed of two entities: a persistent disk layer that is a complete snapshot of an older block (e.g. HEAD-128); and a tree of in-memory diff layers that gather the writes on top.\nWhenever a new block is processed, we do not merge the writes directly into the disk layer, rather just create a new in-memory diff layer with the changes. If enough in-memory diff layers are piled on top, the bottom ones start getting merged together and eventually pushed to disk. Whenever a state item is to be read, we start at the topmost diff layer and keep going backwards until we find it or reach the disk.\nOf course, there are lots and lots of gotchas and caveats.\n- On shutdown, the in-memory diff layers need to be persisted into a journal and loaded back up, otherwise the snapshot will become useless on restart.\n- Use the bottom-most diff layer as an accumulator and only flush to disk when it exceeds some memory usage.\n- Allocate a read cache for the disk layer so that contracts accessing the same ancient slot over and over don't cause disk hits.\n- Use cumulative bloom filters in the in-memory diff layers to quickly detect whether there's a chance for an item to be in the diffs, or if we can go to disk immediately.\n- The keys are not the raw data (account address, storage key), rather the hashes of these, ensuring the snapshot has the same iteration order as the Merkle Patricia tree.\n\nThe snapshot also enables blazing fast state iteration of the most recent blocks. This was actually the main reason for building snapshots, as it permitted the creation of the new snap [sync algorithm](https://github.com/ethereum/devp2p/pull/145).\n\n## Consensus layer syncing\nall consensus logic and block propagation is handled by consensus clients. Blocks are downloaded by the consensus client and verified by the execution client. **Geth cannot sync without being connected to a consensus client.** \nThere are two ways for the consensus client to find a block header that Geth can use as a sync target: optimistic syncing and checkpoint syncing:\n\n### optimistic sync\nOptimistic sync downloads blocks before the execution client has validated them. In optimistic sync the node assumes the data it receives from its peers is correct during the downloading phase but then retroactively verifies each downloaded block.\n[more details](https://github.com/ethereum/consensus-specs/blob/dev/sync/optimistic.md)\n\n### checkpoint sync\nAlternatively, the consensus client can grab a checkpoint from a trusted source which provides a target state to sync up to, before switching to full sync and verifying each block in turn. In this mode, the node trusts that the checkpoint is correct.\n\n## archive nodes\nAn archive node is a node that retains all historical data right back to genesis. There is no need to regenerate any data from checkpoints because all data is directly available in the node's own storage. \n\nIt is also possible to create a partial/recent archive node where the node was synced using snap but the state is never pruned. This creates an archive node that saves all state data from the point that the node first syncs. This is configured by starting Geth with `--syncmode snap --gcmode archive`.\n\n\n## light nodes\nA light node syncs very quickly and stores the bare minimum of blockchain data. Light nodes only process block headers, not entire blocks. they receive a proof from the full node and verify it against their local header chain. **Light nodes are not currently working on proof-of-stake Ethereum.**\n\n\n\n## full node\n### full\nA full block-by-block sync generates the current state by executing every block starting from the genesis block. A full sync independently verifies block provenance as well as all state transitions by re-executing the transactions in the entire historical sequence of blocks. Only the most recent 128 block states are stored in a full node - older block states are pruned periodically and represented as a series of checkpoints from which any previous state can be regenerated on request.\n\n### snap sync (default)\nSnap sync starts from a relatively recent block and syncs from there to the head of the chain, keeping only the most recent 128 block states in memory. The block header to sync up to is provided by the consensus client. Between the initial sync block and the 128 most recent blocks, the node stores occasional snapshots that can be used to rebuild any intermediate state \"on-the-fly\". The difference between the snap-synced node and a full block-by-block synced node is that a snap synced node started from an initial checkpoint that was more recent than the genesis block. Snap sync is much faster than a full block-by-block sync from genesis.\n![sync mode](/images/geth/sync.mode.jpg)\n\nSnap sync works by first downloading the headers for a chunk of blocks. Once the headers have been verified, the block bodies and receipts for those blocks are downloaded. In parallel, Geth also begins state-sync. In state-sync, Geth first downloads the leaves of the state trie for each block without the intermediate nodes along with a range proof. The state trie is then regenerated locally.\n\n\nThe state download is the part of the snap-sync that takes the most time to complete and the progress can be monitored using the ETA values in the log messages. However, the blockchain is also progressing at the same time and invalidating some of the regenerated state data. (don't really understand why regenrated state could be invalidated). This means it is also necessary to have a 'healing' phase where errors in the state are fixed. Geth regularly reports `Syncing, state heal in progress` during state healing - this informs the user that state heal has not finished.\n\nThe healing has to outpace the growth of the blockchain, otherwise the node will never catch up to the current state.\n\nTo summarize, snap sync progresses in the following sequence:\n\n- download and verify headers\n- download block bodies and receipts. In parallel, download raw state data and build state trie\n- heal state trie to account for newly arriving data\n\nA node that is started using snap will switch to block-by-block sync once it has caught up to the head of the chain.\n\n\n\n\n\n\n\n\n\n# references\n- [geth doc on sync mode](https://geth.ethereum.org/docs/fundamentals/sync-modes)\n- [eth.org blog on snapshot acceleration](https://blog.ethereum.org/2020/07/17/ask-about-geth-snapshot-acceleration)","source":"_posts/geth/tech_docs/geth.sync.mode.md","raw":"---\ntitle: geth sync\ndate: 2023-03-18 16:29:43\ntags: [geth]\n---\n\n## state \nEthereum maintains two different types of state: the set of accounts; and a set of storage slots for each contract account. Naively, storing these key-value pairs as flat data would be very efficient, however, verifying their correctness becomes computationally intractable. Every time a modification would be made, we'd need to hash all that data from scratch (which is not efficient).\n\nInstead of hashing the entire dataset all the time, eth uses MPT. The original useful data would be in the leaves, and each internal node would be a hash of everything below it. This would allow us to only recalculate a logarithmic number of hashes when something is modified, inserted, deleted and verified. A tiny extra is that keys are hashed before insertion to balance the tries (secured trie).\n\n## state storage\nMPT make every read/write of O(lnN) complexity. the depth of the trie is continuously growing; LevelDB also organizes its data into a maximum of 7 levels, so there's an extra multiplier from there. The net result is that a single state access is expected to amplify into **25-50** random disk accesses. \n\nOf course all client implementations try their best to minimize this overhead. Geth uses large memory areas for caching trie nodes; and also uses in-memory pruning to avoid writing to disk nodes that get deleted anyway after a few blocks.\n\n## Not all accesses are created equal\nThe Merkle Patricia tree is essential for writes (matain the capability to verify data), but it's an overhead for reads. \nAn Ethereum node accesses state in a few different places:\n- When importing a new block, EVM code execution does a more-or-less balanced number of state reads and writes. \n- When a node operator retrieves state (e.g. eth_call and family), EVM code execution only does reads (it can write too, but those get discarded at the end and are not persisted).\n- When a node is synchronizing, it is requesting state from remote nodes that need to dig it up and serve it over the network.\n\nif we can short circuit reads not to hit the state trie, a slew of node operations will become significantly faster. \n\n## snapshot\nGeth introduced its snapshot acceleration structure (not enabled by default). A snapshot is a complete view of the Ethereum state at a given block. Abstract implementation wise, it is a dump of all accounts and storage slots, represented by a flat key-value store.\nsnapshot is maintained as an extra to MPT. The snapshot essentially reduces reads from O(log n) to O(1) at the cost of increasing writes from O(log n) to O(1 + log n). \n\n## devil's in the details\nto maintain a snapshot, the naitve approach is to apply changes to current snapshot upon new block. If there's a mini reorg however (even a single block), we're in trouble, because there's no undo. \nTo overcome this limitation, Geth's snapshot is composed of two entities: a persistent disk layer that is a complete snapshot of an older block (e.g. HEAD-128); and a tree of in-memory diff layers that gather the writes on top.\nWhenever a new block is processed, we do not merge the writes directly into the disk layer, rather just create a new in-memory diff layer with the changes. If enough in-memory diff layers are piled on top, the bottom ones start getting merged together and eventually pushed to disk. Whenever a state item is to be read, we start at the topmost diff layer and keep going backwards until we find it or reach the disk.\nOf course, there are lots and lots of gotchas and caveats.\n- On shutdown, the in-memory diff layers need to be persisted into a journal and loaded back up, otherwise the snapshot will become useless on restart.\n- Use the bottom-most diff layer as an accumulator and only flush to disk when it exceeds some memory usage.\n- Allocate a read cache for the disk layer so that contracts accessing the same ancient slot over and over don't cause disk hits.\n- Use cumulative bloom filters in the in-memory diff layers to quickly detect whether there's a chance for an item to be in the diffs, or if we can go to disk immediately.\n- The keys are not the raw data (account address, storage key), rather the hashes of these, ensuring the snapshot has the same iteration order as the Merkle Patricia tree.\n\nThe snapshot also enables blazing fast state iteration of the most recent blocks. This was actually the main reason for building snapshots, as it permitted the creation of the new snap [sync algorithm](https://github.com/ethereum/devp2p/pull/145).\n\n## Consensus layer syncing\nall consensus logic and block propagation is handled by consensus clients. Blocks are downloaded by the consensus client and verified by the execution client. **Geth cannot sync without being connected to a consensus client.** \nThere are two ways for the consensus client to find a block header that Geth can use as a sync target: optimistic syncing and checkpoint syncing:\n\n### optimistic sync\nOptimistic sync downloads blocks before the execution client has validated them. In optimistic sync the node assumes the data it receives from its peers is correct during the downloading phase but then retroactively verifies each downloaded block.\n[more details](https://github.com/ethereum/consensus-specs/blob/dev/sync/optimistic.md)\n\n### checkpoint sync\nAlternatively, the consensus client can grab a checkpoint from a trusted source which provides a target state to sync up to, before switching to full sync and verifying each block in turn. In this mode, the node trusts that the checkpoint is correct.\n\n## archive nodes\nAn archive node is a node that retains all historical data right back to genesis. There is no need to regenerate any data from checkpoints because all data is directly available in the node's own storage. \n\nIt is also possible to create a partial/recent archive node where the node was synced using snap but the state is never pruned. This creates an archive node that saves all state data from the point that the node first syncs. This is configured by starting Geth with `--syncmode snap --gcmode archive`.\n\n\n## light nodes\nA light node syncs very quickly and stores the bare minimum of blockchain data. Light nodes only process block headers, not entire blocks. they receive a proof from the full node and verify it against their local header chain. **Light nodes are not currently working on proof-of-stake Ethereum.**\n\n\n\n## full node\n### full\nA full block-by-block sync generates the current state by executing every block starting from the genesis block. A full sync independently verifies block provenance as well as all state transitions by re-executing the transactions in the entire historical sequence of blocks. Only the most recent 128 block states are stored in a full node - older block states are pruned periodically and represented as a series of checkpoints from which any previous state can be regenerated on request.\n\n### snap sync (default)\nSnap sync starts from a relatively recent block and syncs from there to the head of the chain, keeping only the most recent 128 block states in memory. The block header to sync up to is provided by the consensus client. Between the initial sync block and the 128 most recent blocks, the node stores occasional snapshots that can be used to rebuild any intermediate state \"on-the-fly\". The difference between the snap-synced node and a full block-by-block synced node is that a snap synced node started from an initial checkpoint that was more recent than the genesis block. Snap sync is much faster than a full block-by-block sync from genesis.\n![sync mode](/images/geth/sync.mode.jpg)\n\nSnap sync works by first downloading the headers for a chunk of blocks. Once the headers have been verified, the block bodies and receipts for those blocks are downloaded. In parallel, Geth also begins state-sync. In state-sync, Geth first downloads the leaves of the state trie for each block without the intermediate nodes along with a range proof. The state trie is then regenerated locally.\n\n\nThe state download is the part of the snap-sync that takes the most time to complete and the progress can be monitored using the ETA values in the log messages. However, the blockchain is also progressing at the same time and invalidating some of the regenerated state data. (don't really understand why regenrated state could be invalidated). This means it is also necessary to have a 'healing' phase where errors in the state are fixed. Geth regularly reports `Syncing, state heal in progress` during state healing - this informs the user that state heal has not finished.\n\nThe healing has to outpace the growth of the blockchain, otherwise the node will never catch up to the current state.\n\nTo summarize, snap sync progresses in the following sequence:\n\n- download and verify headers\n- download block bodies and receipts. In parallel, download raw state data and build state trie\n- heal state trie to account for newly arriving data\n\nA node that is started using snap will switch to block-by-block sync once it has caught up to the head of the chain.\n\n\n\n\n\n\n\n\n\n# references\n- [geth doc on sync mode](https://geth.ethereum.org/docs/fundamentals/sync-modes)\n- [eth.org blog on snapshot acceleration](https://blog.ethereum.org/2020/07/17/ask-about-geth-snapshot-acceleration)","slug":"geth/tech_docs/geth.sync.mode","published":1,"updated":"2023-11-05T04:21:13.733Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8dx002dqwsj8w7zb4o4","content":"

state

Ethereum maintains two different types of state: the set of accounts; and a set of storage slots for each contract account. Naively, storing these key-value pairs as flat data would be very efficient, however, verifying their correctness becomes computationally intractable. Every time a modification would be made, we’d need to hash all that data from scratch (which is not efficient).

\n

Instead of hashing the entire dataset all the time, eth uses MPT. The original useful data would be in the leaves, and each internal node would be a hash of everything below it. This would allow us to only recalculate a logarithmic number of hashes when something is modified, inserted, deleted and verified. A tiny extra is that keys are hashed before insertion to balance the tries (secured trie).

\n

state storage

MPT make every read/write of O(lnN) complexity. the depth of the trie is continuously growing; LevelDB also organizes its data into a maximum of 7 levels, so there’s an extra multiplier from there. The net result is that a single state access is expected to amplify into 25-50 random disk accesses.

\n

Of course all client implementations try their best to minimize this overhead. Geth uses large memory areas for caching trie nodes; and also uses in-memory pruning to avoid writing to disk nodes that get deleted anyway after a few blocks.

\n

Not all accesses are created equal

The Merkle Patricia tree is essential for writes (matain the capability to verify data), but it’s an overhead for reads.
An Ethereum node accesses state in a few different places:

\n
    \n
  • When importing a new block, EVM code execution does a more-or-less balanced number of state reads and writes.
    • \n
  • When a node operator retrieves state (e.g. eth_call and family), EVM code execution only does reads (it can write too, but those get discarded at the end and are not persisted).
    • \n
  • When a node is synchronizing, it is requesting state from remote nodes that need to dig it up and serve it over the network.
    • \n
\n

if we can short circuit reads not to hit the state trie, a slew of node operations will become significantly faster.

\n

snapshot

Geth introduced its snapshot acceleration structure (not enabled by default). A snapshot is a complete view of the Ethereum state at a given block. Abstract implementation wise, it is a dump of all accounts and storage slots, represented by a flat key-value store.
snapshot is maintained as an extra to MPT. The snapshot essentially reduces reads from O(log n) to O(1) at the cost of increasing writes from O(log n) to O(1 + log n).

\n

devil’s in the details

to maintain a snapshot, the naitve approach is to apply changes to current snapshot upon new block. If there’s a mini reorg however (even a single block), we’re in trouble, because there’s no undo.
To overcome this limitation, Geth’s snapshot is composed of two entities: a persistent disk layer that is a complete snapshot of an older block (e.g. HEAD-128); and a tree of in-memory diff layers that gather the writes on top.
Whenever a new block is processed, we do not merge the writes directly into the disk layer, rather just create a new in-memory diff layer with the changes. If enough in-memory diff layers are piled on top, the bottom ones start getting merged together and eventually pushed to disk. Whenever a state item is to be read, we start at the topmost diff layer and keep going backwards until we find it or reach the disk.
Of course, there are lots and lots of gotchas and caveats.

\n
    \n
  • On shutdown, the in-memory diff layers need to be persisted into a journal and loaded back up, otherwise the snapshot will become useless on restart.
    • \n
  • Use the bottom-most diff layer as an accumulator and only flush to disk when it exceeds some memory usage.
    • \n
  • Allocate a read cache for the disk layer so that contracts accessing the same ancient slot over and over don’t cause disk hits.
    • \n
  • Use cumulative bloom filters in the in-memory diff layers to quickly detect whether there’s a chance for an item to be in the diffs, or if we can go to disk immediately.
    • \n
  • The keys are not the raw data (account address, storage key), rather the hashes of these, ensuring the snapshot has the same iteration order as the Merkle Patricia tree.
    • \n
\n

The snapshot also enables blazing fast state iteration of the most recent blocks. This was actually the main reason for building snapshots, as it permitted the creation of the new snap sync algorithm.

\n

Consensus layer syncing

all consensus logic and block propagation is handled by consensus clients. Blocks are downloaded by the consensus client and verified by the execution client. Geth cannot sync without being connected to a consensus client.
There are two ways for the consensus client to find a block header that Geth can use as a sync target: optimistic syncing and checkpoint syncing:

\n

optimistic sync

Optimistic sync downloads blocks before the execution client has validated them. In optimistic sync the node assumes the data it receives from its peers is correct during the downloading phase but then retroactively verifies each downloaded block.
more details

\n

checkpoint sync

Alternatively, the consensus client can grab a checkpoint from a trusted source which provides a target state to sync up to, before switching to full sync and verifying each block in turn. In this mode, the node trusts that the checkpoint is correct.

\n

archive nodes

An archive node is a node that retains all historical data right back to genesis. There is no need to regenerate any data from checkpoints because all data is directly available in the node’s own storage.

\n

It is also possible to create a partial/recent archive node where the node was synced using snap but the state is never pruned. This creates an archive node that saves all state data from the point that the node first syncs. This is configured by starting Geth with --syncmode snap --gcmode archive.

\n

light nodes

A light node syncs very quickly and stores the bare minimum of blockchain data. Light nodes only process block headers, not entire blocks. they receive a proof from the full node and verify it against their local header chain. Light nodes are not currently working on proof-of-stake Ethereum.

\n

full node

full

A full block-by-block sync generates the current state by executing every block starting from the genesis block. A full sync independently verifies block provenance as well as all state transitions by re-executing the transactions in the entire historical sequence of blocks. Only the most recent 128 block states are stored in a full node - older block states are pruned periodically and represented as a series of checkpoints from which any previous state can be regenerated on request.

\n

snap sync (default)

Snap sync starts from a relatively recent block and syncs from there to the head of the chain, keeping only the most recent 128 block states in memory. The block header to sync up to is provided by the consensus client. Between the initial sync block and the 128 most recent blocks, the node stores occasional snapshots that can be used to rebuild any intermediate state “on-the-fly”. The difference between the snap-synced node and a full block-by-block synced node is that a snap synced node started from an initial checkpoint that was more recent than the genesis block. Snap sync is much faster than a full block-by-block sync from genesis.
\"sync

\n

Snap sync works by first downloading the headers for a chunk of blocks. Once the headers have been verified, the block bodies and receipts for those blocks are downloaded. In parallel, Geth also begins state-sync. In state-sync, Geth first downloads the leaves of the state trie for each block without the intermediate nodes along with a range proof. The state trie is then regenerated locally.

\n

The state download is the part of the snap-sync that takes the most time to complete and the progress can be monitored using the ETA values in the log messages. However, the blockchain is also progressing at the same time and invalidating some of the regenerated state data. (don’t really understand why regenrated state could be invalidated). This means it is also necessary to have a ‘healing’ phase where errors in the state are fixed. Geth regularly reports Syncing, state heal in progress during state healing - this informs the user that state heal has not finished.

\n

The healing has to outpace the growth of the blockchain, otherwise the node will never catch up to the current state.

\n

To summarize, snap sync progresses in the following sequence:

\n
    \n
  • download and verify headers
    • \n
  • download block bodies and receipts. In parallel, download raw state data and build state trie
    • \n
  • heal state trie to account for newly arriving data
    • \n
\n

A node that is started using snap will switch to block-by-block sync once it has caught up to the head of the chain.

\n

references

\n","site":{"data":{}},"excerpt":"","more":"

state

Ethereum maintains two different types of state: the set of accounts; and a set of storage slots for each contract account. Naively, storing these key-value pairs as flat data would be very efficient, however, verifying their correctness becomes computationally intractable. Every time a modification would be made, we’d need to hash all that data from scratch (which is not efficient).

\n

Instead of hashing the entire dataset all the time, eth uses MPT. The original useful data would be in the leaves, and each internal node would be a hash of everything below it. This would allow us to only recalculate a logarithmic number of hashes when something is modified, inserted, deleted and verified. A tiny extra is that keys are hashed before insertion to balance the tries (secured trie).

\n

state storage

MPT make every read/write of O(lnN) complexity. the depth of the trie is continuously growing; LevelDB also organizes its data into a maximum of 7 levels, so there’s an extra multiplier from there. The net result is that a single state access is expected to amplify into 25-50 random disk accesses.

\n

Of course all client implementations try their best to minimize this overhead. Geth uses large memory areas for caching trie nodes; and also uses in-memory pruning to avoid writing to disk nodes that get deleted anyway after a few blocks.

\n

Not all accesses are created equal

The Merkle Patricia tree is essential for writes (matain the capability to verify data), but it’s an overhead for reads.
An Ethereum node accesses state in a few different places:

\n
    \n
  • When importing a new block, EVM code execution does a more-or-less balanced number of state reads and writes.
    • \n
  • When a node operator retrieves state (e.g. eth_call and family), EVM code execution only does reads (it can write too, but those get discarded at the end and are not persisted).
    • \n
  • When a node is synchronizing, it is requesting state from remote nodes that need to dig it up and serve it over the network.
    • \n
\n

if we can short circuit reads not to hit the state trie, a slew of node operations will become significantly faster.

\n

snapshot

Geth introduced its snapshot acceleration structure (not enabled by default). A snapshot is a complete view of the Ethereum state at a given block. Abstract implementation wise, it is a dump of all accounts and storage slots, represented by a flat key-value store.
snapshot is maintained as an extra to MPT. The snapshot essentially reduces reads from O(log n) to O(1) at the cost of increasing writes from O(log n) to O(1 + log n).

\n

devil’s in the details

to maintain a snapshot, the naitve approach is to apply changes to current snapshot upon new block. If there’s a mini reorg however (even a single block), we’re in trouble, because there’s no undo.
To overcome this limitation, Geth’s snapshot is composed of two entities: a persistent disk layer that is a complete snapshot of an older block (e.g. HEAD-128); and a tree of in-memory diff layers that gather the writes on top.
Whenever a new block is processed, we do not merge the writes directly into the disk layer, rather just create a new in-memory diff layer with the changes. If enough in-memory diff layers are piled on top, the bottom ones start getting merged together and eventually pushed to disk. Whenever a state item is to be read, we start at the topmost diff layer and keep going backwards until we find it or reach the disk.
Of course, there are lots and lots of gotchas and caveats.

\n
    \n
  • On shutdown, the in-memory diff layers need to be persisted into a journal and loaded back up, otherwise the snapshot will become useless on restart.
    • \n
  • Use the bottom-most diff layer as an accumulator and only flush to disk when it exceeds some memory usage.
    • \n
  • Allocate a read cache for the disk layer so that contracts accessing the same ancient slot over and over don’t cause disk hits.
    • \n
  • Use cumulative bloom filters in the in-memory diff layers to quickly detect whether there’s a chance for an item to be in the diffs, or if we can go to disk immediately.
    • \n
  • The keys are not the raw data (account address, storage key), rather the hashes of these, ensuring the snapshot has the same iteration order as the Merkle Patricia tree.
    • \n
\n

The snapshot also enables blazing fast state iteration of the most recent blocks. This was actually the main reason for building snapshots, as it permitted the creation of the new snap sync algorithm.

\n

Consensus layer syncing

all consensus logic and block propagation is handled by consensus clients. Blocks are downloaded by the consensus client and verified by the execution client. Geth cannot sync without being connected to a consensus client.
There are two ways for the consensus client to find a block header that Geth can use as a sync target: optimistic syncing and checkpoint syncing:

\n

optimistic sync

Optimistic sync downloads blocks before the execution client has validated them. In optimistic sync the node assumes the data it receives from its peers is correct during the downloading phase but then retroactively verifies each downloaded block.
more details

\n

checkpoint sync

Alternatively, the consensus client can grab a checkpoint from a trusted source which provides a target state to sync up to, before switching to full sync and verifying each block in turn. In this mode, the node trusts that the checkpoint is correct.

\n

archive nodes

An archive node is a node that retains all historical data right back to genesis. There is no need to regenerate any data from checkpoints because all data is directly available in the node’s own storage.

\n

It is also possible to create a partial/recent archive node where the node was synced using snap but the state is never pruned. This creates an archive node that saves all state data from the point that the node first syncs. This is configured by starting Geth with --syncmode snap --gcmode archive.

\n

light nodes

A light node syncs very quickly and stores the bare minimum of blockchain data. Light nodes only process block headers, not entire blocks. they receive a proof from the full node and verify it against their local header chain. Light nodes are not currently working on proof-of-stake Ethereum.

\n

full node

full

A full block-by-block sync generates the current state by executing every block starting from the genesis block. A full sync independently verifies block provenance as well as all state transitions by re-executing the transactions in the entire historical sequence of blocks. Only the most recent 128 block states are stored in a full node - older block states are pruned periodically and represented as a series of checkpoints from which any previous state can be regenerated on request.

\n

snap sync (default)

Snap sync starts from a relatively recent block and syncs from there to the head of the chain, keeping only the most recent 128 block states in memory. The block header to sync up to is provided by the consensus client. Between the initial sync block and the 128 most recent blocks, the node stores occasional snapshots that can be used to rebuild any intermediate state “on-the-fly”. The difference between the snap-synced node and a full block-by-block synced node is that a snap synced node started from an initial checkpoint that was more recent than the genesis block. Snap sync is much faster than a full block-by-block sync from genesis.
\"sync

\n

Snap sync works by first downloading the headers for a chunk of blocks. Once the headers have been verified, the block bodies and receipts for those blocks are downloaded. In parallel, Geth also begins state-sync. In state-sync, Geth first downloads the leaves of the state trie for each block without the intermediate nodes along with a range proof. The state trie is then regenerated locally.

\n

The state download is the part of the snap-sync that takes the most time to complete and the progress can be monitored using the ETA values in the log messages. However, the blockchain is also progressing at the same time and invalidating some of the regenerated state data. (don’t really understand why regenrated state could be invalidated). This means it is also necessary to have a ‘healing’ phase where errors in the state are fixed. Geth regularly reports Syncing, state heal in progress during state healing - this informs the user that state heal has not finished.

\n

The healing has to outpace the growth of the blockchain, otherwise the node will never catch up to the current state.

\n

To summarize, snap sync progresses in the following sequence:

\n
    \n
  • download and verify headers
    • \n
  • download block bodies and receipts. In parallel, download raw state data and build state trie
    • \n
  • heal state trie to account for newly arriving data
    • \n
\n

A node that is started using snap will switch to block-by-block sync once it has caught up to the head of the chain.

\n

references

\n"},{"title":"zkp demystify zokrates","date":"2023-07-14T06:29:26.000Z","_content":"\n\n\n## pipeline\n### source file\nan example, `square.zok`\n```\ndef main(private field a, field b) {\n assert(a * a == b);\n return;\n}\n```\n- The keyword private signals that we do not want to reveal this input, but still prove that we know its value.\n### compile\n```\nzokrates compile -i root.zok\n```\nafter compile, it generates below files\n- out\n- out.r1cs\n- abi.json\n### setup\nPerforms a trusted setup for a given constraint system\n```\nzokrates setup\n```\noptions \n- -i, --input Path of the binary [default: out]\nit generates below two files\n- proving.key\n- verification.key","source":"_posts/cryptography/zkp/zkp-demystify-zokrates.md","raw":"---\ntitle: zkp demystify zokrates\ndate: 2023-07-14 14:29:26\ntags: [cryptography,zkp]\n---\n\n\n\n## pipeline\n### source file\nan example, `square.zok`\n```\ndef main(private field a, field b) {\n assert(a * a == b);\n return;\n}\n```\n- The keyword private signals that we do not want to reveal this input, but still prove that we know its value.\n### compile\n```\nzokrates compile -i root.zok\n```\nafter compile, it generates below files\n- out\n- out.r1cs\n- abi.json\n### setup\nPerforms a trusted setup for a given constraint system\n```\nzokrates setup\n```\noptions \n- -i, --input Path of the binary [default: out]\nit generates below two files\n- proving.key\n- verification.key","slug":"cryptography/zkp/zkp-demystify-zokrates","published":1,"updated":"2023-11-05T04:21:17.035Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8e00039qwsjfxurcxan","content":"\n\n\n

pipeline

source file

an example, square.zok

\n
1
2
3
4
def main(private field a, field b) {
    assert(a * a == b);
    return;
}
\n
    \n
  • The keyword private signals that we do not want to reveal this input, but still prove that we know its value.
    • \n
\n

compile

1
zokrates compile -i root.zok
\n

after compile, it generates below files

\n
    \n
  • out
    • \n
  • out.r1cs
    • \n
  • abi.json
    • \n
\n

setup

Performs a trusted setup for a given constraint system

\n
1
zokrates setup
\n

options

\n
    \n
  • -i, –input Path of the binary [default: out]
    it generates below two files
    • \n
  • proving.key
    • \n
  • verification.key
    • \n
\n","site":{"data":{}},"excerpt":"","more":"\n\n\n

pipeline

source file

an example, square.zok

\n
1
2
3
4
def main(private field a, field b) {
    assert(a * a == b);
    return;
}
\n
    \n
  • The keyword private signals that we do not want to reveal this input, but still prove that we know its value.
    • \n
\n

compile

1
zokrates compile -i root.zok
\n

after compile, it generates below files

\n
    \n
  • out
    • \n
  • out.r1cs
    • \n
  • abi.json
    • \n
\n

setup

Performs a trusted setup for a given constraint system

\n
1
zokrates setup
\n

options

\n
    \n
  • -i, –input Path of the binary [default: out]
    it generates below two files
    • \n
  • proving.key
    • \n
  • verification.key
    • \n
\n"},{"title":"zkp a brief understanding (1)","date":"2023-06-27T06:29:26.000Z","_content":"\n\n\n## introduction\nzk-SNARKs cannot be applied to any computational problem directly; rather, you have to convert the problem into the right “form” for the problem to operate on. The form is called a “quadratic arithmetic program” (QAP), and transforming the code of a function into one of these is itself highly nontrivial.\n\nThe example that we will choose is a simple one: proving that you know the solution to a cubic equation: \\\\(x^3 + x + 5 == 35\\\\)\n\nNote that modulo (%) and comparison operators (<, >, ≤, ≥) are NOT supported, as there is no efficient way to do modulo or comparison directly in finite cyclic group arithmetic (be thankful for this; if there was a way to do either one, then elliptic curve cryptography would be broken faster)\n\nYou can extend the language to modulo and comparisons by providing bit decompositions (eg. \\\\(13 = 2^3 + 2^2 + 1\\\\)) as auxiliary inputs, proving correctness of those decompositions and doing the math in binary circuits; in finite field arithmetic, doing equality `(==)` checks is also doable and in fact a bit easier, but these are both details we won’t get into right now. We can extend the language to support conditionals (eg. `if x < 5: y = 7; else: y = 9`) by converting them to an arithmetic form: `y = 7 * (x < 5) + 9 * (x >= 5)`;though note that both “paths” of the conditional would need to be executed.\n\n## Flattening\nThe first step is a “flattening” procedure, where we convert the original code into a sequence of statements that are of two forms\n```\nsym1 = x * x\ny = sym1 * x\nsym2 = y + x\n~out = sym2 + 5\n```\n\n## Gates to R1CS\nNow, we convert this into something called a rank-1 constraint system (R1CS). An R1CS is a sequence of groups of three vectors `(a, b, c)`, and the solution to an R1CS is a vector s, where s must satisfy the equation `s . a * s . b - s . c = 0`, where `.` represents the dot product. For example, this is a satisfied R1CS:\n![r1cs](/images/cryptography/zkp/r1cs.png)\n\\\\[s \\cdot a = (1,3,35,9,27,30) \\cdot (5,0,0,0,0,1) = 35\\\\]\n\\\\[s \\cdot b = (1,3,35,9,27,30) \\cdot (1,0,0,0,0,0) = 1\\\\]\n\\\\[s \\cdot c = (1,3,35,9,27,30) \\cdot (0,0,1,0,0,0) = 35\\\\]\nHence \n\\\\[ s \\cdot a * s \\cdot b - s \\cdot c = 35 * 1 - 35 = 0\\\\]\n\nBut instead of having just one constraint, we are going to have many constraints: one for each logic gate. There is a standard way of converting a logic gate into a `(a, b, c)` triple depending on what the operation is `(+, -, * or /)` and whether the arguments are variables or numbers. The length of each vector is equal to the total number of variables in the system, including a dummy variable `~one` at the first index representing the number 1, the input variables `x`, a dummy variable `~out` representing the output, and then all of the intermediate variables (`sym1` and `sym2` above);\nFirst, we’ll provide the variable mapping that we’ll use:\n`'~one', 'x', '~out', 'sym1', 'y', 'sym2'`\n\nNow, we’ll give the (a, b, c) triple for the first gate:\n```\na = [0, 1, 0, 0, 0, 0]\nb = [0, 1, 0, 0, 0, 0]\nc = [0, 0, 0, 1, 0, 0]\n```\nwhich is \\\\(x*x -sym1 = 0\\\\)\n\nNow, let’s go on to the second gate:\n```\na = [0, 0, 0, 1, 0, 0]\nb = [0, 1, 0, 0, 0, 0]\nc = [0, 0, 0, 0, 1, 0]\n```\nwhich is \\\\(sym1 * x = y\\\\)\nNow, the third gate:\n```\na = [0, 1, 0, 0, 1, 0]\nb = [1, 0, 0, 0, 0, 0]\nc = [0, 0, 0, 0, 0, 1]\n```\nwhich is \\\\( (x+y) * \\sim one = sym2\\\\)\n\nFinally, the fourth gate:\n```\na = [5, 0, 0, 0, 0, 1]\nb = [1, 0, 0, 0, 0, 0]\nc = [0, 0, 1, 0, 0, 0]\n```\nwhich is \\\\((5 + sym2) * \\sim one = \\sim out\\\\)\nAnd there we have our R1CS with four constraints. The witness is simply the assignment to all the variables, including input, output and internal variables:\n`[1, 3, 35, 9, 27, 30]`\n\n\nThe complete R1CS put together is:\n```\nA\n[0, 1, 0, 0, 0, 0]\n[0, 0, 0, 1, 0, 0]\n[0, 1, 0, 0, 1, 0]\n[5, 0, 0, 0, 0, 1]\nB\n[0, 1, 0, 0, 0, 0]\n[0, 1, 0, 0, 0, 0]\n[1, 0, 0, 0, 0, 0]\n[1, 0, 0, 0, 0, 0]\nC\n[0, 0, 0, 1, 0, 0]\n[0, 0, 0, 0, 1, 0]\n[0, 0, 0, 0, 0, 1]\n[0, 0, 1, 0, 0, 0]\n```\n\n## R1CS to QAP\nThe next step is taking this R1CS and converting it into QAP form, which implements the exact same logic except using polynomials instead of dot products. We do this as follows. We go from four groups of three vectors of length six to six groups of three degree-3 polynomials, where evaluating the polynomials at each x coordinate represents one of the constraints. That is, if we evaluate the polynomials at x=1, then we get our first set of vectors, if we evaluate the polynomials at x=2, then we get our second set of vectors, and so on.\n\nWe can make this transformation using something called a **Lagrange interpolation**. \n![lagrange interpolating](/images/cryptography/zkp/lagrange_interpolating.png)\n\nNow, let’s use Lagrange interpolation to transform our R1CS. What we are going to do is take the first value out of every a vector, use Lagrange interpolation to make a polynomial out of that (where evaluating the polynomial at i gets you the first value of the ith a vector), repeat the process for the first value of every b and c vector, and then repeat that process for the second values, the third, values, and so on. For convenience I'll provide the answers right now:\n\n> Note: the intuition here is to think the R1CS A,B,C matrix vertically (column). For example, for the first column of A, the polynomial should pass (1,0), (2,0), (3,0), (4,5); the second polynomial shoudd pass (1,1), (2,0), (3,1), (4,)\n```\nA polynomials\n[-5.0, 9.166, -5.0, 0.833]\n[8.0, -11.333, 5.0, -0.666]\n[0.0, 0.0, 0.0, 0.0]\n[-6.0, 9.5, -4.0, 0.5]\n[4.0, -7.0, 3.5, -0.5]\n[-1.0, 1.833, -1.0, 0.166]\nB polynomials\n[3.0, -5.166, 2.5, -0.333]\n[-2.0, 5.166, -2.5, 0.333]\n[0.0, 0.0, 0.0, 0.0]\n[0.0, 0.0, 0.0, 0.0]\n[0.0, 0.0, 0.0, 0.0]\n[0.0, 0.0, 0.0, 0.0]\nC polynomials\n[0.0, 0.0, 0.0, 0.0]\n[0.0, 0.0, 0.0, 0.0]\n[-1.0, 1.833, -1.0, 0.166]\n[4.0, -4.333, 1.5, -0.166]\n[-6.0, 9.5, -4.0, 0.5]\n[4.0, -7.0, 3.5, -0.5]\n```\nCoefficients are in ascending order, so the first polynomial above is actually \\\\(0.833 x^3 — 5 x^2 + 9.166 x - 5\\\\)\nLet’s try evaluating all of these polynomials at x=1. \n```\nA results at x=1\n0 = 0.833 * 1^3 - 5 * 1^2 + 9.166 * 1^1 -5 * 1^0 = 0.833 -5 + 9.166 -5 = 0\n1 = -0.666 * 1^3 - 5.0 * 1^2 + -11.333 * 1^1 + 8.0 * 1^0 = -0.666 + 5.0 -11.333 +8.0 = 1\n0\n0\n0\n0\nB results at x=1\n0 = -0.333 * 1^3 +2.5 * 1^2 -5.166 * 1^1 +3.0 * 1^0 = -0.333 +2.5 -5.166 +3.0 = 0\n1\n0\n0\n0\n0\nC results at x=1\n0\n0\n0\n1\n0\n0\n```\n\n## Checking the QAP\nNow what’s the point of this crazy transformation? The answer is that instead of checking the constraints in the R1CS individually, we can now check all of the constraints at the same time by doing the dot product check on the polynomials.\n![checking qap](/images/cryptography/zkp/checking_qap.png)\nBecause in this case the dot product check is a series of additions and multiplications of polynomials, the result is itself going to be a polynomial. If the resulting polynomial, evaluated at every x coordinate that we used above to represent a logic gate, is equal to zero, then that means that all of the checks pass; if the resulting polynomial evaluated at at least one of the x coordinate representing a logic gate gives a nonzero value, then that means that the values going into and out of that logic gate are inconsistent\n\nTo check correctness, we don’t actually evaluate the polynomial `t = A . s * B . s - C . s` at every point corresponding to a gate; instead, we divide `t` by another polynomial, `Z`, and check that `Z` evenly divides `t` - that is, **the division `t / Z` leaves no remainder**.\n\n`Z` is defined as `(x - 1) * (x - 2) * (x - 3) ...` - the simplest polynomial that is equal to zero at all points that correspond to logic gates. It is an elementary fact of algebra that any polynomial that is equal to zero at all of these points has to be a multiple of this minimal polynomial, and if a polynomial is a multiple of `Z` then its evaluation at any of those points will be zero;\n\nNote that the above is a simplification; “in the real world”, the addition, multiplication, subtraction and division will happen not with regular numbers, but rather with finite field elements — a spooky kind of arithmetic which is self-consistent, so all the algebraic laws we know and love still hold true, but where all answers are elements of some finite-sized set\n\n## KEA (Knowledge of Exponent Assumption)\nLet \\\\(q\\\\) be a prime such that \\\\(2q+1\\\\) is also prime, and let \\\\(g\\\\) be a generator\nof the order \\\\(q\\\\) subgroup of \\\\(Z_{2q+1}^{\\ast}\\\\). Suppose we are given input \\\\(q, g, g^a\\\\) and want to output a pair \\\\((C, Y )\\\\) such that \\\\(Y = C^a\\\\). One way to do this is to pick some \\\\(c \\in Z_{q}\\\\), let \\\\(C = g^c\\\\), and let \\\\(Y = (g^a)^c\\\\). Intuitively, `KEA1`` can be viewed as saying that this is the 'only' way to produce such a pair. The assumption captures this by saying that any adversary outputting such a pair must “know” an exponent \\\\(c\\\\) such that \\\\(g^c = C\\\\). The formalization asks that there be an “extractor” that can return \\\\(c\\\\). Roughly:\n***\n**KEA1**\nFor any adversary \\\\(A\\\\) that takes input \\\\(q, g,g^a\\\\) and returns \\\\((C,Y)\\\\) with \\\\(Y = C^a\\\\), there exists an 'extractor' \\\\(\\bar{A}\\\\), which given the same inputs as \\\\(A\\\\) returns \\\\(c\\\\) such that \\\\(g^c = C\\\\)\n***\n## reference\n- [vitalik's blog: qap zero to hero](https://medium.com/@VitalikButerin/quadratic-arithmetic-programs-from-zero-to-hero-f6d558cea649)\n- [lagrange interpolating](https://mathworld.wolfram.com/LagrangeInterpolatingPolynomial.html)\n- [zkSNARKs in a nutshell](https://blog.ethereum.org/2016/12/05/zksnarks-in-a-nutshell)\n- [Pinocchio protocol by Parno, Gentry, Howell](https://eprint.iacr.org/2013/279.pdf)\n- [KEA](https://eprint.iacr.org/2004/008.pdf)","source":"_posts/cryptography/zkp/zkp-a-brief-understanding.md","raw":"---\ntitle: zkp a brief understanding (1)\ndate: 2023-06-27 14:29:26\ntags: [cryptography,zkp]\n---\n\n\n\n## introduction\nzk-SNARKs cannot be applied to any computational problem directly; rather, you have to convert the problem into the right “form” for the problem to operate on. The form is called a “quadratic arithmetic program” (QAP), and transforming the code of a function into one of these is itself highly nontrivial.\n\nThe example that we will choose is a simple one: proving that you know the solution to a cubic equation: \\\\(x^3 + x + 5 == 35\\\\)\n\nNote that modulo (%) and comparison operators (<, >, ≤, ≥) are NOT supported, as there is no efficient way to do modulo or comparison directly in finite cyclic group arithmetic (be thankful for this; if there was a way to do either one, then elliptic curve cryptography would be broken faster)\n\nYou can extend the language to modulo and comparisons by providing bit decompositions (eg. \\\\(13 = 2^3 + 2^2 + 1\\\\)) as auxiliary inputs, proving correctness of those decompositions and doing the math in binary circuits; in finite field arithmetic, doing equality `(==)` checks is also doable and in fact a bit easier, but these are both details we won’t get into right now. We can extend the language to support conditionals (eg. `if x < 5: y = 7; else: y = 9`) by converting them to an arithmetic form: `y = 7 * (x < 5) + 9 * (x >= 5)`;though note that both “paths” of the conditional would need to be executed.\n\n## Flattening\nThe first step is a “flattening” procedure, where we convert the original code into a sequence of statements that are of two forms\n```\nsym1 = x * x\ny = sym1 * x\nsym2 = y + x\n~out = sym2 + 5\n```\n\n## Gates to R1CS\nNow, we convert this into something called a rank-1 constraint system (R1CS). An R1CS is a sequence of groups of three vectors `(a, b, c)`, and the solution to an R1CS is a vector s, where s must satisfy the equation `s . a * s . b - s . c = 0`, where `.` represents the dot product. For example, this is a satisfied R1CS:\n![r1cs](/images/cryptography/zkp/r1cs.png)\n\\\\[s \\cdot a = (1,3,35,9,27,30) \\cdot (5,0,0,0,0,1) = 35\\\\]\n\\\\[s \\cdot b = (1,3,35,9,27,30) \\cdot (1,0,0,0,0,0) = 1\\\\]\n\\\\[s \\cdot c = (1,3,35,9,27,30) \\cdot (0,0,1,0,0,0) = 35\\\\]\nHence \n\\\\[ s \\cdot a * s \\cdot b - s \\cdot c = 35 * 1 - 35 = 0\\\\]\n\nBut instead of having just one constraint, we are going to have many constraints: one for each logic gate. There is a standard way of converting a logic gate into a `(a, b, c)` triple depending on what the operation is `(+, -, * or /)` and whether the arguments are variables or numbers. The length of each vector is equal to the total number of variables in the system, including a dummy variable `~one` at the first index representing the number 1, the input variables `x`, a dummy variable `~out` representing the output, and then all of the intermediate variables (`sym1` and `sym2` above);\nFirst, we’ll provide the variable mapping that we’ll use:\n`'~one', 'x', '~out', 'sym1', 'y', 'sym2'`\n\nNow, we’ll give the (a, b, c) triple for the first gate:\n```\na = [0, 1, 0, 0, 0, 0]\nb = [0, 1, 0, 0, 0, 0]\nc = [0, 0, 0, 1, 0, 0]\n```\nwhich is \\\\(x*x -sym1 = 0\\\\)\n\nNow, let’s go on to the second gate:\n```\na = [0, 0, 0, 1, 0, 0]\nb = [0, 1, 0, 0, 0, 0]\nc = [0, 0, 0, 0, 1, 0]\n```\nwhich is \\\\(sym1 * x = y\\\\)\nNow, the third gate:\n```\na = [0, 1, 0, 0, 1, 0]\nb = [1, 0, 0, 0, 0, 0]\nc = [0, 0, 0, 0, 0, 1]\n```\nwhich is \\\\( (x+y) * \\sim one = sym2\\\\)\n\nFinally, the fourth gate:\n```\na = [5, 0, 0, 0, 0, 1]\nb = [1, 0, 0, 0, 0, 0]\nc = [0, 0, 1, 0, 0, 0]\n```\nwhich is \\\\((5 + sym2) * \\sim one = \\sim out\\\\)\nAnd there we have our R1CS with four constraints. The witness is simply the assignment to all the variables, including input, output and internal variables:\n`[1, 3, 35, 9, 27, 30]`\n\n\nThe complete R1CS put together is:\n```\nA\n[0, 1, 0, 0, 0, 0]\n[0, 0, 0, 1, 0, 0]\n[0, 1, 0, 0, 1, 0]\n[5, 0, 0, 0, 0, 1]\nB\n[0, 1, 0, 0, 0, 0]\n[0, 1, 0, 0, 0, 0]\n[1, 0, 0, 0, 0, 0]\n[1, 0, 0, 0, 0, 0]\nC\n[0, 0, 0, 1, 0, 0]\n[0, 0, 0, 0, 1, 0]\n[0, 0, 0, 0, 0, 1]\n[0, 0, 1, 0, 0, 0]\n```\n\n## R1CS to QAP\nThe next step is taking this R1CS and converting it into QAP form, which implements the exact same logic except using polynomials instead of dot products. We do this as follows. We go from four groups of three vectors of length six to six groups of three degree-3 polynomials, where evaluating the polynomials at each x coordinate represents one of the constraints. That is, if we evaluate the polynomials at x=1, then we get our first set of vectors, if we evaluate the polynomials at x=2, then we get our second set of vectors, and so on.\n\nWe can make this transformation using something called a **Lagrange interpolation**. \n![lagrange interpolating](/images/cryptography/zkp/lagrange_interpolating.png)\n\nNow, let’s use Lagrange interpolation to transform our R1CS. What we are going to do is take the first value out of every a vector, use Lagrange interpolation to make a polynomial out of that (where evaluating the polynomial at i gets you the first value of the ith a vector), repeat the process for the first value of every b and c vector, and then repeat that process for the second values, the third, values, and so on. For convenience I'll provide the answers right now:\n\n> Note: the intuition here is to think the R1CS A,B,C matrix vertically (column). For example, for the first column of A, the polynomial should pass (1,0), (2,0), (3,0), (4,5); the second polynomial shoudd pass (1,1), (2,0), (3,1), (4,)\n```\nA polynomials\n[-5.0, 9.166, -5.0, 0.833]\n[8.0, -11.333, 5.0, -0.666]\n[0.0, 0.0, 0.0, 0.0]\n[-6.0, 9.5, -4.0, 0.5]\n[4.0, -7.0, 3.5, -0.5]\n[-1.0, 1.833, -1.0, 0.166]\nB polynomials\n[3.0, -5.166, 2.5, -0.333]\n[-2.0, 5.166, -2.5, 0.333]\n[0.0, 0.0, 0.0, 0.0]\n[0.0, 0.0, 0.0, 0.0]\n[0.0, 0.0, 0.0, 0.0]\n[0.0, 0.0, 0.0, 0.0]\nC polynomials\n[0.0, 0.0, 0.0, 0.0]\n[0.0, 0.0, 0.0, 0.0]\n[-1.0, 1.833, -1.0, 0.166]\n[4.0, -4.333, 1.5, -0.166]\n[-6.0, 9.5, -4.0, 0.5]\n[4.0, -7.0, 3.5, -0.5]\n```\nCoefficients are in ascending order, so the first polynomial above is actually \\\\(0.833 x^3 — 5 x^2 + 9.166 x - 5\\\\)\nLet’s try evaluating all of these polynomials at x=1. \n```\nA results at x=1\n0 = 0.833 * 1^3 - 5 * 1^2 + 9.166 * 1^1 -5 * 1^0 = 0.833 -5 + 9.166 -5 = 0\n1 = -0.666 * 1^3 - 5.0 * 1^2 + -11.333 * 1^1 + 8.0 * 1^0 = -0.666 + 5.0 -11.333 +8.0 = 1\n0\n0\n0\n0\nB results at x=1\n0 = -0.333 * 1^3 +2.5 * 1^2 -5.166 * 1^1 +3.0 * 1^0 = -0.333 +2.5 -5.166 +3.0 = 0\n1\n0\n0\n0\n0\nC results at x=1\n0\n0\n0\n1\n0\n0\n```\n\n## Checking the QAP\nNow what’s the point of this crazy transformation? The answer is that instead of checking the constraints in the R1CS individually, we can now check all of the constraints at the same time by doing the dot product check on the polynomials.\n![checking qap](/images/cryptography/zkp/checking_qap.png)\nBecause in this case the dot product check is a series of additions and multiplications of polynomials, the result is itself going to be a polynomial. If the resulting polynomial, evaluated at every x coordinate that we used above to represent a logic gate, is equal to zero, then that means that all of the checks pass; if the resulting polynomial evaluated at at least one of the x coordinate representing a logic gate gives a nonzero value, then that means that the values going into and out of that logic gate are inconsistent\n\nTo check correctness, we don’t actually evaluate the polynomial `t = A . s * B . s - C . s` at every point corresponding to a gate; instead, we divide `t` by another polynomial, `Z`, and check that `Z` evenly divides `t` - that is, **the division `t / Z` leaves no remainder**.\n\n`Z` is defined as `(x - 1) * (x - 2) * (x - 3) ...` - the simplest polynomial that is equal to zero at all points that correspond to logic gates. It is an elementary fact of algebra that any polynomial that is equal to zero at all of these points has to be a multiple of this minimal polynomial, and if a polynomial is a multiple of `Z` then its evaluation at any of those points will be zero;\n\nNote that the above is a simplification; “in the real world”, the addition, multiplication, subtraction and division will happen not with regular numbers, but rather with finite field elements — a spooky kind of arithmetic which is self-consistent, so all the algebraic laws we know and love still hold true, but where all answers are elements of some finite-sized set\n\n## KEA (Knowledge of Exponent Assumption)\nLet \\\\(q\\\\) be a prime such that \\\\(2q+1\\\\) is also prime, and let \\\\(g\\\\) be a generator\nof the order \\\\(q\\\\) subgroup of \\\\(Z_{2q+1}^{\\ast}\\\\). Suppose we are given input \\\\(q, g, g^a\\\\) and want to output a pair \\\\((C, Y )\\\\) such that \\\\(Y = C^a\\\\). One way to do this is to pick some \\\\(c \\in Z_{q}\\\\), let \\\\(C = g^c\\\\), and let \\\\(Y = (g^a)^c\\\\). Intuitively, `KEA1`` can be viewed as saying that this is the 'only' way to produce such a pair. The assumption captures this by saying that any adversary outputting such a pair must “know” an exponent \\\\(c\\\\) such that \\\\(g^c = C\\\\). The formalization asks that there be an “extractor” that can return \\\\(c\\\\). Roughly:\n***\n**KEA1**\nFor any adversary \\\\(A\\\\) that takes input \\\\(q, g,g^a\\\\) and returns \\\\((C,Y)\\\\) with \\\\(Y = C^a\\\\), there exists an 'extractor' \\\\(\\bar{A}\\\\), which given the same inputs as \\\\(A\\\\) returns \\\\(c\\\\) such that \\\\(g^c = C\\\\)\n***\n## reference\n- [vitalik's blog: qap zero to hero](https://medium.com/@VitalikButerin/quadratic-arithmetic-programs-from-zero-to-hero-f6d558cea649)\n- [lagrange interpolating](https://mathworld.wolfram.com/LagrangeInterpolatingPolynomial.html)\n- [zkSNARKs in a nutshell](https://blog.ethereum.org/2016/12/05/zksnarks-in-a-nutshell)\n- [Pinocchio protocol by Parno, Gentry, Howell](https://eprint.iacr.org/2013/279.pdf)\n- [KEA](https://eprint.iacr.org/2004/008.pdf)","slug":"cryptography/zkp/zkp-a-brief-understanding","published":1,"updated":"2023-11-05T04:21:17.034Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8e1003aqwsj8bnledgy","content":"\n\n\n

introduction

zk-SNARKs cannot be applied to any computational problem directly; rather, you have to convert the problem into the right “form” for the problem to operate on. The form is called a “quadratic arithmetic program” (QAP), and transforming the code of a function into one of these is itself highly nontrivial.

\n

The example that we will choose is a simple one: proving that you know the solution to a cubic equation: \\(x^3 + x + 5 == 35\\)

\n

Note that modulo (%) and comparison operators (<, >, ≤, ≥) are NOT supported, as there is no efficient way to do modulo or comparison directly in finite cyclic group arithmetic (be thankful for this; if there was a way to do either one, then elliptic curve cryptography would be broken faster)

\n

You can extend the language to modulo and comparisons by providing bit decompositions (eg. \\(13 = 2^3 + 2^2 + 1\\)) as auxiliary inputs, proving correctness of those decompositions and doing the math in binary circuits; in finite field arithmetic, doing equality (==) checks is also doable and in fact a bit easier, but these are both details we won’t get into right now. We can extend the language to support conditionals (eg. if x < 5: y = 7; else: y = 9) by converting them to an arithmetic form: y = 7 * (x < 5) + 9 * (x >= 5);though note that both “paths” of the conditional would need to be executed.

\n

Flattening

The first step is a “flattening” procedure, where we convert the original code into a sequence of statements that are of two forms

\n
1
2
3
4
sym1 = x * x
y = sym1 * x
sym2 = y + x
~out = sym2 + 5
\n\n

Gates to R1CS

Now, we convert this into something called a rank-1 constraint system (R1CS). An R1CS is a sequence of groups of three vectors (a, b, c), and the solution to an R1CS is a vector s, where s must satisfy the equation s . a * s . b - s . c = 0, where . represents the dot product. For example, this is a satisfied R1CS:
\"r1cs\"
\\[s \\cdot a = (1,3,35,9,27,30) \\cdot (5,0,0,0,0,1) = 35\\]
\\[s \\cdot b = (1,3,35,9,27,30) \\cdot (1,0,0,0,0,0) = 1\\]
\\[s \\cdot c = (1,3,35,9,27,30) \\cdot (0,0,1,0,0,0) = 35\\]
Hence
\\[ s \\cdot a * s \\cdot b - s \\cdot c = 35 * 1 - 35 = 0\\]

\n

But instead of having just one constraint, we are going to have many constraints: one for each logic gate. There is a standard way of converting a logic gate into a (a, b, c) triple depending on what the operation is (+, -, * or /) and whether the arguments are variables or numbers. The length of each vector is equal to the total number of variables in the system, including a dummy variable ~one at the first index representing the number 1, the input variables x, a dummy variable ~out representing the output, and then all of the intermediate variables (sym1 and sym2 above);
First, we’ll provide the variable mapping that we’ll use:
'~one', 'x', '~out', 'sym1', 'y', 'sym2'

\n

Now, we’ll give the (a, b, c) triple for the first gate:

\n
1
2
3
a = [0, 1, 0, 0, 0, 0]
b = [0, 1, 0, 0, 0, 0]
c = [0, 0, 0, 1, 0, 0]
\n

which is \\(x*x -sym1 = 0\\)

\n

Now, let’s go on to the second gate:

\n
1
2
3
a = [0, 0, 0, 1, 0, 0]
b = [0, 1, 0, 0, 0, 0]
c = [0, 0, 0, 0, 1, 0]
\n

which is \\(sym1 * x = y\\)
Now, the third gate:

\n
1
2
3
a = [0, 1, 0, 0, 1, 0]
b = [1, 0, 0, 0, 0, 0]
c = [0, 0, 0, 0, 0, 1]
\n

which is \\( (x+y) * \\sim one = sym2\\)

\n

Finally, the fourth gate:

\n
1
2
3
a = [5, 0, 0, 0, 0, 1]
b = [1, 0, 0, 0, 0, 0]
c = [0, 0, 1, 0, 0, 0]
\n

which is \\((5 + sym2) * \\sim one = \\sim out\\)
And there we have our R1CS with four constraints. The witness is simply the assignment to all the variables, including input, output and internal variables:
[1, 3, 35, 9, 27, 30]

\n

The complete R1CS put together is:

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
A
[0, 1, 0, 0, 0, 0]
[0, 0, 0, 1, 0, 0]
[0, 1, 0, 0, 1, 0]
[5, 0, 0, 0, 0, 1]
B
[0, 1, 0, 0, 0, 0]
[0, 1, 0, 0, 0, 0]
[1, 0, 0, 0, 0, 0]
[1, 0, 0, 0, 0, 0]
C
[0, 0, 0, 1, 0, 0]
[0, 0, 0, 0, 1, 0]
[0, 0, 0, 0, 0, 1]
[0, 0, 1, 0, 0, 0]
\n\n

R1CS to QAP

The next step is taking this R1CS and converting it into QAP form, which implements the exact same logic except using polynomials instead of dot products. We do this as follows. We go from four groups of three vectors of length six to six groups of three degree-3 polynomials, where evaluating the polynomials at each x coordinate represents one of the constraints. That is, if we evaluate the polynomials at x=1, then we get our first set of vectors, if we evaluate the polynomials at x=2, then we get our second set of vectors, and so on.

\n

We can make this transformation using something called a Lagrange interpolation.
\"lagrange

\n

Now, let’s use Lagrange interpolation to transform our R1CS. What we are going to do is take the first value out of every a vector, use Lagrange interpolation to make a polynomial out of that (where evaluating the polynomial at i gets you the first value of the ith a vector), repeat the process for the first value of every b and c vector, and then repeat that process for the second values, the third, values, and so on. For convenience I’ll provide the answers right now:

\n
\n

Note: the intuition here is to think the R1CS A,B,C matrix vertically (column). For example, for the first column of A, the polynomial should pass (1,0), (2,0), (3,0), (4,5); the second polynomial shoudd pass (1,1), (2,0), (3,1), (4,)

\n
\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
A polynomials
[-5.0, 9.166, -5.0, 0.833]
[8.0, -11.333, 5.0, -0.666]
[0.0, 0.0, 0.0, 0.0]
[-6.0, 9.5, -4.0, 0.5]
[4.0, -7.0, 3.5, -0.5]
[-1.0, 1.833, -1.0, 0.166]
B polynomials
[3.0, -5.166, 2.5, -0.333]
[-2.0, 5.166, -2.5, 0.333]
[0.0, 0.0, 0.0, 0.0]
[0.0, 0.0, 0.0, 0.0]
[0.0, 0.0, 0.0, 0.0]
[0.0, 0.0, 0.0, 0.0]
C polynomials
[0.0, 0.0, 0.0, 0.0]
[0.0, 0.0, 0.0, 0.0]
[-1.0, 1.833, -1.0, 0.166]
[4.0, -4.333, 1.5, -0.166]
[-6.0, 9.5, -4.0, 0.5]
[4.0, -7.0, 3.5, -0.5]
\n

Coefficients are in ascending order, so the first polynomial above is actually \\(0.833 x^3 — 5 x^2 + 9.166 x - 5\\)
Let’s try evaluating all of these polynomials at x=1.

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
A results at x=1
0 = 0.833 * 1^3 - 5 * 1^2 + 9.166 * 1^1 -5 * 1^0 = 0.833 -5 + 9.166 -5 = 0
1 = -0.666 * 1^3 - 5.0 * 1^2 + -11.333 * 1^1 + 8.0 * 1^0 = -0.666 + 5.0 -11.333 +8.0 = 1
0
0
0
0
B results at x=1
0 = -0.333 * 1^3 +2.5 * 1^2 -5.166 * 1^1 +3.0 * 1^0 = -0.333 +2.5 -5.166 +3.0 = 0
1
0
0
0
0
C results at x=1
0
0
0
1
0
0
\n\n

Checking the QAP

Now what’s the point of this crazy transformation? The answer is that instead of checking the constraints in the R1CS individually, we can now check all of the constraints at the same time by doing the dot product check on the polynomials.
\"checking
Because in this case the dot product check is a series of additions and multiplications of polynomials, the result is itself going to be a polynomial. If the resulting polynomial, evaluated at every x coordinate that we used above to represent a logic gate, is equal to zero, then that means that all of the checks pass; if the resulting polynomial evaluated at at least one of the x coordinate representing a logic gate gives a nonzero value, then that means that the values going into and out of that logic gate are inconsistent

\n

To check correctness, we don’t actually evaluate the polynomial t = A . s * B . s - C . s at every point corresponding to a gate; instead, we divide t by another polynomial, Z, and check that Z evenly divides t - that is, the division t / Z leaves no remainder.

\n

Z is defined as (x - 1) * (x - 2) * (x - 3) ... - the simplest polynomial that is equal to zero at all points that correspond to logic gates. It is an elementary fact of algebra that any polynomial that is equal to zero at all of these points has to be a multiple of this minimal polynomial, and if a polynomial is a multiple of Z then its evaluation at any of those points will be zero;

\n

Note that the above is a simplification; “in the real world”, the addition, multiplication, subtraction and division will happen not with regular numbers, but rather with finite field elements — a spooky kind of arithmetic which is self-consistent, so all the algebraic laws we know and love still hold true, but where all answers are elements of some finite-sized set

\n

KEA (Knowledge of Exponent Assumption)

Let \\(q\\) be a prime such that \\(2q+1\\) is also prime, and let \\(g\\) be a generator
of the order \\(q\\) subgroup of \\(Z_{2q+1}^{\\ast}\\). Suppose we are given input \\(q, g, g^a\\) and want to output a pair \\((C, Y )\\) such that \\(Y = C^a\\). One way to do this is to pick some \\(c \\in Z_{q}\\), let \\(C = g^c\\), and let \\(Y = (g^a)^c\\). Intuitively, `KEA1`` can be viewed as saying that this is the ‘only’ way to produce such a pair. The assumption captures this by saying that any adversary outputting such a pair must “know” an exponent \\(c\\) such that \\(g^c = C\\). The formalization asks that there be an “extractor” that can return \\(c\\). Roughly:

\n
\n

KEA1
For any adversary \\(A\\) that takes input \\(q, g,g^a\\) and returns \\((C,Y)\\) with \\(Y = C^a\\), there exists an ‘extractor’ \\(\\bar{A}\\), which given the same inputs as \\(A\\) returns \\(c\\) such that \\(g^c = C\\)

\n
\n

reference

\n","site":{"data":{}},"excerpt":"","more":"\n\n\n

introduction

zk-SNARKs cannot be applied to any computational problem directly; rather, you have to convert the problem into the right “form” for the problem to operate on. The form is called a “quadratic arithmetic program” (QAP), and transforming the code of a function into one of these is itself highly nontrivial.

\n

The example that we will choose is a simple one: proving that you know the solution to a cubic equation: \\(x^3 + x + 5 == 35\\)

\n

Note that modulo (%) and comparison operators (<, >, ≤, ≥) are NOT supported, as there is no efficient way to do modulo or comparison directly in finite cyclic group arithmetic (be thankful for this; if there was a way to do either one, then elliptic curve cryptography would be broken faster)

\n

You can extend the language to modulo and comparisons by providing bit decompositions (eg. \\(13 = 2^3 + 2^2 + 1\\)) as auxiliary inputs, proving correctness of those decompositions and doing the math in binary circuits; in finite field arithmetic, doing equality (==) checks is also doable and in fact a bit easier, but these are both details we won’t get into right now. We can extend the language to support conditionals (eg. if x < 5: y = 7; else: y = 9) by converting them to an arithmetic form: y = 7 * (x < 5) + 9 * (x >= 5);though note that both “paths” of the conditional would need to be executed.

\n

Flattening

The first step is a “flattening” procedure, where we convert the original code into a sequence of statements that are of two forms

\n
1
2
3
4
sym1 = x * x
y = sym1 * x
sym2 = y + x
~out = sym2 + 5
\n\n

Gates to R1CS

Now, we convert this into something called a rank-1 constraint system (R1CS). An R1CS is a sequence of groups of three vectors (a, b, c), and the solution to an R1CS is a vector s, where s must satisfy the equation s . a * s . b - s . c = 0, where . represents the dot product. For example, this is a satisfied R1CS:
\"r1cs\"
\\[s \\cdot a = (1,3,35,9,27,30) \\cdot (5,0,0,0,0,1) = 35\\]
\\[s \\cdot b = (1,3,35,9,27,30) \\cdot (1,0,0,0,0,0) = 1\\]
\\[s \\cdot c = (1,3,35,9,27,30) \\cdot (0,0,1,0,0,0) = 35\\]
Hence
\\[ s \\cdot a * s \\cdot b - s \\cdot c = 35 * 1 - 35 = 0\\]

\n

But instead of having just one constraint, we are going to have many constraints: one for each logic gate. There is a standard way of converting a logic gate into a (a, b, c) triple depending on what the operation is (+, -, * or /) and whether the arguments are variables or numbers. The length of each vector is equal to the total number of variables in the system, including a dummy variable ~one at the first index representing the number 1, the input variables x, a dummy variable ~out representing the output, and then all of the intermediate variables (sym1 and sym2 above);
First, we’ll provide the variable mapping that we’ll use:
'~one', 'x', '~out', 'sym1', 'y', 'sym2'

\n

Now, we’ll give the (a, b, c) triple for the first gate:

\n
1
2
3
a = [0, 1, 0, 0, 0, 0]
b = [0, 1, 0, 0, 0, 0]
c = [0, 0, 0, 1, 0, 0]
\n

which is \\(x*x -sym1 = 0\\)

\n

Now, let’s go on to the second gate:

\n
1
2
3
a = [0, 0, 0, 1, 0, 0]
b = [0, 1, 0, 0, 0, 0]
c = [0, 0, 0, 0, 1, 0]
\n

which is \\(sym1 * x = y\\)
Now, the third gate:

\n
1
2
3
a = [0, 1, 0, 0, 1, 0]
b = [1, 0, 0, 0, 0, 0]
c = [0, 0, 0, 0, 0, 1]
\n

which is \\( (x+y) * \\sim one = sym2\\)

\n

Finally, the fourth gate:

\n
1
2
3
a = [5, 0, 0, 0, 0, 1]
b = [1, 0, 0, 0, 0, 0]
c = [0, 0, 1, 0, 0, 0]
\n

which is \\((5 + sym2) * \\sim one = \\sim out\\)
And there we have our R1CS with four constraints. The witness is simply the assignment to all the variables, including input, output and internal variables:
[1, 3, 35, 9, 27, 30]

\n

The complete R1CS put together is:

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
A
[0, 1, 0, 0, 0, 0]
[0, 0, 0, 1, 0, 0]
[0, 1, 0, 0, 1, 0]
[5, 0, 0, 0, 0, 1]
B
[0, 1, 0, 0, 0, 0]
[0, 1, 0, 0, 0, 0]
[1, 0, 0, 0, 0, 0]
[1, 0, 0, 0, 0, 0]
C
[0, 0, 0, 1, 0, 0]
[0, 0, 0, 0, 1, 0]
[0, 0, 0, 0, 0, 1]
[0, 0, 1, 0, 0, 0]
\n\n

R1CS to QAP

The next step is taking this R1CS and converting it into QAP form, which implements the exact same logic except using polynomials instead of dot products. We do this as follows. We go from four groups of three vectors of length six to six groups of three degree-3 polynomials, where evaluating the polynomials at each x coordinate represents one of the constraints. That is, if we evaluate the polynomials at x=1, then we get our first set of vectors, if we evaluate the polynomials at x=2, then we get our second set of vectors, and so on.

\n

We can make this transformation using something called a Lagrange interpolation.
\"lagrange

\n

Now, let’s use Lagrange interpolation to transform our R1CS. What we are going to do is take the first value out of every a vector, use Lagrange interpolation to make a polynomial out of that (where evaluating the polynomial at i gets you the first value of the ith a vector), repeat the process for the first value of every b and c vector, and then repeat that process for the second values, the third, values, and so on. For convenience I’ll provide the answers right now:

\n
\n

Note: the intuition here is to think the R1CS A,B,C matrix vertically (column). For example, for the first column of A, the polynomial should pass (1,0), (2,0), (3,0), (4,5); the second polynomial shoudd pass (1,1), (2,0), (3,1), (4,)

\n
\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
A polynomials
[-5.0, 9.166, -5.0, 0.833]
[8.0, -11.333, 5.0, -0.666]
[0.0, 0.0, 0.0, 0.0]
[-6.0, 9.5, -4.0, 0.5]
[4.0, -7.0, 3.5, -0.5]
[-1.0, 1.833, -1.0, 0.166]
B polynomials
[3.0, -5.166, 2.5, -0.333]
[-2.0, 5.166, -2.5, 0.333]
[0.0, 0.0, 0.0, 0.0]
[0.0, 0.0, 0.0, 0.0]
[0.0, 0.0, 0.0, 0.0]
[0.0, 0.0, 0.0, 0.0]
C polynomials
[0.0, 0.0, 0.0, 0.0]
[0.0, 0.0, 0.0, 0.0]
[-1.0, 1.833, -1.0, 0.166]
[4.0, -4.333, 1.5, -0.166]
[-6.0, 9.5, -4.0, 0.5]
[4.0, -7.0, 3.5, -0.5]
\n

Coefficients are in ascending order, so the first polynomial above is actually \\(0.833 x^3 — 5 x^2 + 9.166 x - 5\\)
Let’s try evaluating all of these polynomials at x=1.

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
A results at x=1
0 = 0.833 * 1^3 - 5 * 1^2 + 9.166 * 1^1 -5 * 1^0 = 0.833 -5 + 9.166 -5 = 0
1 = -0.666 * 1^3 - 5.0 * 1^2 + -11.333 * 1^1 + 8.0 * 1^0 = -0.666 + 5.0 -11.333 +8.0 = 1
0
0
0
0
B results at x=1
0 = -0.333 * 1^3 +2.5 * 1^2 -5.166 * 1^1 +3.0 * 1^0 = -0.333 +2.5 -5.166 +3.0 = 0
1
0
0
0
0
C results at x=1
0
0
0
1
0
0
\n\n

Checking the QAP

Now what’s the point of this crazy transformation? The answer is that instead of checking the constraints in the R1CS individually, we can now check all of the constraints at the same time by doing the dot product check on the polynomials.
\"checking
Because in this case the dot product check is a series of additions and multiplications of polynomials, the result is itself going to be a polynomial. If the resulting polynomial, evaluated at every x coordinate that we used above to represent a logic gate, is equal to zero, then that means that all of the checks pass; if the resulting polynomial evaluated at at least one of the x coordinate representing a logic gate gives a nonzero value, then that means that the values going into and out of that logic gate are inconsistent

\n

To check correctness, we don’t actually evaluate the polynomial t = A . s * B . s - C . s at every point corresponding to a gate; instead, we divide t by another polynomial, Z, and check that Z evenly divides t - that is, the division t / Z leaves no remainder.

\n

Z is defined as (x - 1) * (x - 2) * (x - 3) ... - the simplest polynomial that is equal to zero at all points that correspond to logic gates. It is an elementary fact of algebra that any polynomial that is equal to zero at all of these points has to be a multiple of this minimal polynomial, and if a polynomial is a multiple of Z then its evaluation at any of those points will be zero;

\n

Note that the above is a simplification; “in the real world”, the addition, multiplication, subtraction and division will happen not with regular numbers, but rather with finite field elements — a spooky kind of arithmetic which is self-consistent, so all the algebraic laws we know and love still hold true, but where all answers are elements of some finite-sized set

\n

KEA (Knowledge of Exponent Assumption)

Let \\(q\\) be a prime such that \\(2q+1\\) is also prime, and let \\(g\\) be a generator
of the order \\(q\\) subgroup of \\(Z_{2q+1}^{\\ast}\\). Suppose we are given input \\(q, g, g^a\\) and want to output a pair \\((C, Y )\\) such that \\(Y = C^a\\). One way to do this is to pick some \\(c \\in Z_{q}\\), let \\(C = g^c\\), and let \\(Y = (g^a)^c\\). Intuitively, `KEA1`` can be viewed as saying that this is the ‘only’ way to produce such a pair. The assumption captures this by saying that any adversary outputting such a pair must “know” an exponent \\(c\\) such that \\(g^c = C\\). The formalization asks that there be an “extractor” that can return \\(c\\). Roughly:

\n
\n

KEA1
For any adversary \\(A\\) that takes input \\(q, g,g^a\\) and returns \\((C,Y)\\) with \\(Y = C^a\\), there exists an ‘extractor’ \\(\\bar{A}\\), which given the same inputs as \\(A\\) returns \\(c\\) such that \\(g^c = C\\)

\n
\n

reference

\n"},{"title":"zkp groth16 paper review","date":"2023-07-07T06:29:26.000Z","_content":"\n\n## 1. Introduction\nGoldwasser, Micali and Rackoff [GMR89] introduced zero-knowledge proofs that enable a prover to convince a verifier that a statement is true without revealing anything else. They have three core properties:\n- **Completeness**: Given a statement and a witness, the prover can convince the verifier. \n- **Soundness**: A malicious prover cannot convince the verifier of a false statement. \n- **Zero-knowledge**: The proof does not reveal anything but the truth of the statement\n\n### 1.1. Contribution\n**Succinct NIZK** We construct a NIZK argument for arithmetic circuit satisfiability where a proof consists of only 3 group elements. In addition to being small, the proof is also easy to verify. The verifier just needs to compute a number of exponentiations proportional to the statement size and check a single pairing product equation, which only has 3 pairings.\n\n\n## 2. Preliminaries\n### 2.1 Bilinear Groups\nWe will work over bilinear groups \\\\((p, \\mathbb{G_{1}}, \\mathbb{G_{2}}, \\mathbb{G_{T}} , e, g, h)\\\\) with the following properties:\n- \\\\(\\mathbb{G_{1}}, \\mathbb{G_{2}}, \\mathbb{G_{T}} \\\\)are groups of prime order \\\\(p\\\\)\n- The pairing \\\\( e:\\mathbb{G_{1}} \\times \\mathbb{G_{2}} \\rightarrow \\mathbb{G_{T}}\\\\) is a bilinear map\n- \\\\(g\\\\) is a generator for \\\\(\\mathbb{G_{1}}\\\\), \\\\(h\\\\) is a generator for \\\\(\\mathbb{G_{2}}\\\\), and \\\\(e(g,h)\\\\) is a generator for \\\\(\\mathbb{G_{T}}\\\\)\n\nThere are many ways to set up bilinear groups both as symmetric bilinear groups where \\\\(\\mathbb{G_{1}} = \\mathbb{G_{1}}\\\\) and as asymmetric bilinear groups where \\\\(\\mathbb{G_{1}} \\neq \\mathbb{G_{1}}\\\\). Galbraith, Paterson and Smart [GPS08] classify bilinear groups as **Type I** where \\\\(\\mathbb{G_{1}} = \\mathbb{G_{2}}\\\\), **Type II** where there is an efficiently computable non-trivial homomorphism \\\\(\\Phi : \\mathbb{G_{2}} \\rightarrow \\mathbb{G_{1}}\\\\), and **Type III** where no such efficiently computable homomorphism exists in either direction between \\\\(\\mathbb{G_{1}}\\\\) and \\\\(\\mathbb{G_{2}}\\\\). Type III bilinear groups are the most efficient type of bilinear groups and hence the most relevant for practical applications.\nAs a notation for group elements, we write \\\\( \\lbrack a \\rbrack_{1} \\\\) for \\\\( g^a\\\\), \\\\( \\lbrack b \\rbrack_{2}\\\\) for \\\\( h^b\\\\) and \\\\( \\lbrack c \\rbrack_{T}\\\\) for \\\\(e(g,h)^{c} \\\\). A vector of group elements will be represented as \\\\( \\lbrack \\mathbf{a} \\rbrack_{i} \\\\). Given two vectors of \\\\(n\\\\) group elements \\\\( \\lbrack \\mathbf{a} \\rbrack_{1} \\\\) and \\\\( \\lbrack \\mathbf{b} \\rbrack_{2} \\\\), we define their dot product as \\\\( \\lbrack \\mathbf{a} \\rbrack_{1} \\cdot \\lbrack \\mathbf{b} \\rbrack_{2} = \\lbrack \\mathbf{a} \\cdot \\mathbf{b} \\rbrack_{T} \\\\), which can be efficiently computed using the pairing \\\\(e\\\\).\n\n\n### 2.2 Non-interactive zero-knowledge arguments of knowledge\n\n\n## references\n- [1] groth 16 paper, On the Size of Pairing-based Non-interactive Arguments by Jens Groth","source":"_posts/cryptography/zkp/zkp-groth16-paper-review.md","raw":"---\ntitle: zkp groth16 paper review\ndate: 2023-07-07 14:29:26\ntags: [cryptography,zkp]\n---\n\n\n## 1. Introduction\nGoldwasser, Micali and Rackoff [GMR89] introduced zero-knowledge proofs that enable a prover to convince a verifier that a statement is true without revealing anything else. They have three core properties:\n- **Completeness**: Given a statement and a witness, the prover can convince the verifier. \n- **Soundness**: A malicious prover cannot convince the verifier of a false statement. \n- **Zero-knowledge**: The proof does not reveal anything but the truth of the statement\n\n### 1.1. Contribution\n**Succinct NIZK** We construct a NIZK argument for arithmetic circuit satisfiability where a proof consists of only 3 group elements. In addition to being small, the proof is also easy to verify. The verifier just needs to compute a number of exponentiations proportional to the statement size and check a single pairing product equation, which only has 3 pairings.\n\n\n## 2. Preliminaries\n### 2.1 Bilinear Groups\nWe will work over bilinear groups \\\\((p, \\mathbb{G_{1}}, \\mathbb{G_{2}}, \\mathbb{G_{T}} , e, g, h)\\\\) with the following properties:\n- \\\\(\\mathbb{G_{1}}, \\mathbb{G_{2}}, \\mathbb{G_{T}} \\\\)are groups of prime order \\\\(p\\\\)\n- The pairing \\\\( e:\\mathbb{G_{1}} \\times \\mathbb{G_{2}} \\rightarrow \\mathbb{G_{T}}\\\\) is a bilinear map\n- \\\\(g\\\\) is a generator for \\\\(\\mathbb{G_{1}}\\\\), \\\\(h\\\\) is a generator for \\\\(\\mathbb{G_{2}}\\\\), and \\\\(e(g,h)\\\\) is a generator for \\\\(\\mathbb{G_{T}}\\\\)\n\nThere are many ways to set up bilinear groups both as symmetric bilinear groups where \\\\(\\mathbb{G_{1}} = \\mathbb{G_{1}}\\\\) and as asymmetric bilinear groups where \\\\(\\mathbb{G_{1}} \\neq \\mathbb{G_{1}}\\\\). Galbraith, Paterson and Smart [GPS08] classify bilinear groups as **Type I** where \\\\(\\mathbb{G_{1}} = \\mathbb{G_{2}}\\\\), **Type II** where there is an efficiently computable non-trivial homomorphism \\\\(\\Phi : \\mathbb{G_{2}} \\rightarrow \\mathbb{G_{1}}\\\\), and **Type III** where no such efficiently computable homomorphism exists in either direction between \\\\(\\mathbb{G_{1}}\\\\) and \\\\(\\mathbb{G_{2}}\\\\). Type III bilinear groups are the most efficient type of bilinear groups and hence the most relevant for practical applications.\nAs a notation for group elements, we write \\\\( \\lbrack a \\rbrack_{1} \\\\) for \\\\( g^a\\\\), \\\\( \\lbrack b \\rbrack_{2}\\\\) for \\\\( h^b\\\\) and \\\\( \\lbrack c \\rbrack_{T}\\\\) for \\\\(e(g,h)^{c} \\\\). A vector of group elements will be represented as \\\\( \\lbrack \\mathbf{a} \\rbrack_{i} \\\\). Given two vectors of \\\\(n\\\\) group elements \\\\( \\lbrack \\mathbf{a} \\rbrack_{1} \\\\) and \\\\( \\lbrack \\mathbf{b} \\rbrack_{2} \\\\), we define their dot product as \\\\( \\lbrack \\mathbf{a} \\rbrack_{1} \\cdot \\lbrack \\mathbf{b} \\rbrack_{2} = \\lbrack \\mathbf{a} \\cdot \\mathbf{b} \\rbrack_{T} \\\\), which can be efficiently computed using the pairing \\\\(e\\\\).\n\n\n### 2.2 Non-interactive zero-knowledge arguments of knowledge\n\n\n## references\n- [1] groth 16 paper, On the Size of Pairing-based Non-interactive Arguments by Jens Groth","slug":"cryptography/zkp/zkp-groth16-paper-review","published":1,"updated":"2023-11-05T04:21:17.035Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8e1003cqwsjc3p416e4","content":"\n\n

1. Introduction

Goldwasser, Micali and Rackoff [GMR89] introduced zero-knowledge proofs that enable a prover to convince a verifier that a statement is true without revealing anything else. They have three core properties:

\n
    \n
  • Completeness: Given a statement and a witness, the prover can convince the verifier.
    • \n
  • Soundness: A malicious prover cannot convince the verifier of a false statement.
    • \n
  • Zero-knowledge: The proof does not reveal anything but the truth of the statement
    • \n
\n

1.1. Contribution

Succinct NIZK We construct a NIZK argument for arithmetic circuit satisfiability where a proof consists of only 3 group elements. In addition to being small, the proof is also easy to verify. The verifier just needs to compute a number of exponentiations proportional to the statement size and check a single pairing product equation, which only has 3 pairings.

\n

2. Preliminaries

2.1 Bilinear Groups

We will work over bilinear groups \\((p, \\mathbb{G_{1}}, \\mathbb{G_{2}}, \\mathbb{G_{T}} , e, g, h)\\) with the following properties:

\n
    \n
  • \\(\\mathbb{G_{1}}, \\mathbb{G_{2}}, \\mathbb{G_{T}} \\)are groups of prime order \\(p\\)
    • \n
  • The pairing \\( e:\\mathbb{G_{1}} \\times \\mathbb{G_{2}} \\rightarrow \\mathbb{G_{T}}\\) is a bilinear map
    • \n
  • \\(g\\) is a generator for \\(\\mathbb{G_{1}}\\), \\(h\\) is a generator for \\(\\mathbb{G_{2}}\\), and \\(e(g,h)\\) is a generator for \\(\\mathbb{G_{T}}\\)
    • \n
\n

There are many ways to set up bilinear groups both as symmetric bilinear groups where \\(\\mathbb{G_{1}} = \\mathbb{G_{1}}\\) and as asymmetric bilinear groups where \\(\\mathbb{G_{1}} \\neq \\mathbb{G_{1}}\\). Galbraith, Paterson and Smart [GPS08] classify bilinear groups as Type I where \\(\\mathbb{G_{1}} = \\mathbb{G_{2}}\\), Type II where there is an efficiently computable non-trivial homomorphism \\(\\Phi : \\mathbb{G_{2}} \\rightarrow \\mathbb{G_{1}}\\), and Type III where no such efficiently computable homomorphism exists in either direction between \\(\\mathbb{G_{1}}\\) and \\(\\mathbb{G_{2}}\\). Type III bilinear groups are the most efficient type of bilinear groups and hence the most relevant for practical applications.
As a notation for group elements, we write \\( \\lbrack a \\rbrack_{1} \\) for \\( g^a\\), \\( \\lbrack b \\rbrack_{2}\\) for \\( h^b\\) and \\( \\lbrack c \\rbrack_{T}\\) for \\(e(g,h)^{c} \\). A vector of group elements will be represented as \\( \\lbrack \\mathbf{a} \\rbrack_{i} \\). Given two vectors of \\(n\\) group elements \\( \\lbrack \\mathbf{a} \\rbrack_{1} \\) and \\( \\lbrack \\mathbf{b} \\rbrack_{2} \\), we define their dot product as \\( \\lbrack \\mathbf{a} \\rbrack_{1} \\cdot \\lbrack \\mathbf{b} \\rbrack_{2} = \\lbrack \\mathbf{a} \\cdot \\mathbf{b} \\rbrack_{T} \\), which can be efficiently computed using the pairing \\(e\\).

\n

2.2 Non-interactive zero-knowledge arguments of knowledge

references

    \n
  • [1] groth 16 paper, On the Size of Pairing-based Non-interactive Arguments by Jens Groth
    • \n
\n","site":{"data":{}},"excerpt":"","more":"\n\n

1. Introduction

Goldwasser, Micali and Rackoff [GMR89] introduced zero-knowledge proofs that enable a prover to convince a verifier that a statement is true without revealing anything else. They have three core properties:

\n
    \n
  • Completeness: Given a statement and a witness, the prover can convince the verifier.
    • \n
  • Soundness: A malicious prover cannot convince the verifier of a false statement.
    • \n
  • Zero-knowledge: The proof does not reveal anything but the truth of the statement
    • \n
\n

1.1. Contribution

Succinct NIZK We construct a NIZK argument for arithmetic circuit satisfiability where a proof consists of only 3 group elements. In addition to being small, the proof is also easy to verify. The verifier just needs to compute a number of exponentiations proportional to the statement size and check a single pairing product equation, which only has 3 pairings.

\n

2. Preliminaries

2.1 Bilinear Groups

We will work over bilinear groups \\((p, \\mathbb{G_{1}}, \\mathbb{G_{2}}, \\mathbb{G_{T}} , e, g, h)\\) with the following properties:

\n
    \n
  • \\(\\mathbb{G_{1}}, \\mathbb{G_{2}}, \\mathbb{G_{T}} \\)are groups of prime order \\(p\\)
    • \n
  • The pairing \\( e:\\mathbb{G_{1}} \\times \\mathbb{G_{2}} \\rightarrow \\mathbb{G_{T}}\\) is a bilinear map
    • \n
  • \\(g\\) is a generator for \\(\\mathbb{G_{1}}\\), \\(h\\) is a generator for \\(\\mathbb{G_{2}}\\), and \\(e(g,h)\\) is a generator for \\(\\mathbb{G_{T}}\\)
    • \n
\n

There are many ways to set up bilinear groups both as symmetric bilinear groups where \\(\\mathbb{G_{1}} = \\mathbb{G_{1}}\\) and as asymmetric bilinear groups where \\(\\mathbb{G_{1}} \\neq \\mathbb{G_{1}}\\). Galbraith, Paterson and Smart [GPS08] classify bilinear groups as Type I where \\(\\mathbb{G_{1}} = \\mathbb{G_{2}}\\), Type II where there is an efficiently computable non-trivial homomorphism \\(\\Phi : \\mathbb{G_{2}} \\rightarrow \\mathbb{G_{1}}\\), and Type III where no such efficiently computable homomorphism exists in either direction between \\(\\mathbb{G_{1}}\\) and \\(\\mathbb{G_{2}}\\). Type III bilinear groups are the most efficient type of bilinear groups and hence the most relevant for practical applications.
As a notation for group elements, we write \\( \\lbrack a \\rbrack_{1} \\) for \\( g^a\\), \\( \\lbrack b \\rbrack_{2}\\) for \\( h^b\\) and \\( \\lbrack c \\rbrack_{T}\\) for \\(e(g,h)^{c} \\). A vector of group elements will be represented as \\( \\lbrack \\mathbf{a} \\rbrack_{i} \\). Given two vectors of \\(n\\) group elements \\( \\lbrack \\mathbf{a} \\rbrack_{1} \\) and \\( \\lbrack \\mathbf{b} \\rbrack_{2} \\), we define their dot product as \\( \\lbrack \\mathbf{a} \\rbrack_{1} \\cdot \\lbrack \\mathbf{b} \\rbrack_{2} = \\lbrack \\mathbf{a} \\cdot \\mathbf{b} \\rbrack_{T} \\), which can be efficiently computed using the pairing \\(e\\).

\n

2.2 Non-interactive zero-knowledge arguments of knowledge

references

    \n
  • [1] groth 16 paper, On the Size of Pairing-based Non-interactive Arguments by Jens Groth
    • \n
\n"},{"title":"geth v1.10.0 summary","date":"2023-03-15T08:29:43.000Z","_content":"\n## introduction\ngeth v1.10.0 has been [released](https://github.com/ethereum/go-ethereum/releases/tag/v1.10.0) on Mar 4 2021. this is a late summary of v1.10.0.\n\n## snapshots\nthe snapshot feature reduces the cost of accessing an account from `O(logN)` to `O(1)`. Whilst snapshots do grant us a 10x read performance, EVM execution also writes data, and these writes need to be Merkle proven. The Merkle proof requirement retains the necessity for `O(logN)` disk access on writes. \nProblems it solves\n- **DoS** In 2016, Ethereum sustained its worse DoS attack ever - The [Shanghai Attacks](https://2017.edcon.io/ppt/one/Martin%20Holst%20Swende_The%20%27Shanghai%20%27Attacks_EDCON.pdf) - that lasted about 2-3 months. The attack revolved around bloating Ethereum's state and abusing various underpriced opcodes to grind the network to a halt. After numerous client optimizations and repricing hard forks, the attack was repelled. The root cause still lingers: state access opcodes have a fixed EVM gas cost O(1), but an ever slowly increasing execution cost O(logN). Snapshots on the other hand reduce execution cost of state reads to O(1) - in line with EVM costs - thus solves the read-based DoS issues long term.\n- **Call** Checking a smart contract's state in Ethereum entails a mini EVM execution. Part of that is running bytecode and part of it is reading state slots from disk. snap makes the state access faster.\n- **Sync** There are two major ways you can synchronize an Ethereum node. You can download the blocks and execute all the transactions within; or you can download the blocks, verify the PoWs and download the state associated a recent block. The latter is much faster, but it relies on benefactors serving you a copy of the recent state. With the current Merkle-Patricia state model, these benefactors read 16TB of data off disk to serve a syncing node. Snapshots enable serving nodes to read only **96GB** of data off disk to get a new node joined into the network.\n\ndrawbacks of snapshot\n- A snapshot is a redundant copy of the raw Ethereum state already contained in the leaves of the Merkle Patricia trie. \nuser can disable snapshot via `--snapshot=false`\n\n## snap sync\nWhen Ethereum launched, you could choose from two different ways to synchronize the network: full sync and fast sync。 Full sync operated by downloading the entire chain and executing all transactions; vs. fast sync placed an initial trust in a recent-ish block, and directly downloaded the state associated with it (after which it switched to block execution like full sync). \n- **full sync** minimized trust, choosing to execute all transactions from genesis to head. \n- **fast sync** chose to rely on the security of the PoWs.it assumed that a block with 64 valid PoWs on top would be prohibitively expensive for someone to construct, as such it's ok to download the state associated with `HEAD-64`\n\n### delays of fast sync\n- network latency (download node)\n- io latency (level db random disk access)\n- upload latency (requst with node `hash` to remote servers)\n\nThe core idea of `snap sync` is fairly simple: instead of downloading the trie node-by-node, snap sync downloads the contiguous chunks of useful state data, and reconstructs the Merkle trie locally:\n- Without downloading intermediate Merkle trie nodes, state data can be fetched in large batches, removing the delay caused by network latency.\n- Without downloading Merkle nodes, downstream data drops to half; and without addressing each piece of data individually, upstream data gets insignificant, removing the delay caused by bandwidth.\n- Without requesting randomly keyed data, peers do only a couple contiguous disk reads to serve the responses, removing the delay of disk IO\n\n## offline pruning\nWhen processing a new block, a node takes the current state of the network as input data and mutates it according to the transactions in the block. only state diff is kept. Pushing these new pieces of state data, block-by-block, to the database is a problem. They keep accumulating. In theory we could \"just delete\" state data that's old enough to not run the risk of a reorg. it's exceedingly costly to figure out if a node deep within an old state is still referenced by anything newer or not.\nIf you have snapshots enabled and fully generated, Geth can use those as an acceleration structure to relatively quickly determine which trie nodes should be kept and which should be deleted. Pruning trie nodes based on snapshots does have the drawback that the chain may not progress during pruning. This means, that you need to stop Geth, prune its database and then restart it. To prune your database, please run `geth snapshot prune-state`.\n\n## transaction unindexing\nNode operators always took it for granted that they can look up an arbitrary transaction from the past, given only its hash. To make transactions searchable, we need to - at minimum - map the entire range of transaction hashes to the blocks they are in. It's also important to note that transaction indices are not part of consensus and are not part of the network protocol. They are purely a locally generated acceleration structure.\nGeth v1.10.0 switches on transaction unindexing by default and sets it to 2,350,000 blocks (about 1 year). The transaction unindexer will linger in the background, and every time a new block arrives, it ensures that only transactions from the most recent N blocks are indexed, deleting older ones. user can use `--txlookuplimit` to control the indexing block range\n\n## preimage discarding\nEthereum stores all its data in a Merkle Patricia trie. The values in the leaves are the raw data being stored (e.g. storage slot content, account content), and the path to the leaf is the key at which the data is stored. The keys however are not the account addresses or storage addresses, rather the Keccak256 hashes of those. This helps balance the branch depths of the state tries.\nthe preimage is the actual key related to the hash. The preimages aren't particularly heavy. If you do a full sync from genesis - reexecuting all the transactions - you'll only end up with 5GB extra load. Still, there is no reason to keep that data around for users not using it, as it only increases the load on LevelDB compactions. As such, Geth v1.10.0 disables preimage collection by default, but there's no mechanism to actively delete already stored preimages.\nIf you are using your Geth instance to debug transactions, you can retain the original behavior via `--cache.preimages`. \n\n## ETH/66 protocol\nThe eth/66 protocol is a fairly small change, yet has quite a number of beneficial implications. In short, the protocol introduces request and reply IDs for all bidirectional packets. The goal behind these IDs is to more easily match up responses to requests, specifically, to more easily deliver a response to a subsystem that made the original request.\n\n## chainid enforcement\nGeth v1.10.0 supports reverting to the old behavior and accepting non-EIP155 transactions via --rpc.allow-unprotected-txs. Be advised that this is a temporary mechanism that will be removed long term.\n\n## Database introspection\nEvery now and again we receive an issue report about a corrupted database, with no real way to debug it. Geth v1.10.0 ships a built-in database introspection tool to try and alleviate the situation a bit. It is a very low level accessor to LevelDB, but it allows arbitrary data retrievals, insertions and deletions. We are unsure how useful these will turn out to be, but they at least give a fighting chance to restore a broken node without having to resync.\n\n## Unclean shutdown tracking\nFairly often we receive bug reports that Geth started importing old blocks on startup. This phenomenon is generally caused by the node operator terminating Geth abruptly (power outage, OOM killer, too short shutdown timeout). Since Geth keeps a lot of dirty state in memory - to avoid writing to disk things that get stale a few blocks later - an abrupt shutdown can cause these to not be flushed. With recent state missing on startup, Geth has no choice but to rewind it's local chain to the point where it last saved the progress.\n\nGeth v1.10.0 will start tracking and reporting node crashes. We're hopeful that this will allow operatos to detect that their infra is misconfigured or has issue before those turn into irreversible data loss.\n```\nWARN [03-03|06:36:38.734] Unclean shutdown detected booted=2021-02-03T06:47:28+0000 age=3w6d23h\n```\n\n## references\n- [eth foundation blog]()","source":"_posts/geth/tech_docs/geth.v1.10.0.md","raw":"---\ntitle: geth v1.10.0 summary\ndate: 2023-03-15 16:29:43\ntags: [geth]\n---\n\n## introduction\ngeth v1.10.0 has been [released](https://github.com/ethereum/go-ethereum/releases/tag/v1.10.0) on Mar 4 2021. this is a late summary of v1.10.0.\n\n## snapshots\nthe snapshot feature reduces the cost of accessing an account from `O(logN)` to `O(1)`. Whilst snapshots do grant us a 10x read performance, EVM execution also writes data, and these writes need to be Merkle proven. The Merkle proof requirement retains the necessity for `O(logN)` disk access on writes. \nProblems it solves\n- **DoS** In 2016, Ethereum sustained its worse DoS attack ever - The [Shanghai Attacks](https://2017.edcon.io/ppt/one/Martin%20Holst%20Swende_The%20%27Shanghai%20%27Attacks_EDCON.pdf) - that lasted about 2-3 months. The attack revolved around bloating Ethereum's state and abusing various underpriced opcodes to grind the network to a halt. After numerous client optimizations and repricing hard forks, the attack was repelled. The root cause still lingers: state access opcodes have a fixed EVM gas cost O(1), but an ever slowly increasing execution cost O(logN). Snapshots on the other hand reduce execution cost of state reads to O(1) - in line with EVM costs - thus solves the read-based DoS issues long term.\n- **Call** Checking a smart contract's state in Ethereum entails a mini EVM execution. Part of that is running bytecode and part of it is reading state slots from disk. snap makes the state access faster.\n- **Sync** There are two major ways you can synchronize an Ethereum node. You can download the blocks and execute all the transactions within; or you can download the blocks, verify the PoWs and download the state associated a recent block. The latter is much faster, but it relies on benefactors serving you a copy of the recent state. With the current Merkle-Patricia state model, these benefactors read 16TB of data off disk to serve a syncing node. Snapshots enable serving nodes to read only **96GB** of data off disk to get a new node joined into the network.\n\ndrawbacks of snapshot\n- A snapshot is a redundant copy of the raw Ethereum state already contained in the leaves of the Merkle Patricia trie. \nuser can disable snapshot via `--snapshot=false`\n\n## snap sync\nWhen Ethereum launched, you could choose from two different ways to synchronize the network: full sync and fast sync。 Full sync operated by downloading the entire chain and executing all transactions; vs. fast sync placed an initial trust in a recent-ish block, and directly downloaded the state associated with it (after which it switched to block execution like full sync). \n- **full sync** minimized trust, choosing to execute all transactions from genesis to head. \n- **fast sync** chose to rely on the security of the PoWs.it assumed that a block with 64 valid PoWs on top would be prohibitively expensive for someone to construct, as such it's ok to download the state associated with `HEAD-64`\n\n### delays of fast sync\n- network latency (download node)\n- io latency (level db random disk access)\n- upload latency (requst with node `hash` to remote servers)\n\nThe core idea of `snap sync` is fairly simple: instead of downloading the trie node-by-node, snap sync downloads the contiguous chunks of useful state data, and reconstructs the Merkle trie locally:\n- Without downloading intermediate Merkle trie nodes, state data can be fetched in large batches, removing the delay caused by network latency.\n- Without downloading Merkle nodes, downstream data drops to half; and without addressing each piece of data individually, upstream data gets insignificant, removing the delay caused by bandwidth.\n- Without requesting randomly keyed data, peers do only a couple contiguous disk reads to serve the responses, removing the delay of disk IO\n\n## offline pruning\nWhen processing a new block, a node takes the current state of the network as input data and mutates it according to the transactions in the block. only state diff is kept. Pushing these new pieces of state data, block-by-block, to the database is a problem. They keep accumulating. In theory we could \"just delete\" state data that's old enough to not run the risk of a reorg. it's exceedingly costly to figure out if a node deep within an old state is still referenced by anything newer or not.\nIf you have snapshots enabled and fully generated, Geth can use those as an acceleration structure to relatively quickly determine which trie nodes should be kept and which should be deleted. Pruning trie nodes based on snapshots does have the drawback that the chain may not progress during pruning. This means, that you need to stop Geth, prune its database and then restart it. To prune your database, please run `geth snapshot prune-state`.\n\n## transaction unindexing\nNode operators always took it for granted that they can look up an arbitrary transaction from the past, given only its hash. To make transactions searchable, we need to - at minimum - map the entire range of transaction hashes to the blocks they are in. It's also important to note that transaction indices are not part of consensus and are not part of the network protocol. They are purely a locally generated acceleration structure.\nGeth v1.10.0 switches on transaction unindexing by default and sets it to 2,350,000 blocks (about 1 year). The transaction unindexer will linger in the background, and every time a new block arrives, it ensures that only transactions from the most recent N blocks are indexed, deleting older ones. user can use `--txlookuplimit` to control the indexing block range\n\n## preimage discarding\nEthereum stores all its data in a Merkle Patricia trie. The values in the leaves are the raw data being stored (e.g. storage slot content, account content), and the path to the leaf is the key at which the data is stored. The keys however are not the account addresses or storage addresses, rather the Keccak256 hashes of those. This helps balance the branch depths of the state tries.\nthe preimage is the actual key related to the hash. The preimages aren't particularly heavy. If you do a full sync from genesis - reexecuting all the transactions - you'll only end up with 5GB extra load. Still, there is no reason to keep that data around for users not using it, as it only increases the load on LevelDB compactions. As such, Geth v1.10.0 disables preimage collection by default, but there's no mechanism to actively delete already stored preimages.\nIf you are using your Geth instance to debug transactions, you can retain the original behavior via `--cache.preimages`. \n\n## ETH/66 protocol\nThe eth/66 protocol is a fairly small change, yet has quite a number of beneficial implications. In short, the protocol introduces request and reply IDs for all bidirectional packets. The goal behind these IDs is to more easily match up responses to requests, specifically, to more easily deliver a response to a subsystem that made the original request.\n\n## chainid enforcement\nGeth v1.10.0 supports reverting to the old behavior and accepting non-EIP155 transactions via --rpc.allow-unprotected-txs. Be advised that this is a temporary mechanism that will be removed long term.\n\n## Database introspection\nEvery now and again we receive an issue report about a corrupted database, with no real way to debug it. Geth v1.10.0 ships a built-in database introspection tool to try and alleviate the situation a bit. It is a very low level accessor to LevelDB, but it allows arbitrary data retrievals, insertions and deletions. We are unsure how useful these will turn out to be, but they at least give a fighting chance to restore a broken node without having to resync.\n\n## Unclean shutdown tracking\nFairly often we receive bug reports that Geth started importing old blocks on startup. This phenomenon is generally caused by the node operator terminating Geth abruptly (power outage, OOM killer, too short shutdown timeout). Since Geth keeps a lot of dirty state in memory - to avoid writing to disk things that get stale a few blocks later - an abrupt shutdown can cause these to not be flushed. With recent state missing on startup, Geth has no choice but to rewind it's local chain to the point where it last saved the progress.\n\nGeth v1.10.0 will start tracking and reporting node crashes. We're hopeful that this will allow operatos to detect that their infra is misconfigured or has issue before those turn into irreversible data loss.\n```\nWARN [03-03|06:36:38.734] Unclean shutdown detected booted=2021-02-03T06:47:28+0000 age=3w6d23h\n```\n\n## references\n- [eth foundation blog]()","slug":"geth/tech_docs/geth.v1.10.0","published":1,"updated":"2023-11-05T04:21:13.733Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8e1003eqwsj3oda50v5","content":"

introduction

geth v1.10.0 has been released on Mar 4 2021. this is a late summary of v1.10.0.

\n

snapshots

the snapshot feature reduces the cost of accessing an account from O(logN) to O(1). Whilst snapshots do grant us a 10x read performance, EVM execution also writes data, and these writes need to be Merkle proven. The Merkle proof requirement retains the necessity for O(logN) disk access on writes.
Problems it solves

\n
    \n
  • DoS In 2016, Ethereum sustained its worse DoS attack ever - The Shanghai Attacks - that lasted about 2-3 months. The attack revolved around bloating Ethereum’s state and abusing various underpriced opcodes to grind the network to a halt. After numerous client optimizations and repricing hard forks, the attack was repelled. The root cause still lingers: state access opcodes have a fixed EVM gas cost O(1), but an ever slowly increasing execution cost O(logN). Snapshots on the other hand reduce execution cost of state reads to O(1) - in line with EVM costs - thus solves the read-based DoS issues long term.
    • \n
  • Call Checking a smart contract’s state in Ethereum entails a mini EVM execution. Part of that is running bytecode and part of it is reading state slots from disk. snap makes the state access faster.
    • \n
  • Sync There are two major ways you can synchronize an Ethereum node. You can download the blocks and execute all the transactions within; or you can download the blocks, verify the PoWs and download the state associated a recent block. The latter is much faster, but it relies on benefactors serving you a copy of the recent state. With the current Merkle-Patricia state model, these benefactors read 16TB of data off disk to serve a syncing node. Snapshots enable serving nodes to read only 96GB of data off disk to get a new node joined into the network.
    • \n
\n

drawbacks of snapshot

\n
    \n
  • A snapshot is a redundant copy of the raw Ethereum state already contained in the leaves of the Merkle Patricia trie.
    user can disable snapshot via --snapshot=false
    • \n
\n

snap sync

When Ethereum launched, you could choose from two different ways to synchronize the network: full sync and fast sync。 Full sync operated by downloading the entire chain and executing all transactions; vs. fast sync placed an initial trust in a recent-ish block, and directly downloaded the state associated with it (after which it switched to block execution like full sync).

\n
    \n
  • full sync minimized trust, choosing to execute all transactions from genesis to head.
    • \n
  • fast sync chose to rely on the security of the PoWs.it assumed that a block with 64 valid PoWs on top would be prohibitively expensive for someone to construct, as such it’s ok to download the state associated with HEAD-64
    • \n
\n

delays of fast sync

    \n
  • network latency (download node)
    • \n
  • io latency (level db random disk access)
    • \n
  • upload latency (requst with node hash to remote servers)
    • \n
\n

The core idea of snap sync is fairly simple: instead of downloading the trie node-by-node, snap sync downloads the contiguous chunks of useful state data, and reconstructs the Merkle trie locally:

\n
    \n
  • Without downloading intermediate Merkle trie nodes, state data can be fetched in large batches, removing the delay caused by network latency.
    • \n
  • Without downloading Merkle nodes, downstream data drops to half; and without addressing each piece of data individually, upstream data gets insignificant, removing the delay caused by bandwidth.
    • \n
  • Without requesting randomly keyed data, peers do only a couple contiguous disk reads to serve the responses, removing the delay of disk IO
    • \n
\n

offline pruning

When processing a new block, a node takes the current state of the network as input data and mutates it according to the transactions in the block. only state diff is kept. Pushing these new pieces of state data, block-by-block, to the database is a problem. They keep accumulating. In theory we could “just delete” state data that’s old enough to not run the risk of a reorg. it’s exceedingly costly to figure out if a node deep within an old state is still referenced by anything newer or not.
If you have snapshots enabled and fully generated, Geth can use those as an acceleration structure to relatively quickly determine which trie nodes should be kept and which should be deleted. Pruning trie nodes based on snapshots does have the drawback that the chain may not progress during pruning. This means, that you need to stop Geth, prune its database and then restart it. To prune your database, please run geth snapshot prune-state.

\n

transaction unindexing

Node operators always took it for granted that they can look up an arbitrary transaction from the past, given only its hash. To make transactions searchable, we need to - at minimum - map the entire range of transaction hashes to the blocks they are in. It’s also important to note that transaction indices are not part of consensus and are not part of the network protocol. They are purely a locally generated acceleration structure.
Geth v1.10.0 switches on transaction unindexing by default and sets it to 2,350,000 blocks (about 1 year). The transaction unindexer will linger in the background, and every time a new block arrives, it ensures that only transactions from the most recent N blocks are indexed, deleting older ones. user can use --txlookuplimit to control the indexing block range

\n

preimage discarding

Ethereum stores all its data in a Merkle Patricia trie. The values in the leaves are the raw data being stored (e.g. storage slot content, account content), and the path to the leaf is the key at which the data is stored. The keys however are not the account addresses or storage addresses, rather the Keccak256 hashes of those. This helps balance the branch depths of the state tries.
the preimage is the actual key related to the hash. The preimages aren’t particularly heavy. If you do a full sync from genesis - reexecuting all the transactions - you’ll only end up with 5GB extra load. Still, there is no reason to keep that data around for users not using it, as it only increases the load on LevelDB compactions. As such, Geth v1.10.0 disables preimage collection by default, but there’s no mechanism to actively delete already stored preimages.
If you are using your Geth instance to debug transactions, you can retain the original behavior via --cache.preimages.

\n

ETH/66 protocol

The eth/66 protocol is a fairly small change, yet has quite a number of beneficial implications. In short, the protocol introduces request and reply IDs for all bidirectional packets. The goal behind these IDs is to more easily match up responses to requests, specifically, to more easily deliver a response to a subsystem that made the original request.

\n

chainid enforcement

Geth v1.10.0 supports reverting to the old behavior and accepting non-EIP155 transactions via –rpc.allow-unprotected-txs. Be advised that this is a temporary mechanism that will be removed long term.

\n

Database introspection

Every now and again we receive an issue report about a corrupted database, with no real way to debug it. Geth v1.10.0 ships a built-in database introspection tool to try and alleviate the situation a bit. It is a very low level accessor to LevelDB, but it allows arbitrary data retrievals, insertions and deletions. We are unsure how useful these will turn out to be, but they at least give a fighting chance to restore a broken node without having to resync.

\n

Unclean shutdown tracking

Fairly often we receive bug reports that Geth started importing old blocks on startup. This phenomenon is generally caused by the node operator terminating Geth abruptly (power outage, OOM killer, too short shutdown timeout). Since Geth keeps a lot of dirty state in memory - to avoid writing to disk things that get stale a few blocks later - an abrupt shutdown can cause these to not be flushed. With recent state missing on startup, Geth has no choice but to rewind it’s local chain to the point where it last saved the progress.

\n

Geth v1.10.0 will start tracking and reporting node crashes. We’re hopeful that this will allow operatos to detect that their infra is misconfigured or has issue before those turn into irreversible data loss.

\n
1
WARN [03-03|06:36:38.734] Unclean shutdown detected        booted=2021-02-03T06:47:28+0000 age=3w6d23h
\n\n

references

\n","site":{"data":{}},"excerpt":"","more":"

introduction

geth v1.10.0 has been released on Mar 4 2021. this is a late summary of v1.10.0.

\n

snapshots

the snapshot feature reduces the cost of accessing an account from O(logN) to O(1). Whilst snapshots do grant us a 10x read performance, EVM execution also writes data, and these writes need to be Merkle proven. The Merkle proof requirement retains the necessity for O(logN) disk access on writes.
Problems it solves

\n
    \n
  • DoS In 2016, Ethereum sustained its worse DoS attack ever - The Shanghai Attacks - that lasted about 2-3 months. The attack revolved around bloating Ethereum’s state and abusing various underpriced opcodes to grind the network to a halt. After numerous client optimizations and repricing hard forks, the attack was repelled. The root cause still lingers: state access opcodes have a fixed EVM gas cost O(1), but an ever slowly increasing execution cost O(logN). Snapshots on the other hand reduce execution cost of state reads to O(1) - in line with EVM costs - thus solves the read-based DoS issues long term.
    • \n
  • Call Checking a smart contract’s state in Ethereum entails a mini EVM execution. Part of that is running bytecode and part of it is reading state slots from disk. snap makes the state access faster.
    • \n
  • Sync There are two major ways you can synchronize an Ethereum node. You can download the blocks and execute all the transactions within; or you can download the blocks, verify the PoWs and download the state associated a recent block. The latter is much faster, but it relies on benefactors serving you a copy of the recent state. With the current Merkle-Patricia state model, these benefactors read 16TB of data off disk to serve a syncing node. Snapshots enable serving nodes to read only 96GB of data off disk to get a new node joined into the network.
    • \n
\n

drawbacks of snapshot

\n
    \n
  • A snapshot is a redundant copy of the raw Ethereum state already contained in the leaves of the Merkle Patricia trie.
    user can disable snapshot via --snapshot=false
    • \n
\n

snap sync

When Ethereum launched, you could choose from two different ways to synchronize the network: full sync and fast sync。 Full sync operated by downloading the entire chain and executing all transactions; vs. fast sync placed an initial trust in a recent-ish block, and directly downloaded the state associated with it (after which it switched to block execution like full sync).

\n
    \n
  • full sync minimized trust, choosing to execute all transactions from genesis to head.
    • \n
  • fast sync chose to rely on the security of the PoWs.it assumed that a block with 64 valid PoWs on top would be prohibitively expensive for someone to construct, as such it’s ok to download the state associated with HEAD-64
    • \n
\n

delays of fast sync

    \n
  • network latency (download node)
    • \n
  • io latency (level db random disk access)
    • \n
  • upload latency (requst with node hash to remote servers)
    • \n
\n

The core idea of snap sync is fairly simple: instead of downloading the trie node-by-node, snap sync downloads the contiguous chunks of useful state data, and reconstructs the Merkle trie locally:

\n
    \n
  • Without downloading intermediate Merkle trie nodes, state data can be fetched in large batches, removing the delay caused by network latency.
    • \n
  • Without downloading Merkle nodes, downstream data drops to half; and without addressing each piece of data individually, upstream data gets insignificant, removing the delay caused by bandwidth.
    • \n
  • Without requesting randomly keyed data, peers do only a couple contiguous disk reads to serve the responses, removing the delay of disk IO
    • \n
\n

offline pruning

When processing a new block, a node takes the current state of the network as input data and mutates it according to the transactions in the block. only state diff is kept. Pushing these new pieces of state data, block-by-block, to the database is a problem. They keep accumulating. In theory we could “just delete” state data that’s old enough to not run the risk of a reorg. it’s exceedingly costly to figure out if a node deep within an old state is still referenced by anything newer or not.
If you have snapshots enabled and fully generated, Geth can use those as an acceleration structure to relatively quickly determine which trie nodes should be kept and which should be deleted. Pruning trie nodes based on snapshots does have the drawback that the chain may not progress during pruning. This means, that you need to stop Geth, prune its database and then restart it. To prune your database, please run geth snapshot prune-state.

\n

transaction unindexing

Node operators always took it for granted that they can look up an arbitrary transaction from the past, given only its hash. To make transactions searchable, we need to - at minimum - map the entire range of transaction hashes to the blocks they are in. It’s also important to note that transaction indices are not part of consensus and are not part of the network protocol. They are purely a locally generated acceleration structure.
Geth v1.10.0 switches on transaction unindexing by default and sets it to 2,350,000 blocks (about 1 year). The transaction unindexer will linger in the background, and every time a new block arrives, it ensures that only transactions from the most recent N blocks are indexed, deleting older ones. user can use --txlookuplimit to control the indexing block range

\n

preimage discarding

Ethereum stores all its data in a Merkle Patricia trie. The values in the leaves are the raw data being stored (e.g. storage slot content, account content), and the path to the leaf is the key at which the data is stored. The keys however are not the account addresses or storage addresses, rather the Keccak256 hashes of those. This helps balance the branch depths of the state tries.
the preimage is the actual key related to the hash. The preimages aren’t particularly heavy. If you do a full sync from genesis - reexecuting all the transactions - you’ll only end up with 5GB extra load. Still, there is no reason to keep that data around for users not using it, as it only increases the load on LevelDB compactions. As such, Geth v1.10.0 disables preimage collection by default, but there’s no mechanism to actively delete already stored preimages.
If you are using your Geth instance to debug transactions, you can retain the original behavior via --cache.preimages.

\n

ETH/66 protocol

The eth/66 protocol is a fairly small change, yet has quite a number of beneficial implications. In short, the protocol introduces request and reply IDs for all bidirectional packets. The goal behind these IDs is to more easily match up responses to requests, specifically, to more easily deliver a response to a subsystem that made the original request.

\n

chainid enforcement

Geth v1.10.0 supports reverting to the old behavior and accepting non-EIP155 transactions via –rpc.allow-unprotected-txs. Be advised that this is a temporary mechanism that will be removed long term.

\n

Database introspection

Every now and again we receive an issue report about a corrupted database, with no real way to debug it. Geth v1.10.0 ships a built-in database introspection tool to try and alleviate the situation a bit. It is a very low level accessor to LevelDB, but it allows arbitrary data retrievals, insertions and deletions. We are unsure how useful these will turn out to be, but they at least give a fighting chance to restore a broken node without having to resync.

\n

Unclean shutdown tracking

Fairly often we receive bug reports that Geth started importing old blocks on startup. This phenomenon is generally caused by the node operator terminating Geth abruptly (power outage, OOM killer, too short shutdown timeout). Since Geth keeps a lot of dirty state in memory - to avoid writing to disk things that get stale a few blocks later - an abrupt shutdown can cause these to not be flushed. With recent state missing on startup, Geth has no choice but to rewind it’s local chain to the point where it last saved the progress.

\n

Geth v1.10.0 will start tracking and reporting node crashes. We’re hopeful that this will allow operatos to detect that their infra is misconfigured or has issue before those turn into irreversible data loss.

\n
1
WARN [03-03|06:36:38.734] Unclean shutdown detected        booted=2021-02-03T06:47:28+0000 age=3w6d23h
\n\n

references

\n"},{"title":"elliptic curve paring","date":"2023-06-27T06:29:26.000Z","_content":"\n\n\n## introduction\n\nPairings allow you to check certain kinds of more complicated equations on elliptic curve points — for example, if \\\\(P = G * p\\\\), \\\\(Q = G * q\\\\) and \\\\(R = G * r\\\\), you can check whether or not \\\\(p * q = r\\\\), having just P, Q and R as inputs. This might seem like the fundamental security guarantees of elliptic curves are being broken, as information about p is leaking from just knowing P, but it turns out that the leakage is highly contained — specifically, the ***decisional Diffie Hellman problem*** is easy, but the computational Diffie Hellman problem (knowing P and Q in the above example, computing R = G * p * q) and the discrete logarithm problem (recovering p from P) remain computationally infeasible.\n\n whereas traditional elliptic curve math lets you check linear constraints on the numbers (eg. if P = G * p, Q = G * q and R = G * r, checking 5 * P + 7 * Q = 11 * R is really checking that 5 * p + 7 * q = 11 * r), pairings let you check quadratic constraints (eg. checking e(P, Q) * e(G, G * 5) = 1 is really checking that p * q + 5 = 0).\n\n Now, what is this funny e(P, Q) operator that we introduced above? This is the pairing. Mathematicians also sometimes call it a bilinear map; the word “bilinear” here basically means that it satisfies the constraints:\n\n\\\\[ e(P, Q + R) = e(P, Q) * e(P, R) \\\\]\n\\\\[ e(P + S, Q) = e(P, Q) * e(S, Q) \\\\]\nNote that + and * can be arbitrary operators; when you’re creating fancy new kinds of mathematical objects, abstract algebra doesn’t care how + and * are defined, as long as they are consistent in the usual ways.\n\nIf P, Q, R and S were simple numbers, then making a simple pairing is easy: we can do e(x, y) = 2^xy. Then, we can see:\n\\\\[e(3, 4+ 5) = 2^(3 * 9) = 2^{27}\\\\]\n\\\\[e(3, 4) * e(3, 5) = 2^(3 * 4) * 2^(3 * 5) = 2^{12} * 2^{15} = 2^{27} \\\\]\nHowever, such simple pairings are not suitable for cryptography because the objects that they work on are simple integers and are too easy to analyze;\n\nIt turns out that it is possible to make a bilinear map over elliptic curve points — that is, come up with a function e(P, Q) where the inputs P and Q are elliptic curve points, and where the output is what’s called an \\\\(F_{p^¹²}\\\\) (extension field) element\n\nAn elliptic curve pairing is a map G2 x G1 -> Gt, where:\n- G1 is an elliptic curve, where points satisfy an equation of the form y² = x³ + b, and where both coordinates are elements of F_p (ie. they are simple numbers, except arithmetic is all done modulo some prime number)\n- G2 is an elliptic curve, where points satisfy the same equation as G1, except where the coordinates are elements of \\\\(F_{p^¹²}\\\\). we define a new “magic number” w, which is defined by a 12th degree polynomial like w^12 - 18 * w^6 + 82 = 0\n- Gt is the type of object that the result of the elliptic curve goes into. In the curves that we look at, Gt is \\\\(F_{p^¹²}\\\\)\nThe main property that it must satisfy is bilinearity, which in presented above\n\nThere are two other important criteria:\n- **Efficient computability** (eg. we can make an easy pairing by simply taking the discrete logarithms of all points and multiplying them together, but this is as computationally hard as breaking elliptic curve cryptography in the first place, so it doesn’t count)\n- **Non-degeneracy** (sure, you could just define e(P, Q) = 1, but that’s not a particularly useful pairing)\n\n## math intuition behind paring\nThe math behind why pairing functions work is quite tricky and involves quite a bit of advanced algebra going even beyond what we’ve seen so far, but I’ll provide an outline. First of all, we need to define the concept of a divisor, basically an alternative way of representing functions on elliptic curve points. A **divisor** of a function basically counts the zeroes and the infinities of the function. To see what this means, let’s go through a few examples. Let us fix some point P = (P_x, P_y), and consider the following function:\n\\\\[f(x, y) = x - P_x\\\\]\n\nThe divisor is [P] + [-P] - 2 * [O] (the square brackets are used to represent the fact that we are referring to the presence of the point P in the set of zeroes and infinities of the function, not the point P itself; [P] + [Q] is not the same thing as [P + Q]). The reasoning is as follows:\n- The function is equal to zero at P, since x is P_x, so x - P_x = 0\n- The function is equal to zero at -P, since -P and P share the same x coordinate\n- The function goes to infinity as x goes to infinity, so we say the function is equal to infinity at O. There’s a technical reason why this infinity needs to be counted twice, so O gets added with a “multiplicity” of -2 (negative because it’s an infinity and not a zero, two because of this double counting).\nThe technical reason is roughly this: because the equation of the curve is x³ = y² + b, y goes to infinity “1.5 times faster” than x does in order for y² to keep up with x³; hence, if a linear function includes only x then it is represented as an infinity of multiplicity 2, but if it includes y then it is represented as an infinity of multiplicity 3.\n\nWhere a, b and c are carefully chosen so that the line passes through points P and Q. Because of how elliptic curve addition works (see the diagram at the top), this also means that it passes through -P-Q. And it goes up to infinity dependent on both x and y, so the divisor becomes [P]+ [Q] + [-P-Q] - 3 * [O]. (multiplicity of 3 because it involves y)\n![ec_pariling](/images/cryptography/elliptic_curve/paring_ec.png)\nWe know that every “rational function” (ie. a function defined only using a finite number of +, -, * and / operations on the coordinates of the point) uniquely corresponds to some divisor, up to multiplication by a constant (ie. if two functions F and G have the same divisor, then F = G * k for some constant k).\nFor any two functions F and G, the divisor of F * G is equal to the divisor of F plus the divisor of G (in math textbooks, you’ll see (F * G) = (F) + (G)), so for example if f(x, y) = P_x - x, then (f³) = 3 * [P] + 3 * [-P] - 6 * [O]; P and -P are “triple-counted” to account for the fact that f³ approaches 0 at those points “three times as quickly” in a certain mathematical sense.\n\nNote that there is a theorem that states that if you “remove the square brackets” from a divisor of a function, the points must add up to O ([P] + [Q] + [-P-Q] - 3 * [O] clearly fits, as P + Q - P - Q - 3 * O = O), and any divisor that has this property is the divisor of a function.\n\n## Tate pairings\nConsider the following functions, defined via their divisors:\n- (F_P) = n * [P] - n * [O], where n is the order of G1, ie. n * P = O for any P\n- (F_Q) = n * [Q] - n * [O]\n- (g) = [P + Q] - [P] - [Q] + [O]\nNow, let’s look at the product F_P * F_Q * g^n. The divisor is:\n\nn * [P] - n * [O] + n * [Q] - n * [O] + n * [P + Q] - n * [P] - n * [Q] + n * [O]\n\nWhich simplifies neatly to:\n\nn * [P + Q] - n * [O]\nNotice that this divisor is of exactly the same format as the divisor for F_P and F_Q above. Hence, F_P * F_Q * g^n = F_(P + Q).\n\nNow, we introduce a procedure called the “final exponentiation” step, where we take the result of our functions above (F_P, F_Q, etc.) and raise it to the power z = (p¹² - 1) / n, where p¹² - 1 is the order of the multiplicative group in F_p¹² (ie. for any x ϵ F_p¹², x^(p¹² - 1) = 1). Notice that if you apply this exponentiation to any result that has already been raised to the power of n, you get an exponentiation to the power of p¹² - 1, so the result turns into 1. Hence, after final exponentiation, g^n cancels out and we get F_P^z * F_Q^z = F_(P + Q)^z. There’s some bilinearity for you.\nNow, if you want to make a function that’s bilinear in both arguments, you need to go into spookier math, where instead of taking F_P of a value directly, you take F_P of a divisor, and that’s where the full “Tate pairing” comes from. To prove some more results you have to deal with notions like “linear equivalence” and “Weil reciprocity”, and the rabbit hole goes on from there. You can find more reading material on all of this [2] and he[3].\nEvery elliptic curve has a value called an embedding degree; essentially, the smallest k such that p^k - 1 is a multiple of n (where p is the prime used for the field and n is the curve order). In the fields above, k = 12, and in the fields used for traditional ECC (ie. where we don’t care about pairings), the embedding degree is often extremely large, to the point that pairings are computationally infeasible to compute; however, if we are not careful then we can generate fields where k = 4 or even 1.\n\n## references\n- [1] [vitalik's blog on paring](https://medium.com/@VitalikButerin/exploring-elliptic-curve-pairings-c73c1864e627)\n- [2] [bilinear pairings in cryptography by Dennis Meffert](https://www.math.ru.nl/~bosma/Students/MScThesis_DennisMeffert.pdf)\n- [3] [Elliptic Curves number theory and cryptography by Kenneth H. Rosen](https://people.cs.nctu.edu.tw/~rjchen/ECC2012S/Elliptic%20Curves%20Number%20Theory%20And%20Cryptography%202n.pdf)\n- [4] [Miller's Algorithm](https://crypto.stanford.edu/pbc/notes/ep/miller.html)","source":"_posts/cryptography/elliptic_curve/elliptic-curve-pairing.md","raw":"---\ntitle: elliptic curve paring\ndate: 2023-06-27 14:29:26\ntags: [cryptography,ec]\n---\n\n\n\n## introduction\n\nPairings allow you to check certain kinds of more complicated equations on elliptic curve points — for example, if \\\\(P = G * p\\\\), \\\\(Q = G * q\\\\) and \\\\(R = G * r\\\\), you can check whether or not \\\\(p * q = r\\\\), having just P, Q and R as inputs. This might seem like the fundamental security guarantees of elliptic curves are being broken, as information about p is leaking from just knowing P, but it turns out that the leakage is highly contained — specifically, the ***decisional Diffie Hellman problem*** is easy, but the computational Diffie Hellman problem (knowing P and Q in the above example, computing R = G * p * q) and the discrete logarithm problem (recovering p from P) remain computationally infeasible.\n\n whereas traditional elliptic curve math lets you check linear constraints on the numbers (eg. if P = G * p, Q = G * q and R = G * r, checking 5 * P + 7 * Q = 11 * R is really checking that 5 * p + 7 * q = 11 * r), pairings let you check quadratic constraints (eg. checking e(P, Q) * e(G, G * 5) = 1 is really checking that p * q + 5 = 0).\n\n Now, what is this funny e(P, Q) operator that we introduced above? This is the pairing. Mathematicians also sometimes call it a bilinear map; the word “bilinear” here basically means that it satisfies the constraints:\n\n\\\\[ e(P, Q + R) = e(P, Q) * e(P, R) \\\\]\n\\\\[ e(P + S, Q) = e(P, Q) * e(S, Q) \\\\]\nNote that + and * can be arbitrary operators; when you’re creating fancy new kinds of mathematical objects, abstract algebra doesn’t care how + and * are defined, as long as they are consistent in the usual ways.\n\nIf P, Q, R and S were simple numbers, then making a simple pairing is easy: we can do e(x, y) = 2^xy. Then, we can see:\n\\\\[e(3, 4+ 5) = 2^(3 * 9) = 2^{27}\\\\]\n\\\\[e(3, 4) * e(3, 5) = 2^(3 * 4) * 2^(3 * 5) = 2^{12} * 2^{15} = 2^{27} \\\\]\nHowever, such simple pairings are not suitable for cryptography because the objects that they work on are simple integers and are too easy to analyze;\n\nIt turns out that it is possible to make a bilinear map over elliptic curve points — that is, come up with a function e(P, Q) where the inputs P and Q are elliptic curve points, and where the output is what’s called an \\\\(F_{p^¹²}\\\\) (extension field) element\n\nAn elliptic curve pairing is a map G2 x G1 -> Gt, where:\n- G1 is an elliptic curve, where points satisfy an equation of the form y² = x³ + b, and where both coordinates are elements of F_p (ie. they are simple numbers, except arithmetic is all done modulo some prime number)\n- G2 is an elliptic curve, where points satisfy the same equation as G1, except where the coordinates are elements of \\\\(F_{p^¹²}\\\\). we define a new “magic number” w, which is defined by a 12th degree polynomial like w^12 - 18 * w^6 + 82 = 0\n- Gt is the type of object that the result of the elliptic curve goes into. In the curves that we look at, Gt is \\\\(F_{p^¹²}\\\\)\nThe main property that it must satisfy is bilinearity, which in presented above\n\nThere are two other important criteria:\n- **Efficient computability** (eg. we can make an easy pairing by simply taking the discrete logarithms of all points and multiplying them together, but this is as computationally hard as breaking elliptic curve cryptography in the first place, so it doesn’t count)\n- **Non-degeneracy** (sure, you could just define e(P, Q) = 1, but that’s not a particularly useful pairing)\n\n## math intuition behind paring\nThe math behind why pairing functions work is quite tricky and involves quite a bit of advanced algebra going even beyond what we’ve seen so far, but I’ll provide an outline. First of all, we need to define the concept of a divisor, basically an alternative way of representing functions on elliptic curve points. A **divisor** of a function basically counts the zeroes and the infinities of the function. To see what this means, let’s go through a few examples. Let us fix some point P = (P_x, P_y), and consider the following function:\n\\\\[f(x, y) = x - P_x\\\\]\n\nThe divisor is [P] + [-P] - 2 * [O] (the square brackets are used to represent the fact that we are referring to the presence of the point P in the set of zeroes and infinities of the function, not the point P itself; [P] + [Q] is not the same thing as [P + Q]). The reasoning is as follows:\n- The function is equal to zero at P, since x is P_x, so x - P_x = 0\n- The function is equal to zero at -P, since -P and P share the same x coordinate\n- The function goes to infinity as x goes to infinity, so we say the function is equal to infinity at O. There’s a technical reason why this infinity needs to be counted twice, so O gets added with a “multiplicity” of -2 (negative because it’s an infinity and not a zero, two because of this double counting).\nThe technical reason is roughly this: because the equation of the curve is x³ = y² + b, y goes to infinity “1.5 times faster” than x does in order for y² to keep up with x³; hence, if a linear function includes only x then it is represented as an infinity of multiplicity 2, but if it includes y then it is represented as an infinity of multiplicity 3.\n\nWhere a, b and c are carefully chosen so that the line passes through points P and Q. Because of how elliptic curve addition works (see the diagram at the top), this also means that it passes through -P-Q. And it goes up to infinity dependent on both x and y, so the divisor becomes [P]+ [Q] + [-P-Q] - 3 * [O]. (multiplicity of 3 because it involves y)\n![ec_pariling](/images/cryptography/elliptic_curve/paring_ec.png)\nWe know that every “rational function” (ie. a function defined only using a finite number of +, -, * and / operations on the coordinates of the point) uniquely corresponds to some divisor, up to multiplication by a constant (ie. if two functions F and G have the same divisor, then F = G * k for some constant k).\nFor any two functions F and G, the divisor of F * G is equal to the divisor of F plus the divisor of G (in math textbooks, you’ll see (F * G) = (F) + (G)), so for example if f(x, y) = P_x - x, then (f³) = 3 * [P] + 3 * [-P] - 6 * [O]; P and -P are “triple-counted” to account for the fact that f³ approaches 0 at those points “three times as quickly” in a certain mathematical sense.\n\nNote that there is a theorem that states that if you “remove the square brackets” from a divisor of a function, the points must add up to O ([P] + [Q] + [-P-Q] - 3 * [O] clearly fits, as P + Q - P - Q - 3 * O = O), and any divisor that has this property is the divisor of a function.\n\n## Tate pairings\nConsider the following functions, defined via their divisors:\n- (F_P) = n * [P] - n * [O], where n is the order of G1, ie. n * P = O for any P\n- (F_Q) = n * [Q] - n * [O]\n- (g) = [P + Q] - [P] - [Q] + [O]\nNow, let’s look at the product F_P * F_Q * g^n. The divisor is:\n\nn * [P] - n * [O] + n * [Q] - n * [O] + n * [P + Q] - n * [P] - n * [Q] + n * [O]\n\nWhich simplifies neatly to:\n\nn * [P + Q] - n * [O]\nNotice that this divisor is of exactly the same format as the divisor for F_P and F_Q above. Hence, F_P * F_Q * g^n = F_(P + Q).\n\nNow, we introduce a procedure called the “final exponentiation” step, where we take the result of our functions above (F_P, F_Q, etc.) and raise it to the power z = (p¹² - 1) / n, where p¹² - 1 is the order of the multiplicative group in F_p¹² (ie. for any x ϵ F_p¹², x^(p¹² - 1) = 1). Notice that if you apply this exponentiation to any result that has already been raised to the power of n, you get an exponentiation to the power of p¹² - 1, so the result turns into 1. Hence, after final exponentiation, g^n cancels out and we get F_P^z * F_Q^z = F_(P + Q)^z. There’s some bilinearity for you.\nNow, if you want to make a function that’s bilinear in both arguments, you need to go into spookier math, where instead of taking F_P of a value directly, you take F_P of a divisor, and that’s where the full “Tate pairing” comes from. To prove some more results you have to deal with notions like “linear equivalence” and “Weil reciprocity”, and the rabbit hole goes on from there. You can find more reading material on all of this [2] and he[3].\nEvery elliptic curve has a value called an embedding degree; essentially, the smallest k such that p^k - 1 is a multiple of n (where p is the prime used for the field and n is the curve order). In the fields above, k = 12, and in the fields used for traditional ECC (ie. where we don’t care about pairings), the embedding degree is often extremely large, to the point that pairings are computationally infeasible to compute; however, if we are not careful then we can generate fields where k = 4 or even 1.\n\n## references\n- [1] [vitalik's blog on paring](https://medium.com/@VitalikButerin/exploring-elliptic-curve-pairings-c73c1864e627)\n- [2] [bilinear pairings in cryptography by Dennis Meffert](https://www.math.ru.nl/~bosma/Students/MScThesis_DennisMeffert.pdf)\n- [3] [Elliptic Curves number theory and cryptography by Kenneth H. Rosen](https://people.cs.nctu.edu.tw/~rjchen/ECC2012S/Elliptic%20Curves%20Number%20Theory%20And%20Cryptography%202n.pdf)\n- [4] [Miller's Algorithm](https://crypto.stanford.edu/pbc/notes/ep/miller.html)","slug":"cryptography/elliptic_curve/elliptic-curve-pairing","published":1,"updated":"2023-12-01T02:50:40.735Z","_id":"clokyy8e1003gqwsj9p1scg9k","comments":1,"layout":"post","photos":[],"link":"","content":"\n\n\n

introduction

Pairings allow you to check certain kinds of more complicated equations on elliptic curve points — for example, if \\(P = G * p\\), \\(Q = G * q\\) and \\(R = G * r\\), you can check whether or not \\(p * q = r\\), having just P, Q and R as inputs. This might seem like the fundamental security guarantees of elliptic curves are being broken, as information about p is leaking from just knowing P, but it turns out that the leakage is highly contained — specifically, the decisional Diffie Hellman problem is easy, but the computational Diffie Hellman problem (knowing P and Q in the above example, computing R = G * p * q) and the discrete logarithm problem (recovering p from P) remain computationally infeasible.

\n

whereas traditional elliptic curve math lets you check linear constraints on the numbers (eg. if P = G * p, Q = G * q and R = G * r, checking 5 * P + 7 * Q = 11 * R is really checking that 5 * p + 7 * q = 11 * r), pairings let you check quadratic constraints (eg. checking e(P, Q) * e(G, G * 5) = 1 is really checking that p * q + 5 = 0).

\n

Now, what is this funny e(P, Q) operator that we introduced above? This is the pairing. Mathematicians also sometimes call it a bilinear map; the word “bilinear” here basically means that it satisfies the constraints:

\n

\\[ e(P, Q + R) = e(P, Q) * e(P, R) \\]
\\[ e(P + S, Q) = e(P, Q) * e(S, Q) \\]
Note that + and * can be arbitrary operators; when you’re creating fancy new kinds of mathematical objects, abstract algebra doesn’t care how + and * are defined, as long as they are consistent in the usual ways.

\n

If P, Q, R and S were simple numbers, then making a simple pairing is easy: we can do e(x, y) = 2^xy. Then, we can see:
\\[e(3, 4+ 5) = 2^(3 * 9) = 2^{27}\\]
\\[e(3, 4) * e(3, 5) = 2^(3 * 4) * 2^(3 * 5) = 2^{12} * 2^{15} = 2^{27} \\]
However, such simple pairings are not suitable for cryptography because the objects that they work on are simple integers and are too easy to analyze;

\n

It turns out that it is possible to make a bilinear map over elliptic curve points — that is, come up with a function e(P, Q) where the inputs P and Q are elliptic curve points, and where the output is what’s called an \\(F_{p^¹²}\\) (extension field) element

\n

An elliptic curve pairing is a map G2 x G1 -> Gt, where:

\n
    \n
  • G1 is an elliptic curve, where points satisfy an equation of the form y² = x³ + b, and where both coordinates are elements of F_p (ie. they are simple numbers, except arithmetic is all done modulo some prime number)
    • \n
  • G2 is an elliptic curve, where points satisfy the same equation as G1, except where the coordinates are elements of \\(F_{p^¹²}\\). we define a new “magic number” w, which is defined by a 12th degree polynomial like w^12 - 18 * w^6 + 82 = 0
    • \n
  • Gt is the type of object that the result of the elliptic curve goes into. In the curves that we look at, Gt is \\(F_{p^¹²}\\)
    The main property that it must satisfy is bilinearity, which in presented above
    • \n
\n

There are two other important criteria:

\n
    \n
  • Efficient computability (eg. we can make an easy pairing by simply taking the discrete logarithms of all points and multiplying them together, but this is as computationally hard as breaking elliptic curve cryptography in the first place, so it doesn’t count)
    • \n
  • Non-degeneracy (sure, you could just define e(P, Q) = 1, but that’s not a particularly useful pairing)
    • \n
\n

math intuition behind paring

The math behind why pairing functions work is quite tricky and involves quite a bit of advanced algebra going even beyond what we’ve seen so far, but I’ll provide an outline. First of all, we need to define the concept of a divisor, basically an alternative way of representing functions on elliptic curve points. A divisor of a function basically counts the zeroes and the infinities of the function. To see what this means, let’s go through a few examples. Let us fix some point P = (P_x, P_y), and consider the following function:
\\[f(x, y) = x - P_x\\]

\n

The divisor is [P] + [-P] - 2 * [O] (the square brackets are used to represent the fact that we are referring to the presence of the point P in the set of zeroes and infinities of the function, not the point P itself; [P] + [Q] is not the same thing as [P + Q]). The reasoning is as follows:

\n
    \n
  • The function is equal to zero at P, since x is P_x, so x - P_x = 0
    • \n
  • The function is equal to zero at -P, since -P and P share the same x coordinate
    • \n
  • The function goes to infinity as x goes to infinity, so we say the function is equal to infinity at O. There’s a technical reason why this infinity needs to be counted twice, so O gets added with a “multiplicity” of -2 (negative because it’s an infinity and not a zero, two because of this double counting).
    The technical reason is roughly this: because the equation of the curve is x³ = y² + b, y goes to infinity “1.5 times faster” than x does in order for y² to keep up with x³; hence, if a linear function includes only x then it is represented as an infinity of multiplicity 2, but if it includes y then it is represented as an infinity of multiplicity 3.
    • \n
\n

Where a, b and c are carefully chosen so that the line passes through points P and Q. Because of how elliptic curve addition works (see the diagram at the top), this also means that it passes through -P-Q. And it goes up to infinity dependent on both x and y, so the divisor becomes [P]+ [Q] + [-P-Q] - 3 * [O]. (multiplicity of 3 because it involves y)
\"ec_pariling\"
We know that every “rational function” (ie. a function defined only using a finite number of +, -, * and / operations on the coordinates of the point) uniquely corresponds to some divisor, up to multiplication by a constant (ie. if two functions F and G have the same divisor, then F = G * k for some constant k).
For any two functions F and G, the divisor of F * G is equal to the divisor of F plus the divisor of G (in math textbooks, you’ll see (F * G) = (F) + (G)), so for example if f(x, y) = P_x - x, then (f³) = 3 * [P] + 3 * [-P] - 6 * [O]; P and -P are “triple-counted” to account for the fact that f³ approaches 0 at those points “three times as quickly” in a certain mathematical sense.

\n

Note that there is a theorem that states that if you “remove the square brackets” from a divisor of a function, the points must add up to O ([P] + [Q] + [-P-Q] - 3 * [O] clearly fits, as P + Q - P - Q - 3 * O = O), and any divisor that has this property is the divisor of a function.

\n

Tate pairings

Consider the following functions, defined via their divisors:

\n
    \n
  • (F_P) = n * [P] - n * [O], where n is the order of G1, ie. n * P = O for any P
    • \n
  • (F_Q) = n * [Q] - n * [O]
    • \n
  • (g) = [P + Q] - [P] - [Q] + [O]
    Now, let’s look at the product F_P * F_Q * g^n. The divisor is:
    • \n
\n

n * [P] - n * [O] + n * [Q] - n * [O] + n * [P + Q] - n * [P] - n * [Q] + n * [O]

\n

Which simplifies neatly to:

\n

n * [P + Q] - n * [O]
Notice that this divisor is of exactly the same format as the divisor for F_P and F_Q above. Hence, F_P * F_Q * g^n = F_(P + Q).

\n

Now, we introduce a procedure called the “final exponentiation” step, where we take the result of our functions above (F_P, F_Q, etc.) and raise it to the power z = (p¹² - 1) / n, where p¹² - 1 is the order of the multiplicative group in F_p¹² (ie. for any x ϵ F_p¹², x^(p¹² - 1) = 1). Notice that if you apply this exponentiation to any result that has already been raised to the power of n, you get an exponentiation to the power of p¹² - 1, so the result turns into 1. Hence, after final exponentiation, g^n cancels out and we get F_P^z * F_Q^z = F_(P + Q)^z. There’s some bilinearity for you.
Now, if you want to make a function that’s bilinear in both arguments, you need to go into spookier math, where instead of taking F_P of a value directly, you take F_P of a divisor, and that’s where the full “Tate pairing” comes from. To prove some more results you have to deal with notions like “linear equivalence” and “Weil reciprocity”, and the rabbit hole goes on from there. You can find more reading material on all of this [2] and he[3].
Every elliptic curve has a value called an embedding degree; essentially, the smallest k such that p^k - 1 is a multiple of n (where p is the prime used for the field and n is the curve order). In the fields above, k = 12, and in the fields used for traditional ECC (ie. where we don’t care about pairings), the embedding degree is often extremely large, to the point that pairings are computationally infeasible to compute; however, if we are not careful then we can generate fields where k = 4 or even 1.

\n

references

\n","site":{"data":{}},"excerpt":"","more":"\n\n\n

introduction

Pairings allow you to check certain kinds of more complicated equations on elliptic curve points — for example, if \\(P = G * p\\), \\(Q = G * q\\) and \\(R = G * r\\), you can check whether or not \\(p * q = r\\), having just P, Q and R as inputs. This might seem like the fundamental security guarantees of elliptic curves are being broken, as information about p is leaking from just knowing P, but it turns out that the leakage is highly contained — specifically, the decisional Diffie Hellman problem is easy, but the computational Diffie Hellman problem (knowing P and Q in the above example, computing R = G * p * q) and the discrete logarithm problem (recovering p from P) remain computationally infeasible.

\n

whereas traditional elliptic curve math lets you check linear constraints on the numbers (eg. if P = G * p, Q = G * q and R = G * r, checking 5 * P + 7 * Q = 11 * R is really checking that 5 * p + 7 * q = 11 * r), pairings let you check quadratic constraints (eg. checking e(P, Q) * e(G, G * 5) = 1 is really checking that p * q + 5 = 0).

\n

Now, what is this funny e(P, Q) operator that we introduced above? This is the pairing. Mathematicians also sometimes call it a bilinear map; the word “bilinear” here basically means that it satisfies the constraints:

\n

\\[ e(P, Q + R) = e(P, Q) * e(P, R) \\]
\\[ e(P + S, Q) = e(P, Q) * e(S, Q) \\]
Note that + and * can be arbitrary operators; when you’re creating fancy new kinds of mathematical objects, abstract algebra doesn’t care how + and * are defined, as long as they are consistent in the usual ways.

\n

If P, Q, R and S were simple numbers, then making a simple pairing is easy: we can do e(x, y) = 2^xy. Then, we can see:
\\[e(3, 4+ 5) = 2^(3 * 9) = 2^{27}\\]
\\[e(3, 4) * e(3, 5) = 2^(3 * 4) * 2^(3 * 5) = 2^{12} * 2^{15} = 2^{27} \\]
However, such simple pairings are not suitable for cryptography because the objects that they work on are simple integers and are too easy to analyze;

\n

It turns out that it is possible to make a bilinear map over elliptic curve points — that is, come up with a function e(P, Q) where the inputs P and Q are elliptic curve points, and where the output is what’s called an \\(F_{p^¹²}\\) (extension field) element

\n

An elliptic curve pairing is a map G2 x G1 -> Gt, where:

\n
    \n
  • G1 is an elliptic curve, where points satisfy an equation of the form y² = x³ + b, and where both coordinates are elements of F_p (ie. they are simple numbers, except arithmetic is all done modulo some prime number)
    • \n
  • G2 is an elliptic curve, where points satisfy the same equation as G1, except where the coordinates are elements of \\(F_{p^¹²}\\). we define a new “magic number” w, which is defined by a 12th degree polynomial like w^12 - 18 * w^6 + 82 = 0
    • \n
  • Gt is the type of object that the result of the elliptic curve goes into. In the curves that we look at, Gt is \\(F_{p^¹²}\\)
    The main property that it must satisfy is bilinearity, which in presented above
    • \n
\n

There are two other important criteria:

\n
    \n
  • Efficient computability (eg. we can make an easy pairing by simply taking the discrete logarithms of all points and multiplying them together, but this is as computationally hard as breaking elliptic curve cryptography in the first place, so it doesn’t count)
    • \n
  • Non-degeneracy (sure, you could just define e(P, Q) = 1, but that’s not a particularly useful pairing)
    • \n
\n

math intuition behind paring

The math behind why pairing functions work is quite tricky and involves quite a bit of advanced algebra going even beyond what we’ve seen so far, but I’ll provide an outline. First of all, we need to define the concept of a divisor, basically an alternative way of representing functions on elliptic curve points. A divisor of a function basically counts the zeroes and the infinities of the function. To see what this means, let’s go through a few examples. Let us fix some point P = (P_x, P_y), and consider the following function:
\\[f(x, y) = x - P_x\\]

\n

The divisor is [P] + [-P] - 2 * [O] (the square brackets are used to represent the fact that we are referring to the presence of the point P in the set of zeroes and infinities of the function, not the point P itself; [P] + [Q] is not the same thing as [P + Q]). The reasoning is as follows:

\n
    \n
  • The function is equal to zero at P, since x is P_x, so x - P_x = 0
    • \n
  • The function is equal to zero at -P, since -P and P share the same x coordinate
    • \n
  • The function goes to infinity as x goes to infinity, so we say the function is equal to infinity at O. There’s a technical reason why this infinity needs to be counted twice, so O gets added with a “multiplicity” of -2 (negative because it’s an infinity and not a zero, two because of this double counting).
    The technical reason is roughly this: because the equation of the curve is x³ = y² + b, y goes to infinity “1.5 times faster” than x does in order for y² to keep up with x³; hence, if a linear function includes only x then it is represented as an infinity of multiplicity 2, but if it includes y then it is represented as an infinity of multiplicity 3.
    • \n
\n

Where a, b and c are carefully chosen so that the line passes through points P and Q. Because of how elliptic curve addition works (see the diagram at the top), this also means that it passes through -P-Q. And it goes up to infinity dependent on both x and y, so the divisor becomes [P]+ [Q] + [-P-Q] - 3 * [O]. (multiplicity of 3 because it involves y)
\"ec_pariling\"
We know that every “rational function” (ie. a function defined only using a finite number of +, -, * and / operations on the coordinates of the point) uniquely corresponds to some divisor, up to multiplication by a constant (ie. if two functions F and G have the same divisor, then F = G * k for some constant k).
For any two functions F and G, the divisor of F * G is equal to the divisor of F plus the divisor of G (in math textbooks, you’ll see (F * G) = (F) + (G)), so for example if f(x, y) = P_x - x, then (f³) = 3 * [P] + 3 * [-P] - 6 * [O]; P and -P are “triple-counted” to account for the fact that f³ approaches 0 at those points “three times as quickly” in a certain mathematical sense.

\n

Note that there is a theorem that states that if you “remove the square brackets” from a divisor of a function, the points must add up to O ([P] + [Q] + [-P-Q] - 3 * [O] clearly fits, as P + Q - P - Q - 3 * O = O), and any divisor that has this property is the divisor of a function.

\n

Tate pairings

Consider the following functions, defined via their divisors:

\n
    \n
  • (F_P) = n * [P] - n * [O], where n is the order of G1, ie. n * P = O for any P
    • \n
  • (F_Q) = n * [Q] - n * [O]
    • \n
  • (g) = [P + Q] - [P] - [Q] + [O]
    Now, let’s look at the product F_P * F_Q * g^n. The divisor is:
    • \n
\n

n * [P] - n * [O] + n * [Q] - n * [O] + n * [P + Q] - n * [P] - n * [Q] + n * [O]

\n

Which simplifies neatly to:

\n

n * [P + Q] - n * [O]
Notice that this divisor is of exactly the same format as the divisor for F_P and F_Q above. Hence, F_P * F_Q * g^n = F_(P + Q).

\n

Now, we introduce a procedure called the “final exponentiation” step, where we take the result of our functions above (F_P, F_Q, etc.) and raise it to the power z = (p¹² - 1) / n, where p¹² - 1 is the order of the multiplicative group in F_p¹² (ie. for any x ϵ F_p¹², x^(p¹² - 1) = 1). Notice that if you apply this exponentiation to any result that has already been raised to the power of n, you get an exponentiation to the power of p¹² - 1, so the result turns into 1. Hence, after final exponentiation, g^n cancels out and we get F_P^z * F_Q^z = F_(P + Q)^z. There’s some bilinearity for you.
Now, if you want to make a function that’s bilinear in both arguments, you need to go into spookier math, where instead of taking F_P of a value directly, you take F_P of a divisor, and that’s where the full “Tate pairing” comes from. To prove some more results you have to deal with notions like “linear equivalence” and “Weil reciprocity”, and the rabbit hole goes on from there. You can find more reading material on all of this [2] and he[3].
Every elliptic curve has a value called an embedding degree; essentially, the smallest k such that p^k - 1 is a multiple of n (where p is the prime used for the field and n is the curve order). In the fields above, k = 12, and in the fields used for traditional ECC (ie. where we don’t care about pairings), the embedding degree is often extremely large, to the point that pairings are computationally infeasible to compute; however, if we are not careful then we can generate fields where k = 4 or even 1.

\n

references

\n"},{"title":"zkp how and why it works","date":"2023-07-01T06:29:26.000Z","_content":"\n\n\n## The medium of proof: Polynomial\nIf a prover claims to know some polynomial (no matter how large its degree is) that the verifier also knows, they can follow a simple protocol to verify the statement\n• Verifier chooses a random value for x and evaluates his polynomial locally\n• Verifier gives x to the prover and asks to evaluate the polynomial in question\n• Prover evaluates his polynomial at x and gives the result to the verifier\n• Verifier checks if the local result is equal to the prover’s result, and if so then the statement is proven with a high confidence\n\n## Non-Interactive Zero-Knowledge of a Polynomial\n1. Proving Knowledge of a Polynomial\nA polynomial can be expressed in the form (where n is the degree of the polynomial):\n\\\\[c_n x^n + ...+ c_1 x^1 + c_0 x^0\\\\]\nIt one claims that he know a polynomial, it is actually the knowledge of the polynomial's coefficients.\n\n2. Factorization\nThe Fundamental Theorem of Algebra states that any polynomial can be factored into linear po- lynomials (i.e., a degree 1 polynomials representing a line), as long it is solvable. Consequently, we can represent any valid polynomial as a product of its factors:\n\\\\[(x-a_0)(x-a_1)...(x-a_n) = 0\\\\]\n\nif the prover wants to prove that indeed his polynomial has specific roots without disclosing the polynomial itself, he needs to prove that his polynomial \\\\(p(x)\\\\) is the multiplication of those cofactors \\\\(t(x) = (x − x_0)(x − x_1)\\\\), called target polynomial, where \\\\(x_0, x_1\\\\) are the specific roots. i.e.:\n\\\\[p(x) = t(x) \\cdot h(x)\\\\]\nA natural way to find \\\\(h(x)\\\\) is through the division \\\\(h(x) = \\frac{p(x)}{t(x)}\\\\)\n> for simplicity, onwards we will use polynomial’s letter variable to denote its evaluation, e.g., \\\\(p = p(r)\\\\)\n\nUsing our polynomial identity check protocol we can compare polynomials \\\\(p(x)\\\\) and \\\\(t(x)·h(x)\\\\):\n- Verifier samples a random value \\\\(r\\\\), calculates \\\\(t = t(r)\\\\) (i.e., evaluates) and gives \\\\(r\\\\) to the prover\n- Prover calculates \\\\(h(x) = \\frac{p(x)}{t(x)}\\\\) and evaluates \\\\(p(r)\\\\) and \\\\(h(r)\\\\); the resulting values \\\\(p,h\\\\) are\nprovided to the verifier\n- Verifier then checks that \\\\(p = t \\cdot h\\\\), if so those polynomials are equal, meaning that \\\\(p(x)\\\\) has \\\\(t(x)\\\\) as a cofactor.\n\n**Remark 3.1** Now we can check a polynomial for specific properties without learning the polyno- mial itself, so this already gives us some form of zero-knowledge and succinctness. Nonetheless, there are multiple issues with this construction:\n• Prover may not know the claimed polynomial \\\\(p(x)\\\\) at all. He can calculate evaluation \\\\(t = t(r)\\\\), select a random number \\\\(h\\\\) and set \\\\(p = t \\cdot h\\\\), which will be accepted by the verifier as valid, since equation holds.\n• Because prover knows the random point \\\\(x = r\\\\), he can construct any polynomial which has one shared point at \\\\(r\\\\) with \\\\(t(r) \\cdot h(r)\\\\).\n• In the original statement, prover claims to know a polynomial of a particular degree, in the current protocol there is no enforcement of degree. Hence prover can cheat by using a polynomial of higher degree which also satisfies the cofactors check.\n\n3. Obscure Evaluation\nTwo first issues of remark 3.1 are possible because values are presented at raw, prover knows r and t(r). It would be ideal if those values would be given as a black box, so one cannot temper with the protocol, but still able to compute operations on those obscure values. \n3.1. Homomorphic Encryption\nThere are multiple ways to achieve homomorphic properties of encryption, and we will briefly introduce a simple one. The general idea is to choose a base number \\\\(g\\\\) and do ecncryption of a value \\\\(x\\\\) by exponentiate of \\\\(g\\\\)\n\\\\[E(x) = g^{x} \\bmod p\\\\]\nFor example, let \\\\(E(x_1) = g^{x_1} \\bmod p\\\\), and \\\\(E(x_2) = g^{x_2} \\bmod p\\\\), then\n\\\\[E(x_2) \\cdot E(x_1) = g^{x_1 + x_2} = E(x_1 + x_2) \\\\]\n\n3.2. Encrypted Polynomial\nLet us see how we can evaluate a polynomial \\\\(p(x) = x^3 − 3x^2 + 2x\\\\). Because homomorphic encryption does not allows to exponentiate an encrypted value, we’ve must been given encrypted values of powers of x from 1 to 3: \\\\(E(x),E(x2),E(x3)\\\\) so that\n\\\\[ E(x^3)^1 \\cdot E(x^2)^{-3} \\cdot E(x)^2 = (g^{x^3})^{1} \\cdot (g^{x^2})^{-3} \\cdot (g^{x})^{2} = g^{1x^3} \\cdot g^{-3x^2} \\cdot g^{2x} = g^{x^3-3x^2+2x}\\\\]\nHence, we have an encrypted evaluation of our polynomial at some unknown to us \\\\(x\\\\) \n\nWe can now update the previous version of the protocol, for a polynomial fo degree \\\\(d\\\\):\n- Verifier\n - samples a random value \\\\(s\\\\), i.e., secret\n - calculates encryptions of \\\\(s\\\\) for all powers \\\\(i\\\\) in \\\\(0,1,...,d\\\\), i.e. : \\\\(E(s^{i}) = g^{s^{i}}\\\\)\n - evaluates unencrypted target polynomial with \\\\(s: t(s)\\\\)\n - encrypted powers of \\\\(s\\\\) are provided to the prover: \\\\(E(s^{0}),E(s^{1}),...,E(s^{d})\\\\)\n- Prover\n - calculates polynomial \\\\(h(x) = \\frac{p(x)}{t(x)}\\\\)\n - using encrypted powers \\\\(g^{s^{0}},g^{s^{1}},...,g^{s^{d}}\\\\) and coefficients \\\\(c_0, c_1,...,c_n \\\\) evaluates \\\\(E(p(s)) = g^{p(s)} = (g^{s^{d}})^{c_d} \\cdot \\cdot \\cdot (g^{s^{1}})^{c_1} \\cdot (g^{s^{0}})^{c_0}\\\\) and similarly \\\\(E(h(s)) = g^{h(s)}\\\\)\n - the resulting \\\\(g^p\\\\) and \\\\(g^h\\\\) are provided to the verifier\n- Verifier\n - The alst step for the verifier is to checks that \\\\(p = t(s) \\cdot h \\\\) in encrypted space: \\\\(g^p = (g^h)^{t(s)}\\\\) => \\\\(g^p = g^{t(s) \\cdot h}\\\\)\n\n> Note: because the prover does not know anything about s, it makes it hard to come up with non-legitimate but still matching evaluations.\n## referneces\n- [why and how zk-SNARK works by Maksym](https://arxiv.org/pdf/1906.07221.pdf)","source":"_posts/cryptography/zkp/zkp-under-the-hood.md","raw":"---\ntitle: zkp how and why it works\ndate: 2023-07-01 14:29:26\ntags: [cryptography,zkp]\n---\n\n\n\n## The medium of proof: Polynomial\nIf a prover claims to know some polynomial (no matter how large its degree is) that the verifier also knows, they can follow a simple protocol to verify the statement\n• Verifier chooses a random value for x and evaluates his polynomial locally\n• Verifier gives x to the prover and asks to evaluate the polynomial in question\n• Prover evaluates his polynomial at x and gives the result to the verifier\n• Verifier checks if the local result is equal to the prover’s result, and if so then the statement is proven with a high confidence\n\n## Non-Interactive Zero-Knowledge of a Polynomial\n1. Proving Knowledge of a Polynomial\nA polynomial can be expressed in the form (where n is the degree of the polynomial):\n\\\\[c_n x^n + ...+ c_1 x^1 + c_0 x^0\\\\]\nIt one claims that he know a polynomial, it is actually the knowledge of the polynomial's coefficients.\n\n2. Factorization\nThe Fundamental Theorem of Algebra states that any polynomial can be factored into linear po- lynomials (i.e., a degree 1 polynomials representing a line), as long it is solvable. Consequently, we can represent any valid polynomial as a product of its factors:\n\\\\[(x-a_0)(x-a_1)...(x-a_n) = 0\\\\]\n\nif the prover wants to prove that indeed his polynomial has specific roots without disclosing the polynomial itself, he needs to prove that his polynomial \\\\(p(x)\\\\) is the multiplication of those cofactors \\\\(t(x) = (x − x_0)(x − x_1)\\\\), called target polynomial, where \\\\(x_0, x_1\\\\) are the specific roots. i.e.:\n\\\\[p(x) = t(x) \\cdot h(x)\\\\]\nA natural way to find \\\\(h(x)\\\\) is through the division \\\\(h(x) = \\frac{p(x)}{t(x)}\\\\)\n> for simplicity, onwards we will use polynomial’s letter variable to denote its evaluation, e.g., \\\\(p = p(r)\\\\)\n\nUsing our polynomial identity check protocol we can compare polynomials \\\\(p(x)\\\\) and \\\\(t(x)·h(x)\\\\):\n- Verifier samples a random value \\\\(r\\\\), calculates \\\\(t = t(r)\\\\) (i.e., evaluates) and gives \\\\(r\\\\) to the prover\n- Prover calculates \\\\(h(x) = \\frac{p(x)}{t(x)}\\\\) and evaluates \\\\(p(r)\\\\) and \\\\(h(r)\\\\); the resulting values \\\\(p,h\\\\) are\nprovided to the verifier\n- Verifier then checks that \\\\(p = t \\cdot h\\\\), if so those polynomials are equal, meaning that \\\\(p(x)\\\\) has \\\\(t(x)\\\\) as a cofactor.\n\n**Remark 3.1** Now we can check a polynomial for specific properties without learning the polyno- mial itself, so this already gives us some form of zero-knowledge and succinctness. Nonetheless, there are multiple issues with this construction:\n• Prover may not know the claimed polynomial \\\\(p(x)\\\\) at all. He can calculate evaluation \\\\(t = t(r)\\\\), select a random number \\\\(h\\\\) and set \\\\(p = t \\cdot h\\\\), which will be accepted by the verifier as valid, since equation holds.\n• Because prover knows the random point \\\\(x = r\\\\), he can construct any polynomial which has one shared point at \\\\(r\\\\) with \\\\(t(r) \\cdot h(r)\\\\).\n• In the original statement, prover claims to know a polynomial of a particular degree, in the current protocol there is no enforcement of degree. Hence prover can cheat by using a polynomial of higher degree which also satisfies the cofactors check.\n\n3. Obscure Evaluation\nTwo first issues of remark 3.1 are possible because values are presented at raw, prover knows r and t(r). It would be ideal if those values would be given as a black box, so one cannot temper with the protocol, but still able to compute operations on those obscure values. \n3.1. Homomorphic Encryption\nThere are multiple ways to achieve homomorphic properties of encryption, and we will briefly introduce a simple one. The general idea is to choose a base number \\\\(g\\\\) and do ecncryption of a value \\\\(x\\\\) by exponentiate of \\\\(g\\\\)\n\\\\[E(x) = g^{x} \\bmod p\\\\]\nFor example, let \\\\(E(x_1) = g^{x_1} \\bmod p\\\\), and \\\\(E(x_2) = g^{x_2} \\bmod p\\\\), then\n\\\\[E(x_2) \\cdot E(x_1) = g^{x_1 + x_2} = E(x_1 + x_2) \\\\]\n\n3.2. Encrypted Polynomial\nLet us see how we can evaluate a polynomial \\\\(p(x) = x^3 − 3x^2 + 2x\\\\). Because homomorphic encryption does not allows to exponentiate an encrypted value, we’ve must been given encrypted values of powers of x from 1 to 3: \\\\(E(x),E(x2),E(x3)\\\\) so that\n\\\\[ E(x^3)^1 \\cdot E(x^2)^{-3} \\cdot E(x)^2 = (g^{x^3})^{1} \\cdot (g^{x^2})^{-3} \\cdot (g^{x})^{2} = g^{1x^3} \\cdot g^{-3x^2} \\cdot g^{2x} = g^{x^3-3x^2+2x}\\\\]\nHence, we have an encrypted evaluation of our polynomial at some unknown to us \\\\(x\\\\) \n\nWe can now update the previous version of the protocol, for a polynomial fo degree \\\\(d\\\\):\n- Verifier\n - samples a random value \\\\(s\\\\), i.e., secret\n - calculates encryptions of \\\\(s\\\\) for all powers \\\\(i\\\\) in \\\\(0,1,...,d\\\\), i.e. : \\\\(E(s^{i}) = g^{s^{i}}\\\\)\n - evaluates unencrypted target polynomial with \\\\(s: t(s)\\\\)\n - encrypted powers of \\\\(s\\\\) are provided to the prover: \\\\(E(s^{0}),E(s^{1}),...,E(s^{d})\\\\)\n- Prover\n - calculates polynomial \\\\(h(x) = \\frac{p(x)}{t(x)}\\\\)\n - using encrypted powers \\\\(g^{s^{0}},g^{s^{1}},...,g^{s^{d}}\\\\) and coefficients \\\\(c_0, c_1,...,c_n \\\\) evaluates \\\\(E(p(s)) = g^{p(s)} = (g^{s^{d}})^{c_d} \\cdot \\cdot \\cdot (g^{s^{1}})^{c_1} \\cdot (g^{s^{0}})^{c_0}\\\\) and similarly \\\\(E(h(s)) = g^{h(s)}\\\\)\n - the resulting \\\\(g^p\\\\) and \\\\(g^h\\\\) are provided to the verifier\n- Verifier\n - The alst step for the verifier is to checks that \\\\(p = t(s) \\cdot h \\\\) in encrypted space: \\\\(g^p = (g^h)^{t(s)}\\\\) => \\\\(g^p = g^{t(s) \\cdot h}\\\\)\n\n> Note: because the prover does not know anything about s, it makes it hard to come up with non-legitimate but still matching evaluations.\n## referneces\n- [why and how zk-SNARK works by Maksym](https://arxiv.org/pdf/1906.07221.pdf)","slug":"cryptography/zkp/zkp-under-the-hood","published":1,"updated":"2023-11-05T04:21:17.035Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8e2003iqwsj23zw9035","content":"\n\n\n

The medium of proof: Polynomial

If a prover claims to know some polynomial (no matter how large its degree is) that the verifier also knows, they can follow a simple protocol to verify the statement
• Verifier chooses a random value for x and evaluates his polynomial locally
• Verifier gives x to the prover and asks to evaluate the polynomial in question
• Prover evaluates his polynomial at x and gives the result to the verifier
• Verifier checks if the local result is equal to the prover’s result, and if so then the statement is proven with a high confidence

\n

Non-Interactive Zero-Knowledge of a Polynomial

    \n

  1. Proving Knowledge of a Polynomial
    A polynomial can be expressed in the form (where n is the degree of the polynomial):
    \\[c_n x^n + …+ c_1 x^1 + c_0 x^0\\]
    It one claims that he know a polynomial, it is actually the knowledge of the polynomial’s coefficients.

    \n
    2. \n

  2. Factorization
    The Fundamental Theorem of Algebra states that any polynomial can be factored into linear po- lynomials (i.e., a degree 1 polynomials representing a line), as long it is solvable. Consequently, we can represent any valid polynomial as a product of its factors:
    \\[(x-a_0)(x-a_1)…(x-a_n) = 0\\]

    \n
    3. \n
\n

if the prover wants to prove that indeed his polynomial has specific roots without disclosing the polynomial itself, he needs to prove that his polynomial \\(p(x)\\) is the multiplication of those cofactors \\(t(x) = (x − x_0)(x − x_1)\\), called target polynomial, where \\(x_0, x_1\\) are the specific roots. i.e.:
\\[p(x) = t(x) \\cdot h(x)\\]
A natural way to find \\(h(x)\\) is through the division \\(h(x) = \\frac{p(x)}{t(x)}\\)

\n
\n

for simplicity, onwards we will use polynomial’s letter variable to denote its evaluation, e.g., \\(p = p(r)\\)

\n
\n

Using our polynomial identity check protocol we can compare polynomials \\(p(x)\\) and \\(t(x)·h(x)\\):

\n
    \n
  • Verifier samples a random value \\(r\\), calculates \\(t = t(r)\\) (i.e., evaluates) and gives \\(r\\) to the prover
    • \n
  • Prover calculates \\(h(x) = \\frac{p(x)}{t(x)}\\) and evaluates \\(p(r)\\) and \\(h(r)\\); the resulting values \\(p,h\\) are
    provided to the verifier
    • \n
  • Verifier then checks that \\(p = t \\cdot h\\), if so those polynomials are equal, meaning that \\(p(x)\\) has \\(t(x)\\) as a cofactor.
    • \n
\n

Remark 3.1 Now we can check a polynomial for specific properties without learning the polyno- mial itself, so this already gives us some form of zero-knowledge and succinctness. Nonetheless, there are multiple issues with this construction:
• Prover may not know the claimed polynomial \\(p(x)\\) at all. He can calculate evaluation \\(t = t(r)\\), select a random number \\(h\\) and set \\(p = t \\cdot h\\), which will be accepted by the verifier as valid, since equation holds.
• Because prover knows the random point \\(x = r\\), he can construct any polynomial which has one shared point at \\(r\\) with \\(t(r) \\cdot h(r)\\).
• In the original statement, prover claims to know a polynomial of a particular degree, in the current protocol there is no enforcement of degree. Hence prover can cheat by using a polynomial of higher degree which also satisfies the cofactors check.

\n
    \n
  1. Obscure Evaluation
    Two first issues of remark 3.1 are possible because values are presented at raw, prover knows r and t(r). It would be ideal if those values would be given as a black box, so one cannot temper with the protocol, but still able to compute operations on those obscure values.
    3.1. Homomorphic Encryption
    There are multiple ways to achieve homomorphic properties of encryption, and we will briefly introduce a simple one. The general idea is to choose a base number \\(g\\) and do ecncryption of a value \\(x\\) by exponentiate of \\(g\\)
    \\[E(x) = g^{x} \\bmod p\\]
    For example, let \\(E(x_1) = g^{x_1} \\bmod p\\), and \\(E(x_2) = g^{x_2} \\bmod p\\), then
    \\[E(x_2) \\cdot E(x_1) = g^{x_1 + x_2} = E(x_1 + x_2) \\]
    2. \n
\n

3.2. Encrypted Polynomial
Let us see how we can evaluate a polynomial \\(p(x) = x^3 − 3x^2 + 2x\\). Because homomorphic encryption does not allows to exponentiate an encrypted value, we’ve must been given encrypted values of powers of x from 1 to 3: \\(E(x),E(x2),E(x3)\\) so that
\\[ E(x^3)^1 \\cdot E(x^2)^{-3} \\cdot E(x)^2 = (g^{x^3})^{1} \\cdot (g^{x^2})^{-3} \\cdot (g^{x})^{2} = g^{1x^3} \\cdot g^{-3x^2} \\cdot g^{2x} = g^{x^3-3x^2+2x}\\]
Hence, we have an encrypted evaluation of our polynomial at some unknown to us \\(x\\)

\n

We can now update the previous version of the protocol, for a polynomial fo degree \\(d\\):

\n
    \n
  • Verifier
      \n
    • samples a random value \\(s\\), i.e., secret
      • \n
    • calculates encryptions of \\(s\\) for all powers \\(i\\) in \\(0,1,…,d\\), i.e. : \\(E(s^{i}) = g^{s^{i}}\\)
      • \n
    • evaluates unencrypted target polynomial with \\(s: t(s)\\)
      • \n
    • encrypted powers of \\(s\\) are provided to the prover: \\(E(s^{0}),E(s^{1}),…,E(s^{d})\\)
      • \n
    \n
    • \n
  • Prover
      \n
    • calculates polynomial \\(h(x) = \\frac{p(x)}{t(x)}\\)
      • \n
    • using encrypted powers \\(g^{s^{0}},g^{s^{1}},…,g^{s^{d}}\\) and coefficients \\(c_0, c_1,…,c_n \\) evaluates \\(E(p(s)) = g^{p(s)} = (g^{s^{d}})^{c_d} \\cdot \\cdot \\cdot (g^{s^{1}})^{c_1} \\cdot (g^{s^{0}})^{c_0}\\) and similarly \\(E(h(s)) = g^{h(s)}\\)
      • \n
    • the resulting \\(g^p\\) and \\(g^h\\) are provided to the verifier
      • \n
    \n
    • \n
  • Verifier
      \n
    • The alst step for the verifier is to checks that \\(p = t(s) \\cdot h \\) in encrypted space: \\(g^p = (g^h)^{t(s)}\\) => \\(g^p = g^{t(s) \\cdot h}\\)
      • \n
    \n
    • \n
\n
\n

Note: because the prover does not know anything about s, it makes it hard to come up with non-legitimate but still matching evaluations.

\n
\n

referneces

\n","site":{"data":{}},"excerpt":"","more":"\n\n\n

The medium of proof: Polynomial

If a prover claims to know some polynomial (no matter how large its degree is) that the verifier also knows, they can follow a simple protocol to verify the statement
• Verifier chooses a random value for x and evaluates his polynomial locally
• Verifier gives x to the prover and asks to evaluate the polynomial in question
• Prover evaluates his polynomial at x and gives the result to the verifier
• Verifier checks if the local result is equal to the prover’s result, and if so then the statement is proven with a high confidence

\n

Non-Interactive Zero-Knowledge of a Polynomial

    \n

  1. Proving Knowledge of a Polynomial
    A polynomial can be expressed in the form (where n is the degree of the polynomial):
    \\[c_n x^n + …+ c_1 x^1 + c_0 x^0\\]
    It one claims that he know a polynomial, it is actually the knowledge of the polynomial’s coefficients.

    \n
    2. \n

  2. Factorization
    The Fundamental Theorem of Algebra states that any polynomial can be factored into linear po- lynomials (i.e., a degree 1 polynomials representing a line), as long it is solvable. Consequently, we can represent any valid polynomial as a product of its factors:
    \\[(x-a_0)(x-a_1)…(x-a_n) = 0\\]

    \n
    3. \n
\n

if the prover wants to prove that indeed his polynomial has specific roots without disclosing the polynomial itself, he needs to prove that his polynomial \\(p(x)\\) is the multiplication of those cofactors \\(t(x) = (x − x_0)(x − x_1)\\), called target polynomial, where \\(x_0, x_1\\) are the specific roots. i.e.:
\\[p(x) = t(x) \\cdot h(x)\\]
A natural way to find \\(h(x)\\) is through the division \\(h(x) = \\frac{p(x)}{t(x)}\\)

\n
\n

for simplicity, onwards we will use polynomial’s letter variable to denote its evaluation, e.g., \\(p = p(r)\\)

\n
\n

Using our polynomial identity check protocol we can compare polynomials \\(p(x)\\) and \\(t(x)·h(x)\\):

\n
    \n
  • Verifier samples a random value \\(r\\), calculates \\(t = t(r)\\) (i.e., evaluates) and gives \\(r\\) to the prover
    • \n
  • Prover calculates \\(h(x) = \\frac{p(x)}{t(x)}\\) and evaluates \\(p(r)\\) and \\(h(r)\\); the resulting values \\(p,h\\) are
    provided to the verifier
    • \n
  • Verifier then checks that \\(p = t \\cdot h\\), if so those polynomials are equal, meaning that \\(p(x)\\) has \\(t(x)\\) as a cofactor.
    • \n
\n

Remark 3.1 Now we can check a polynomial for specific properties without learning the polyno- mial itself, so this already gives us some form of zero-knowledge and succinctness. Nonetheless, there are multiple issues with this construction:
• Prover may not know the claimed polynomial \\(p(x)\\) at all. He can calculate evaluation \\(t = t(r)\\), select a random number \\(h\\) and set \\(p = t \\cdot h\\), which will be accepted by the verifier as valid, since equation holds.
• Because prover knows the random point \\(x = r\\), he can construct any polynomial which has one shared point at \\(r\\) with \\(t(r) \\cdot h(r)\\).
• In the original statement, prover claims to know a polynomial of a particular degree, in the current protocol there is no enforcement of degree. Hence prover can cheat by using a polynomial of higher degree which also satisfies the cofactors check.

\n
    \n
  1. Obscure Evaluation
    Two first issues of remark 3.1 are possible because values are presented at raw, prover knows r and t(r). It would be ideal if those values would be given as a black box, so one cannot temper with the protocol, but still able to compute operations on those obscure values.
    3.1. Homomorphic Encryption
    There are multiple ways to achieve homomorphic properties of encryption, and we will briefly introduce a simple one. The general idea is to choose a base number \\(g\\) and do ecncryption of a value \\(x\\) by exponentiate of \\(g\\)
    \\[E(x) = g^{x} \\bmod p\\]
    For example, let \\(E(x_1) = g^{x_1} \\bmod p\\), and \\(E(x_2) = g^{x_2} \\bmod p\\), then
    \\[E(x_2) \\cdot E(x_1) = g^{x_1 + x_2} = E(x_1 + x_2) \\]
    2. \n
\n

3.2. Encrypted Polynomial
Let us see how we can evaluate a polynomial \\(p(x) = x^3 − 3x^2 + 2x\\). Because homomorphic encryption does not allows to exponentiate an encrypted value, we’ve must been given encrypted values of powers of x from 1 to 3: \\(E(x),E(x2),E(x3)\\) so that
\\[ E(x^3)^1 \\cdot E(x^2)^{-3} \\cdot E(x)^2 = (g^{x^3})^{1} \\cdot (g^{x^2})^{-3} \\cdot (g^{x})^{2} = g^{1x^3} \\cdot g^{-3x^2} \\cdot g^{2x} = g^{x^3-3x^2+2x}\\]
Hence, we have an encrypted evaluation of our polynomial at some unknown to us \\(x\\)

\n

We can now update the previous version of the protocol, for a polynomial fo degree \\(d\\):

\n
    \n
  • Verifier
      \n
    • samples a random value \\(s\\), i.e., secret
      • \n
    • calculates encryptions of \\(s\\) for all powers \\(i\\) in \\(0,1,…,d\\), i.e. : \\(E(s^{i}) = g^{s^{i}}\\)
      • \n
    • evaluates unencrypted target polynomial with \\(s: t(s)\\)
      • \n
    • encrypted powers of \\(s\\) are provided to the prover: \\(E(s^{0}),E(s^{1}),…,E(s^{d})\\)
      • \n
    \n
    • \n
  • Prover
      \n
    • calculates polynomial \\(h(x) = \\frac{p(x)}{t(x)}\\)
      • \n
    • using encrypted powers \\(g^{s^{0}},g^{s^{1}},…,g^{s^{d}}\\) and coefficients \\(c_0, c_1,…,c_n \\) evaluates \\(E(p(s)) = g^{p(s)} = (g^{s^{d}})^{c_d} \\cdot \\cdot \\cdot (g^{s^{1}})^{c_1} \\cdot (g^{s^{0}})^{c_0}\\) and similarly \\(E(h(s)) = g^{h(s)}\\)
      • \n
    • the resulting \\(g^p\\) and \\(g^h\\) are provided to the verifier
      • \n
    \n
    • \n
  • Verifier
      \n
    • The alst step for the verifier is to checks that \\(p = t(s) \\cdot h \\) in encrypted space: \\(g^p = (g^h)^{t(s)}\\) => \\(g^p = g^{t(s) \\cdot h}\\)
      • \n
    \n
    • \n
\n
\n

Note: because the prover does not know anything about s, it makes it hard to come up with non-legitimate but still matching evaluations.

\n
\n

referneces

\n"},{"title":"rust frequently used crates","date":"2022-12-13T09:15:23.000Z","_content":"\n\ntokio-trace -> tracing\n\ncontexts, multi threads\ncausality -\nstructured diagnostics ( no grep)\n\ntracing is part of tokio, tokio not requied\n\nspans: a perido of time, entered and exited\nevents: singular moment in time\nsubscriber: collect trace","source":"_posts/rust/crates/rust-frequently-used-crates.md","raw":"---\ntitle: rust frequently used crates\ndate: 2022-12-13 17:15:23\ntags: [rust]\n---\n\n\ntokio-trace -> tracing\n\ncontexts, multi threads\ncausality -\nstructured diagnostics ( no grep)\n\ntracing is part of tokio, tokio not requied\n\nspans: a perido of time, entered and exited\nevents: singular moment in time\nsubscriber: collect trace","slug":"rust/crates/rust-frequently-used-crates","published":1,"updated":"2023-11-05T04:21:13.733Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8e2003kqwsjg4hf5noa","content":"

tokio-trace -> tracing

\n

contexts, multi threads
causality -
structured diagnostics ( no grep)

\n

tracing is part of tokio, tokio not requied

\n

spans: a perido of time, entered and exited
events: singular moment in time
subscriber: collect trace

\n","site":{"data":{}},"excerpt":"","more":"

tokio-trace -> tracing

\n

contexts, multi threads
causality -
structured diagnostics ( no grep)

\n

tracing is part of tokio, tokio not requied

\n

spans: a perido of time, entered and exited
events: singular moment in time
subscriber: collect trace

\n"},{"title":"rust crate serde","date":"2023-04-01T14:04:38.000Z","_content":"\n```rust\n#[serde(tag = \"filterType\")]\n#[serde(untagged)]\n#[serde(rename = \"PRICE_FILTER\")]\n#[serde(rename_all = \"camelCase\")]\n\n#[serde(with = \"string_or_float\")]\npub stop_price: f64,\n```","source":"_posts/rust/crates/rust-serde.md","raw":"---\ntitle: rust crate serde\ndate: 2023-04-01 22:04:38\ntags: [rust-crate]\n---\n\n```rust\n#[serde(tag = \"filterType\")]\n#[serde(untagged)]\n#[serde(rename = \"PRICE_FILTER\")]\n#[serde(rename_all = \"camelCase\")]\n\n#[serde(with = \"string_or_float\")]\npub stop_price: f64,\n```","slug":"rust/crates/rust-serde","published":1,"updated":"2023-11-05T04:21:13.734Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8e2003mqwsj9ztifql4","content":"
1
2
3
4
5
6
7
#[serde(tag = "filterType")]
#[serde(untagged)]
#[serde(rename = "PRICE_FILTER")]
#[serde(rename_all = "camelCase")]

#[serde(with = "string_or_float")]
pub stop_price: f64,
","site":{"data":{}},"excerpt":"","more":"
1
2
3
4
5
6
7
#[serde(tag = "filterType")]
#[serde(untagged)]
#[serde(rename = "PRICE_FILTER")]
#[serde(rename_all = "camelCase")]

#[serde(with = "string_or_float")]
pub stop_price: f64,
"},{"title":"rust std data structure (1D)","date":"2023-05-01T14:04:38.000Z","_content":"\n## array\nA **fixed-size** array, denoted [T; N], for the element type, T, and the non-negative compile-time constant size, N.\n```rust\ntodo!()\n```\n\n## slice\nA **dynamically-sized view** into a contiguous sequence, [T].\n- `len()`: Returns the number of elements in the slice\n- `is_empty()`\n- `first()` Returns the first element of the slice, or `None` if it is empty.\n- `first_mut()` Returns a mutable **pointer** to the first element of the slice, or `None` if it is empty\n- `split_first()` Returns the first and all the rest of the elements of the slice, or `None` if it is empty.\n- `split_first_mut()` \n- `split_last()`\n- `split_last_mut()`\n- `last()`\n- `last_mut()`\n- `get(index: I)` Returns a reference to an element or subslice depending on the type of index.\n```rust\nlet v = [10, 40, 30];\nassert_eq!(Some(&40), v.get(1));\nassert_eq!(Some(&[10, 40][..]), v.get(0..2));\n```\n- `get_mut(index: I)`\n- `get_unchecked(index: I)` Returns a reference to an element or subslice, without doing bounds checking\n- `get_unchecked_mut(index: I)`\n- `as_ptr(&self) -> *const T` Returns a raw pointer to the slice's buffer\n```rust\nlet x = &[1, 2, 4];\nlet x_ptr = x.as_ptr();\nunsafe {\n for i in 0..x.len() {\n assert_eq!(x.get_unchecked(i), &*x_ptr.add(i));\n }\n}\n```\n- `as_mut_ptr(&mut self) -> *mut T` \n```rust\nlet x = &mut [1, 2, 4];\nlet x_ptr = x.as_mut_ptr();\nunsafe {\n for i in 0..x.len() {\n *x_ptr.add(i) += 2;\n }\n}\nassert_eq!(x, &[3, 4, 6]);\n```\n- `as_ptr_range(&self) -> Range<*const T>` Returns the two raw pointers spanning the slice.\n```rust\npub const fn as_ptr_range(&self) -> Range<*const T> {\n let start = self.as_ptr();\n let end = unsafe { start.add(self.len()) };\n start..end\n}\n```\n- `as_mut_ptr_range(&mut self) -> Range<*mut T>`\n- `swap(&mut self, a: usize, b: usize)` Swaps two elements in the slice.\n- `reverse(&mut self)` Reverses the order of elements in the slice, in place.\n- `windows(&self, size: usize)` Returns an iterator over all contiguous windows of length `size`. The windows overlap. If the slice is shorter than `size`, the iterator returns no values.\n```rust\nlet slice = ['r', 'u', 's', 't'];\nlet mut iter = slice.windows(2);\nassert_eq!(iter.next().unwrap(), &['r', 'u']);\nassert_eq!(iter.next().unwrap(), &['u', 's']);\nassert_eq!(iter.next().unwrap(), &['s', 't']);\nassert!(iter.next().is_none());\n```\n- `chunks(&self, chunk_size: usize)` Returns an iterator over `chunk_size` elements of the slice at a time\n```rust\nlet slice = ['l', 'o', 'r', 'e', 'm'];\nlet mut iter = slice.chunks(2);\nassert_eq!(iter.next().unwrap(), &['l', 'o']);\nassert_eq!(iter.next().unwrap(), &['r', 'e']);\nassert_eq!(iter.next().unwrap(), &['m']);\nassert!(iter.next().is_none());\n```\n- `chunks_mut()`\n- `chunks_exact(&self, chunk_size: usize)`\n```rust\nlet slice = ['l', 'o', 'r', 'e', 'm'];\nlet mut iter = slice.chunks_exact(2);\nassert_eq!(iter.next().unwrap(), &['l', 'o']);\nassert_eq!(iter.next().unwrap(), &['r', 'e']);\nassert!(iter.next().is_none());\nassert_eq!(iter.remainder(), &['m']);\n```\n- `as_chunks_unchecked(&self)` Splits the slice into a slice of `N`-element arrays, assuming that there's no remainder\n- `as_chunks(&self)` Splits the slice into a slice of `N`-element arrays, starting at the beginning of the slice, and a remainder slice with length strictly less than `N`\n- `as_rchunks(&self)` r means reverse\n- `group_by(&self, pred: F)` Returns an iterator over the slice producing non-overlapping runs of elements using the predicate to separate them. The predicate is called on two elements following themselves, it means the predicate is called on `slice[0]` and `slice[1]` then on `slice[1]` and `slice[2]` and so on\n```rust\n#![feature(slice_group_by)]\nlet slice = &[1, 1, 2, 3, 2, 3, 2, 3, 4];\nlet mut iter = slice.group_by(|a, b| a <= b);\nassert_eq!(iter.next(), Some(&[1, 1, 2, 3][..]));\nassert_eq!(iter.next(), Some(&[2, 3][..]));\nassert_eq!(iter.next(), Some(&[2, 3, 4][..]));\nassert_eq!(iter.next(), None);\n```\n- `split_at(&self, mid: usize)` Divides one slice into two at an index.\n- `split(&self, pred: F)` Returns an iterator over subslices separated by elements that match `pred`. The matched element is not contained in the subslices.\n- `splitn(&self, n: usize, pred: F)` \n- `contains(&self, x: &T)` Returns `true` if the slice contains an element with the given value.\n- `starts_with(&self, needle: &[T])` eturns `true` if `needle` is a prefix of the slice\n```rust\nlet v = [10, 40, 30];\nassert!(v.starts_with(&[10]));\nassert!(v.starts_with(&[10, 40]));\nassert!(!v.starts_with(&[50]));\n```\n- `ends_with(&self, needle: &[T])` \n- `strip_prefix` Returns a subslice with the prefix removed.\n```rust\nlet v = &[10, 40, 30];\nassert_eq!(v.strip_prefix(&[10]), Some(&[40, 30][..]));\nassert_eq!(v.strip_prefix(&[50]), None);\nlet prefix : &str = \"he\";\nassert_eq!(b\"hello\".strip_prefix(prefix.as_bytes()),\n Some(b\"llo\".as_ref()));\n```\n- `strip_suffix`\n- `binary_search(&self, x: &T)` Binary searches this slice for a given element.\n- `sort_unstable(&mut self)` Sorts the slice, but might not preserve the order of equal elements.\n- `rotate_left(&mut self, mid: usize)` Rotates the slice in-place such that the first `mid` elements of the slice move to the end while the last `self.len() - mid` elements move to the front.\n- `fill(&mut self, value: T)` Fills `self` with elements by cloning `value`.\n- `clone_from_slice(&mut self, src: &[T])` Copies the elements from `src` into `self`.\n- `copy_from_slice(&mut self, src: &[T])` \n- `is_sorted(&self)` \n- `take<'a, R: OneSidedRange>(self: &mut &'a Self, range: R)` Removes the subslice corresponding to the given range\n- `get_many_mut` Returns mutable references to many indices at once.\n```rust\n#![feature(get_many_mut)]\nlet v = &mut [1, 2, 3];\nif let Ok([a, b]) = v.get_many_mut([0, 2]) {\n *a = 413;\n *b = 612;\n}\nassert_eq!(v, &[413, 2, 612]);\n```\n\n## alloc::vec::Vec\n- `fn truncate(&mut self, len: usize)` Shortens the vector, keeping the first `len` elements and dropping the rest\n\n## std::collections::VecDeque\nA double-ended queue (deque) implemented with a growable ring buffer.\nSince VecDeque is a ring buffer, its elements are not necessarily contiguous in memory. If you want to access the elements as a single slice, such as for efficient sorting, you can use make_contiguous. It rotates the VecDeque so that its elements do not wrap, and returns a mutable slice to the now-contiguous element sequence.\n\n- `swap(&mut self, i: usize, j: usize)`\n- `reserve_exact(&mut self, additional: usize)` Reserves the minimum capacity for at least `additional` more elements to be inserted in the given deque. Does nothing if the capacity is already sufficient.\n- `reserve(&mut self, additional: usize)`\n- `shrink_to_fit(&mut self)` Shrinks the capacity of the deque as much as possible.\n- `truncate(&mut self, len: usize)` Shortens the deque, keeping the first `len` elements and dropping the rest.\n```rust\nuse std::collections::VecDeque;\nlet mut buf = VecDeque::new();\nbuf.push_back(5);\nbuf.push_back(10);\nbuf.push_back(15);\nassert_eq!(buf, [5, 10, 15]);\nbuf.truncate(1);\nassert_eq!(buf, [5]);\n```\n- `iter(&self)`\n- `as_slices(&self)`\n- `slice_ranges(&self, range: R)` Given a range into the logical buffer of the deque, this function return two ranges into the physical buffer that correspond to the given range\n- `range(&self, range: R)` Creates an iterator that covers the specified range in the deque.\n```rust\nuse std::collections::VecDeque;\nlet deque: VecDeque<_> = [1, 2, 3].into();\nlet range = deque.range(2..).copied().collect::>();\nassert_eq!(range, [3]);\n// A full range covers all contents\nlet all = deque.range(..);\nassert_eq!(all.len(), 3);\n```\n- `drain(&mut self, range: R)` Removes the specified range from the deque in bulk, returning all removed elements as an iterator.\n- `clear(&mut self)`\n- `contains(&self, x: &T)` Returns `true` if the deque contains an element equal to the given value\n- `front(&self)` Provides a reference to the front element\n- `front_mut(&mut self)`\n- `back(&self)`\n- `back_mut(&mut self)`\n- `pop_front(&mut self)`\n- `pop_back(&mut self)`\n- `push_front(&mut self, value: T)`\n- `push_back(&mut self, value: T)`\n\n## [std::collections::LinkedList](https://doc.rust-lang.org/std/collections/struct.LinkedList.html)","source":"_posts/rust/rust_std/rust-std-data-structure-1.md","raw":"---\ntitle: rust std data structure (1D)\ndate: 2023-05-01 22:04:38\ntags: [rust-std]\n---\n\n## array\nA **fixed-size** array, denoted [T; N], for the element type, T, and the non-negative compile-time constant size, N.\n```rust\ntodo!()\n```\n\n## slice\nA **dynamically-sized view** into a contiguous sequence, [T].\n- `len()`: Returns the number of elements in the slice\n- `is_empty()`\n- `first()` Returns the first element of the slice, or `None` if it is empty.\n- `first_mut()` Returns a mutable **pointer** to the first element of the slice, or `None` if it is empty\n- `split_first()` Returns the first and all the rest of the elements of the slice, or `None` if it is empty.\n- `split_first_mut()` \n- `split_last()`\n- `split_last_mut()`\n- `last()`\n- `last_mut()`\n- `get(index: I)` Returns a reference to an element or subslice depending on the type of index.\n```rust\nlet v = [10, 40, 30];\nassert_eq!(Some(&40), v.get(1));\nassert_eq!(Some(&[10, 40][..]), v.get(0..2));\n```\n- `get_mut(index: I)`\n- `get_unchecked(index: I)` Returns a reference to an element or subslice, without doing bounds checking\n- `get_unchecked_mut(index: I)`\n- `as_ptr(&self) -> *const T` Returns a raw pointer to the slice's buffer\n```rust\nlet x = &[1, 2, 4];\nlet x_ptr = x.as_ptr();\nunsafe {\n for i in 0..x.len() {\n assert_eq!(x.get_unchecked(i), &*x_ptr.add(i));\n }\n}\n```\n- `as_mut_ptr(&mut self) -> *mut T` \n```rust\nlet x = &mut [1, 2, 4];\nlet x_ptr = x.as_mut_ptr();\nunsafe {\n for i in 0..x.len() {\n *x_ptr.add(i) += 2;\n }\n}\nassert_eq!(x, &[3, 4, 6]);\n```\n- `as_ptr_range(&self) -> Range<*const T>` Returns the two raw pointers spanning the slice.\n```rust\npub const fn as_ptr_range(&self) -> Range<*const T> {\n let start = self.as_ptr();\n let end = unsafe { start.add(self.len()) };\n start..end\n}\n```\n- `as_mut_ptr_range(&mut self) -> Range<*mut T>`\n- `swap(&mut self, a: usize, b: usize)` Swaps two elements in the slice.\n- `reverse(&mut self)` Reverses the order of elements in the slice, in place.\n- `windows(&self, size: usize)` Returns an iterator over all contiguous windows of length `size`. The windows overlap. If the slice is shorter than `size`, the iterator returns no values.\n```rust\nlet slice = ['r', 'u', 's', 't'];\nlet mut iter = slice.windows(2);\nassert_eq!(iter.next().unwrap(), &['r', 'u']);\nassert_eq!(iter.next().unwrap(), &['u', 's']);\nassert_eq!(iter.next().unwrap(), &['s', 't']);\nassert!(iter.next().is_none());\n```\n- `chunks(&self, chunk_size: usize)` Returns an iterator over `chunk_size` elements of the slice at a time\n```rust\nlet slice = ['l', 'o', 'r', 'e', 'm'];\nlet mut iter = slice.chunks(2);\nassert_eq!(iter.next().unwrap(), &['l', 'o']);\nassert_eq!(iter.next().unwrap(), &['r', 'e']);\nassert_eq!(iter.next().unwrap(), &['m']);\nassert!(iter.next().is_none());\n```\n- `chunks_mut()`\n- `chunks_exact(&self, chunk_size: usize)`\n```rust\nlet slice = ['l', 'o', 'r', 'e', 'm'];\nlet mut iter = slice.chunks_exact(2);\nassert_eq!(iter.next().unwrap(), &['l', 'o']);\nassert_eq!(iter.next().unwrap(), &['r', 'e']);\nassert!(iter.next().is_none());\nassert_eq!(iter.remainder(), &['m']);\n```\n- `as_chunks_unchecked(&self)` Splits the slice into a slice of `N`-element arrays, assuming that there's no remainder\n- `as_chunks(&self)` Splits the slice into a slice of `N`-element arrays, starting at the beginning of the slice, and a remainder slice with length strictly less than `N`\n- `as_rchunks(&self)` r means reverse\n- `group_by(&self, pred: F)` Returns an iterator over the slice producing non-overlapping runs of elements using the predicate to separate them. The predicate is called on two elements following themselves, it means the predicate is called on `slice[0]` and `slice[1]` then on `slice[1]` and `slice[2]` and so on\n```rust\n#![feature(slice_group_by)]\nlet slice = &[1, 1, 2, 3, 2, 3, 2, 3, 4];\nlet mut iter = slice.group_by(|a, b| a <= b);\nassert_eq!(iter.next(), Some(&[1, 1, 2, 3][..]));\nassert_eq!(iter.next(), Some(&[2, 3][..]));\nassert_eq!(iter.next(), Some(&[2, 3, 4][..]));\nassert_eq!(iter.next(), None);\n```\n- `split_at(&self, mid: usize)` Divides one slice into two at an index.\n- `split(&self, pred: F)` Returns an iterator over subslices separated by elements that match `pred`. The matched element is not contained in the subslices.\n- `splitn(&self, n: usize, pred: F)` \n- `contains(&self, x: &T)` Returns `true` if the slice contains an element with the given value.\n- `starts_with(&self, needle: &[T])` eturns `true` if `needle` is a prefix of the slice\n```rust\nlet v = [10, 40, 30];\nassert!(v.starts_with(&[10]));\nassert!(v.starts_with(&[10, 40]));\nassert!(!v.starts_with(&[50]));\n```\n- `ends_with(&self, needle: &[T])` \n- `strip_prefix` Returns a subslice with the prefix removed.\n```rust\nlet v = &[10, 40, 30];\nassert_eq!(v.strip_prefix(&[10]), Some(&[40, 30][..]));\nassert_eq!(v.strip_prefix(&[50]), None);\nlet prefix : &str = \"he\";\nassert_eq!(b\"hello\".strip_prefix(prefix.as_bytes()),\n Some(b\"llo\".as_ref()));\n```\n- `strip_suffix`\n- `binary_search(&self, x: &T)` Binary searches this slice for a given element.\n- `sort_unstable(&mut self)` Sorts the slice, but might not preserve the order of equal elements.\n- `rotate_left(&mut self, mid: usize)` Rotates the slice in-place such that the first `mid` elements of the slice move to the end while the last `self.len() - mid` elements move to the front.\n- `fill(&mut self, value: T)` Fills `self` with elements by cloning `value`.\n- `clone_from_slice(&mut self, src: &[T])` Copies the elements from `src` into `self`.\n- `copy_from_slice(&mut self, src: &[T])` \n- `is_sorted(&self)` \n- `take<'a, R: OneSidedRange>(self: &mut &'a Self, range: R)` Removes the subslice corresponding to the given range\n- `get_many_mut` Returns mutable references to many indices at once.\n```rust\n#![feature(get_many_mut)]\nlet v = &mut [1, 2, 3];\nif let Ok([a, b]) = v.get_many_mut([0, 2]) {\n *a = 413;\n *b = 612;\n}\nassert_eq!(v, &[413, 2, 612]);\n```\n\n## alloc::vec::Vec\n- `fn truncate(&mut self, len: usize)` Shortens the vector, keeping the first `len` elements and dropping the rest\n\n## std::collections::VecDeque\nA double-ended queue (deque) implemented with a growable ring buffer.\nSince VecDeque is a ring buffer, its elements are not necessarily contiguous in memory. If you want to access the elements as a single slice, such as for efficient sorting, you can use make_contiguous. It rotates the VecDeque so that its elements do not wrap, and returns a mutable slice to the now-contiguous element sequence.\n\n- `swap(&mut self, i: usize, j: usize)`\n- `reserve_exact(&mut self, additional: usize)` Reserves the minimum capacity for at least `additional` more elements to be inserted in the given deque. Does nothing if the capacity is already sufficient.\n- `reserve(&mut self, additional: usize)`\n- `shrink_to_fit(&mut self)` Shrinks the capacity of the deque as much as possible.\n- `truncate(&mut self, len: usize)` Shortens the deque, keeping the first `len` elements and dropping the rest.\n```rust\nuse std::collections::VecDeque;\nlet mut buf = VecDeque::new();\nbuf.push_back(5);\nbuf.push_back(10);\nbuf.push_back(15);\nassert_eq!(buf, [5, 10, 15]);\nbuf.truncate(1);\nassert_eq!(buf, [5]);\n```\n- `iter(&self)`\n- `as_slices(&self)`\n- `slice_ranges(&self, range: R)` Given a range into the logical buffer of the deque, this function return two ranges into the physical buffer that correspond to the given range\n- `range(&self, range: R)` Creates an iterator that covers the specified range in the deque.\n```rust\nuse std::collections::VecDeque;\nlet deque: VecDeque<_> = [1, 2, 3].into();\nlet range = deque.range(2..).copied().collect::>();\nassert_eq!(range, [3]);\n// A full range covers all contents\nlet all = deque.range(..);\nassert_eq!(all.len(), 3);\n```\n- `drain(&mut self, range: R)` Removes the specified range from the deque in bulk, returning all removed elements as an iterator.\n- `clear(&mut self)`\n- `contains(&self, x: &T)` Returns `true` if the deque contains an element equal to the given value\n- `front(&self)` Provides a reference to the front element\n- `front_mut(&mut self)`\n- `back(&self)`\n- `back_mut(&mut self)`\n- `pop_front(&mut self)`\n- `pop_back(&mut self)`\n- `push_front(&mut self, value: T)`\n- `push_back(&mut self, value: T)`\n\n## [std::collections::LinkedList](https://doc.rust-lang.org/std/collections/struct.LinkedList.html)","slug":"rust/rust_std/rust-std-data-structure-1","published":1,"updated":"2023-11-05T04:21:13.735Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8e3003oqwsj76cl15kp","content":"

array

A fixed-size array, denoted [T; N], for the element type, T, and the non-negative compile-time constant size, N.

\n
1
todo!()
\n\n

slice

A dynamically-sized view into a contiguous sequence, [T].

\n
    \n
  • len(): Returns the number of elements in the slice
    • \n
  • is_empty()
    • \n
  • first() Returns the first element of the slice, or None if it is empty.
    • \n
  • first_mut() Returns a mutable pointer to the first element of the slice, or None if it is empty
    • \n
  • split_first() Returns the first and all the rest of the elements of the slice, or None if it is empty.
    • \n
  • split_first_mut()
    • \n
  • split_last()
    • \n
  • split_last_mut()
    • \n
  • last()
    • \n
  • last_mut()
    • \n
  • get<I>(index: I) Returns a reference to an element or subslice depending on the type of index.
    1
    2
    3
    let v = [10, 40, 30];
    assert_eq!(Some(&40), v.get(1));
    assert_eq!(Some(&[10, 40][..]), v.get(0..2));
    • \n
  • get_mut<I>(index: I)
    • \n
  • get_unchecked<I>(index: I) Returns a reference to an element or subslice, without doing bounds checking
    • \n
  • get_unchecked_mut<I>(index: I)
    • \n
  • as_ptr(&self) -> *const T Returns a raw pointer to the slice’s buffer
    1
    2
    3
    4
    5
    6
    7
    let x = &[1, 2, 4];
    let x_ptr = x.as_ptr();
    unsafe {
        for i in 0..x.len() {
            assert_eq!(x.get_unchecked(i), &*x_ptr.add(i));
        }
    }
    • \n
  • as_mut_ptr(&mut self) -> *mut T 
    1
    2
    3
    4
    5
    6
    7
    8
    let x = &mut [1, 2, 4];
    let x_ptr = x.as_mut_ptr();
    unsafe {
        for i in 0..x.len() {
            *x_ptr.add(i) += 2;
        }
    }
    assert_eq!(x, &[3, 4, 6]);
    • \n
  • as_ptr_range(&self) -> Range<*const T> Returns the two raw pointers spanning the slice.
    1
    2
    3
    4
    5
    pub const fn as_ptr_range(&self) -> Range<*const T> {
        let start = self.as_ptr();
        let end = unsafe { start.add(self.len()) };
        start..end
    }
    • \n
  • as_mut_ptr_range(&mut self) -> Range<*mut T>
    • \n
  • swap(&mut self, a: usize, b: usize) Swaps two elements in the slice.
    • \n
  • reverse(&mut self) Reverses the order of elements in the slice, in place.
    • \n
  • windows(&self, size: usize) Returns an iterator over all contiguous windows of length size. The windows overlap. If the slice is shorter than size, the iterator returns no values.
    1
    2
    3
    4
    5
    6
    let slice = ['r', 'u', 's', 't'];
    let mut iter = slice.windows(2);
    assert_eq!(iter.next().unwrap(), &['r', 'u']);
    assert_eq!(iter.next().unwrap(), &['u', 's']);
    assert_eq!(iter.next().unwrap(), &['s', 't']);
    assert!(iter.next().is_none());
    • \n
  • chunks(&self, chunk_size: usize) Returns an iterator over chunk_size elements of the slice at a time
    1
    2
    3
    4
    5
    6
    let slice = ['l', 'o', 'r', 'e', 'm'];
    let mut iter = slice.chunks(2);
    assert_eq!(iter.next().unwrap(), &['l', 'o']);
    assert_eq!(iter.next().unwrap(), &['r', 'e']);
    assert_eq!(iter.next().unwrap(), &['m']);
    assert!(iter.next().is_none());
    • \n
  • chunks_mut()
    • \n
  • chunks_exact(&self, chunk_size: usize)
    1
    2
    3
    4
    5
    6
    let slice = ['l', 'o', 'r', 'e', 'm'];
    let mut iter = slice.chunks_exact(2);
    assert_eq!(iter.next().unwrap(), &['l', 'o']);
    assert_eq!(iter.next().unwrap(), &['r', 'e']);
    assert!(iter.next().is_none());
    assert_eq!(iter.remainder(), &['m']);
    • \n
  • as_chunks_unchecked<const N: usize>(&self) Splits the slice into a slice of N-element arrays, assuming that there’s no remainder
    • \n
  • as_chunks<const N: usize>(&self) Splits the slice into a slice of N-element arrays, starting at the beginning of the slice, and a remainder slice with length strictly less than N
    • \n
  • as_rchunks<const N: usize>(&self) r means reverse
    • \n
  • group_by<F>(&self, pred: F) Returns an iterator over the slice producing non-overlapping runs of elements using the predicate to separate them. The predicate is called on two elements following themselves, it means the predicate is called on slice[0] and slice[1] then on slice[1] and slice[2] and so on
    1
    2
    3
    4
    5
    6
    7
    #![feature(slice_group_by)]
    let slice = &[1, 1, 2, 3, 2, 3, 2, 3, 4];
    let mut iter = slice.group_by(|a, b| a <= b);
    assert_eq!(iter.next(), Some(&[1, 1, 2, 3][..]));
    assert_eq!(iter.next(), Some(&[2, 3][..]));
    assert_eq!(iter.next(), Some(&[2, 3, 4][..]));
    assert_eq!(iter.next(), None);
    • \n
  • split_at(&self, mid: usize) Divides one slice into two at an index.
    • \n
  • split<F>(&self, pred: F) Returns an iterator over subslices separated by elements that match pred. The matched element is not contained in the subslices.
    • \n
  • splitn<F>(&self, n: usize, pred: F)
    • \n
  • contains(&self, x: &T) Returns true if the slice contains an element with the given value.
    • \n
  • starts_with(&self, needle: &[T]) eturns true if needle is a prefix of the slice
    1
    2
    3
    4
    let v = [10, 40, 30];
    assert!(v.starts_with(&[10]));
    assert!(v.starts_with(&[10, 40]));
    assert!(!v.starts_with(&[50]));
    • \n
  • ends_with(&self, needle: &[T])
    • \n
  • strip_prefix Returns a subslice with the prefix removed.
    1
    2
    3
    4
    5
    6
    let v = &[10, 40, 30];
    assert_eq!(v.strip_prefix(&[10]), Some(&[40, 30][..]));
    assert_eq!(v.strip_prefix(&[50]), None);
    let prefix : &str = "he";
    assert_eq!(b"hello".strip_prefix(prefix.as_bytes()),
               Some(b"llo".as_ref()));
    • \n
  • strip_suffix
    • \n
  • binary_search(&self, x: &T) Binary searches this slice for a given element.
    • \n
  • sort_unstable(&mut self) Sorts the slice, but might not preserve the order of equal elements.
    • \n
  • rotate_left(&mut self, mid: usize) Rotates the slice in-place such that the first mid elements of the slice move to the end while the last self.len() - mid elements move to the front.
    • \n
  • fill(&mut self, value: T) Fills self with elements by cloning value.
    • \n
  • clone_from_slice(&mut self, src: &[T]) Copies the elements from src into self.
    • \n
  • copy_from_slice(&mut self, src: &[T])
    • \n
  • is_sorted(&self)
    • \n
  • take<'a, R: OneSidedRange<usize>>(self: &mut &'a Self, range: R) Removes the subslice corresponding to the given range
    • \n
  • get_many_mut<const N: usize> Returns mutable references to many indices at once.
    1
    2
    3
    4
    5
    6
    7
    #![feature(get_many_mut)]
    let v = &mut [1, 2, 3];
    if let Ok([a, b]) = v.get_many_mut([0, 2]) {
        *a = 413;
        *b = 612;
    }
    assert_eq!(v, &[413, 2, 612]);
    • \n
\n

alloc::vec::Vec

    \n
  • fn truncate(&mut self, len: usize) Shortens the vector, keeping the first len elements and dropping the rest
    • \n
\n

std::collections::VecDeque

A double-ended queue (deque) implemented with a growable ring buffer.
Since VecDeque is a ring buffer, its elements are not necessarily contiguous in memory. If you want to access the elements as a single slice, such as for efficient sorting, you can use make_contiguous. It rotates the VecDeque so that its elements do not wrap, and returns a mutable slice to the now-contiguous element sequence.

\n
    \n
  • swap(&mut self, i: usize, j: usize)
    • \n
  • reserve_exact(&mut self, additional: usize) Reserves the minimum capacity for at least additional more elements to be inserted in the given deque. Does nothing if the capacity is already sufficient.
    • \n
  • reserve(&mut self, additional: usize)
    • \n
  • shrink_to_fit(&mut self) Shrinks the capacity of the deque as much as possible.
    • \n
  • truncate(&mut self, len: usize) Shortens the deque, keeping the first len elements and dropping the rest.
    1
    2
    3
    4
    5
    6
    7
    8
    use std::collections::VecDeque;
    let mut buf = VecDeque::new();
    buf.push_back(5);
    buf.push_back(10);
    buf.push_back(15);
    assert_eq!(buf, [5, 10, 15]);
    buf.truncate(1);
    assert_eq!(buf, [5]);
    • \n
  • iter(&self)
    • \n
  • as_slices(&self)
    • \n
  • slice_ranges<R>(&self, range: R) Given a range into the logical buffer of the deque, this function return two ranges into the physical buffer that correspond to the given range
    • \n
  • range<R>(&self, range: R) Creates an iterator that covers the specified range in the deque.
    1
    2
    3
    4
    5
    6
    7
    use std::collections::VecDeque;
    let deque: VecDeque<_> = [1, 2, 3].into();
    let range = deque.range(2..).copied().collect::<VecDeque<_>>();
    assert_eq!(range, [3]);
    // A full range covers all contents
    let all = deque.range(..);
    assert_eq!(all.len(), 3);
    • \n
  • drain<R>(&mut self, range: R) Removes the specified range from the deque in bulk, returning all removed elements as an iterator.
    • \n
  • clear(&mut self)
    • \n
  • contains(&self, x: &T) Returns true if the deque contains an element equal to the given value
    • \n
  • front(&self) Provides a reference to the front element
    • \n
  • front_mut(&mut self)
    • \n
  • back(&self)
    • \n
  • back_mut(&mut self)
    • \n
  • pop_front(&mut self)
    • \n
  • pop_back(&mut self)
    • \n
  • push_front(&mut self, value: T)
    • \n
  • push_back(&mut self, value: T)
    • \n
\n

std::collections::LinkedList

","site":{"data":{}},"excerpt":"","more":"

array

A fixed-size array, denoted [T; N], for the element type, T, and the non-negative compile-time constant size, N.

\n
1
todo!()
\n\n

slice

A dynamically-sized view into a contiguous sequence, [T].

\n
    \n
  • len(): Returns the number of elements in the slice
    • \n
  • is_empty()
    • \n
  • first() Returns the first element of the slice, or None if it is empty.
    • \n
  • first_mut() Returns a mutable pointer to the first element of the slice, or None if it is empty
    • \n
  • split_first() Returns the first and all the rest of the elements of the slice, or None if it is empty.
    • \n
  • split_first_mut()
    • \n
  • split_last()
    • \n
  • split_last_mut()
    • \n
  • last()
    • \n
  • last_mut()
    • \n
  • get<I>(index: I) Returns a reference to an element or subslice depending on the type of index.
    1
    2
    3
    let v = [10, 40, 30];
    assert_eq!(Some(&40), v.get(1));
    assert_eq!(Some(&[10, 40][..]), v.get(0..2));
    • \n
  • get_mut<I>(index: I)
    • \n
  • get_unchecked<I>(index: I) Returns a reference to an element or subslice, without doing bounds checking
    • \n
  • get_unchecked_mut<I>(index: I)
    • \n
  • as_ptr(&self) -> *const T Returns a raw pointer to the slice’s buffer
    1
    2
    3
    4
    5
    6
    7
    let x = &[1, 2, 4];
    let x_ptr = x.as_ptr();
    unsafe {
        for i in 0..x.len() {
            assert_eq!(x.get_unchecked(i), &*x_ptr.add(i));
        }
    }
    • \n
  • as_mut_ptr(&mut self) -> *mut T 
    1
    2
    3
    4
    5
    6
    7
    8
    let x = &mut [1, 2, 4];
    let x_ptr = x.as_mut_ptr();
    unsafe {
        for i in 0..x.len() {
            *x_ptr.add(i) += 2;
        }
    }
    assert_eq!(x, &[3, 4, 6]);
    • \n
  • as_ptr_range(&self) -> Range<*const T> Returns the two raw pointers spanning the slice.
    1
    2
    3
    4
    5
    pub const fn as_ptr_range(&self) -> Range<*const T> {
        let start = self.as_ptr();
        let end = unsafe { start.add(self.len()) };
        start..end
    }
    • \n
  • as_mut_ptr_range(&mut self) -> Range<*mut T>
    • \n
  • swap(&mut self, a: usize, b: usize) Swaps two elements in the slice.
    • \n
  • reverse(&mut self) Reverses the order of elements in the slice, in place.
    • \n
  • windows(&self, size: usize) Returns an iterator over all contiguous windows of length size. The windows overlap. If the slice is shorter than size, the iterator returns no values.
    1
    2
    3
    4
    5
    6
    let slice = ['r', 'u', 's', 't'];
    let mut iter = slice.windows(2);
    assert_eq!(iter.next().unwrap(), &['r', 'u']);
    assert_eq!(iter.next().unwrap(), &['u', 's']);
    assert_eq!(iter.next().unwrap(), &['s', 't']);
    assert!(iter.next().is_none());
    • \n
  • chunks(&self, chunk_size: usize) Returns an iterator over chunk_size elements of the slice at a time
    1
    2
    3
    4
    5
    6
    let slice = ['l', 'o', 'r', 'e', 'm'];
    let mut iter = slice.chunks(2);
    assert_eq!(iter.next().unwrap(), &['l', 'o']);
    assert_eq!(iter.next().unwrap(), &['r', 'e']);
    assert_eq!(iter.next().unwrap(), &['m']);
    assert!(iter.next().is_none());
    • \n
  • chunks_mut()
    • \n
  • chunks_exact(&self, chunk_size: usize)
    1
    2
    3
    4
    5
    6
    let slice = ['l', 'o', 'r', 'e', 'm'];
    let mut iter = slice.chunks_exact(2);
    assert_eq!(iter.next().unwrap(), &['l', 'o']);
    assert_eq!(iter.next().unwrap(), &['r', 'e']);
    assert!(iter.next().is_none());
    assert_eq!(iter.remainder(), &['m']);
    • \n
  • as_chunks_unchecked<const N: usize>(&self) Splits the slice into a slice of N-element arrays, assuming that there’s no remainder
    • \n
  • as_chunks<const N: usize>(&self) Splits the slice into a slice of N-element arrays, starting at the beginning of the slice, and a remainder slice with length strictly less than N
    • \n
  • as_rchunks<const N: usize>(&self) r means reverse
    • \n
  • group_by<F>(&self, pred: F) Returns an iterator over the slice producing non-overlapping runs of elements using the predicate to separate them. The predicate is called on two elements following themselves, it means the predicate is called on slice[0] and slice[1] then on slice[1] and slice[2] and so on
    1
    2
    3
    4
    5
    6
    7
    #![feature(slice_group_by)]
    let slice = &[1, 1, 2, 3, 2, 3, 2, 3, 4];
    let mut iter = slice.group_by(|a, b| a <= b);
    assert_eq!(iter.next(), Some(&[1, 1, 2, 3][..]));
    assert_eq!(iter.next(), Some(&[2, 3][..]));
    assert_eq!(iter.next(), Some(&[2, 3, 4][..]));
    assert_eq!(iter.next(), None);
    • \n
  • split_at(&self, mid: usize) Divides one slice into two at an index.
    • \n
  • split<F>(&self, pred: F) Returns an iterator over subslices separated by elements that match pred. The matched element is not contained in the subslices.
    • \n
  • splitn<F>(&self, n: usize, pred: F)
    • \n
  • contains(&self, x: &T) Returns true if the slice contains an element with the given value.
    • \n
  • starts_with(&self, needle: &[T]) eturns true if needle is a prefix of the slice
    1
    2
    3
    4
    let v = [10, 40, 30];
    assert!(v.starts_with(&[10]));
    assert!(v.starts_with(&[10, 40]));
    assert!(!v.starts_with(&[50]));
    • \n
  • ends_with(&self, needle: &[T])
    • \n
  • strip_prefix Returns a subslice with the prefix removed.
    1
    2
    3
    4
    5
    6
    let v = &[10, 40, 30];
    assert_eq!(v.strip_prefix(&[10]), Some(&[40, 30][..]));
    assert_eq!(v.strip_prefix(&[50]), None);
    let prefix : &str = "he";
    assert_eq!(b"hello".strip_prefix(prefix.as_bytes()),
               Some(b"llo".as_ref()));
    • \n
  • strip_suffix
    • \n
  • binary_search(&self, x: &T) Binary searches this slice for a given element.
    • \n
  • sort_unstable(&mut self) Sorts the slice, but might not preserve the order of equal elements.
    • \n
  • rotate_left(&mut self, mid: usize) Rotates the slice in-place such that the first mid elements of the slice move to the end while the last self.len() - mid elements move to the front.
    • \n
  • fill(&mut self, value: T) Fills self with elements by cloning value.
    • \n
  • clone_from_slice(&mut self, src: &[T]) Copies the elements from src into self.
    • \n
  • copy_from_slice(&mut self, src: &[T])
    • \n
  • is_sorted(&self)
    • \n
  • take<'a, R: OneSidedRange<usize>>(self: &mut &'a Self, range: R) Removes the subslice corresponding to the given range
    • \n
  • get_many_mut<const N: usize> Returns mutable references to many indices at once.
    1
    2
    3
    4
    5
    6
    7
    #![feature(get_many_mut)]
    let v = &mut [1, 2, 3];
    if let Ok([a, b]) = v.get_many_mut([0, 2]) {
        *a = 413;
        *b = 612;
    }
    assert_eq!(v, &[413, 2, 612]);
    • \n
\n

alloc::vec::Vec

    \n
  • fn truncate(&mut self, len: usize) Shortens the vector, keeping the first len elements and dropping the rest
    • \n
\n

std::collections::VecDeque

A double-ended queue (deque) implemented with a growable ring buffer.
Since VecDeque is a ring buffer, its elements are not necessarily contiguous in memory. If you want to access the elements as a single slice, such as for efficient sorting, you can use make_contiguous. It rotates the VecDeque so that its elements do not wrap, and returns a mutable slice to the now-contiguous element sequence.

\n
    \n
  • swap(&mut self, i: usize, j: usize)
    • \n
  • reserve_exact(&mut self, additional: usize) Reserves the minimum capacity for at least additional more elements to be inserted in the given deque. Does nothing if the capacity is already sufficient.
    • \n
  • reserve(&mut self, additional: usize)
    • \n
  • shrink_to_fit(&mut self) Shrinks the capacity of the deque as much as possible.
    • \n
  • truncate(&mut self, len: usize) Shortens the deque, keeping the first len elements and dropping the rest.
    1
    2
    3
    4
    5
    6
    7
    8
    use std::collections::VecDeque;
    let mut buf = VecDeque::new();
    buf.push_back(5);
    buf.push_back(10);
    buf.push_back(15);
    assert_eq!(buf, [5, 10, 15]);
    buf.truncate(1);
    assert_eq!(buf, [5]);
    • \n
  • iter(&self)
    • \n
  • as_slices(&self)
    • \n
  • slice_ranges<R>(&self, range: R) Given a range into the logical buffer of the deque, this function return two ranges into the physical buffer that correspond to the given range
    • \n
  • range<R>(&self, range: R) Creates an iterator that covers the specified range in the deque.
    1
    2
    3
    4
    5
    6
    7
    use std::collections::VecDeque;
    let deque: VecDeque<_> = [1, 2, 3].into();
    let range = deque.range(2..).copied().collect::<VecDeque<_>>();
    assert_eq!(range, [3]);
    // A full range covers all contents
    let all = deque.range(..);
    assert_eq!(all.len(), 3);
    • \n
  • drain<R>(&mut self, range: R) Removes the specified range from the deque in bulk, returning all removed elements as an iterator.
    • \n
  • clear(&mut self)
    • \n
  • contains(&self, x: &T) Returns true if the deque contains an element equal to the given value
    • \n
  • front(&self) Provides a reference to the front element
    • \n
  • front_mut(&mut self)
    • \n
  • back(&self)
    • \n
  • back_mut(&mut self)
    • \n
  • pop_front(&mut self)
    • \n
  • pop_back(&mut self)
    • \n
  • push_front(&mut self, value: T)
    • \n
  • push_back(&mut self, value: T)
    • \n
\n

std::collections::LinkedList

"},{"title":"rust std smart pointer & interior mutability","date":"2023-06-03T14:04:38.000Z","_content":"# smart pointer\n## [Rc](https://doc.rust-lang.org/std/rc/struct.Rc.html)\nA single-threaded reference-counting pointer. The inherent methods of Rc are all associated functions, which means that you have to call them as e.g., Rc::get_mut(&mut value) instead of value.get_mut(). This avoids conflicts with methods of the inner type T.\n\n\n\n# internal mutibility\n## [Cell](https://doc.rust-lang.org/stable/std/cell/struct.Cell.html)\n`Cell` enables mutation inside an immutable value. In other words, it enables `interior mutability`. It never gives out mutable pointer to the inner value; A Cell can be shared by multiple references.\n### methods\n- `fn get(&self) -> T`\n- `fn set(&self, val: T)`\n- `fn swap(&self, other: &Cell)`\n- `fn replace(&self, val: T) -> T`\nReplaces the contained value with val, and returns the old contained value\n- `fn into_inner(self) -> T`\n- `const fn as_ptr(&self) -> *mut T`\n- `fn get_mut(&mut self) -> &mut T`\n- `fn from_mut(t: &mut T) -> &Cell`\n\n### traits\n```rust\nimpl !Sync for Cell // cannot be used in other threads\n```\n\n## [OnceCell](https://doc.rust-lang.org/stable/std/cell/struct.OnceCell.html)\nA cell which can be written to only once.\n### special methods\n- `fn get_or_init(&self, f: F) -> &T`\n\n## [LazyCell](https://doc.rust-lang.org/stable/std/cell/struct.LazyCell.html)\nA value which is initialized on the first access\n\n## [UnsafeCell](https://doc.rust-lang.org/stable/std/cell/struct.UnsafeCell.html#)\n`UnsafeCell` opts-out of the immutability guarantee for `&T`: a shared reference `&UnsafeCell` may point to data that is being mutated. This is called `interior mutability`.\nAll other types that allow internal mutability, such as `Cell` and `RefCell`, internally use `UnsafeCell` to wrap their data.\nNote that only the immutability guarantee for shared references is affected by `UnsafeCell`. The uniqueness guarantee for mutable references is unaffected (only one mutable reference at one time, or multiple immutable reference). \n\n### methods\n- `pub const fn get(&self) -> *mut T`\nGets a mutable pointer to the wrapped value.\n- `pub fn get_mut(&mut self) -> &mut T`\nReturns a mutable reference to the underlying data\n- `pub const fn raw_get(this: *const UnsafeCell) -> *mut T`\nGets a mutable pointer to the wrapped value. The difference from get is that this function accepts a raw pointer, which is useful to avoid the creation of temporary references. e.g. Gradual initialization of an UnsafeCell requires raw_get, as calling get would require creating a reference to uninitialized data:\n```rust\nuse std::cell::UnsafeCell;\nuse std::mem::MaybeUninit;\n\nlet m = MaybeUninit::>::uninit();\nunsafe { UnsafeCell::raw_get(m.as_ptr()).write(5); }\nlet uc = unsafe { m.assume_init() };\n\nassert_eq!(uc.into_inner(), 5);\n```\n- `fn into_inner(self) -> T`\nUnwraps the value, consuming the cell.\n\n## [SyncUnsafeCell](https://doc.rust-lang.org/stable/std/cell/struct.SyncUnsafeCell.html)\nThis is just an `UnsafeCell`, except it implements `Sync` if T implements Sync.\n\n## [std::cell::RefCell](https://doc.rust-lang.org/stable/std/cell/struct.RefCell.html)\nA mutable memory location with **dynamically** checked borrow rules\n- `fn borrow(&self) -> Ref<'_, T>`\n- `fn borrow_mut(&self) -> RefMut<'_, T>`\n- `fn as_ptr(&self) -> *mut T`\n\n# borrow\n## [std::borrow::Cow](https://doc.rust-lang.org/std/borrow/enum.Cow.html)\n","source":"_posts/rust/rust_std/rust-smart-pointer-and-internal-mutibility.md","raw":"---\ntitle: rust std smart pointer & interior mutability\ndate: 2023-06-03 22:04:38\ntags: [rust-std]\n---\n# smart pointer\n## [Rc](https://doc.rust-lang.org/std/rc/struct.Rc.html)\nA single-threaded reference-counting pointer. The inherent methods of Rc are all associated functions, which means that you have to call them as e.g., Rc::get_mut(&mut value) instead of value.get_mut(). This avoids conflicts with methods of the inner type T.\n\n\n\n# internal mutibility\n## [Cell](https://doc.rust-lang.org/stable/std/cell/struct.Cell.html)\n`Cell` enables mutation inside an immutable value. In other words, it enables `interior mutability`. It never gives out mutable pointer to the inner value; A Cell can be shared by multiple references.\n### methods\n- `fn get(&self) -> T`\n- `fn set(&self, val: T)`\n- `fn swap(&self, other: &Cell)`\n- `fn replace(&self, val: T) -> T`\nReplaces the contained value with val, and returns the old contained value\n- `fn into_inner(self) -> T`\n- `const fn as_ptr(&self) -> *mut T`\n- `fn get_mut(&mut self) -> &mut T`\n- `fn from_mut(t: &mut T) -> &Cell`\n\n### traits\n```rust\nimpl !Sync for Cell // cannot be used in other threads\n```\n\n## [OnceCell](https://doc.rust-lang.org/stable/std/cell/struct.OnceCell.html)\nA cell which can be written to only once.\n### special methods\n- `fn get_or_init(&self, f: F) -> &T`\n\n## [LazyCell](https://doc.rust-lang.org/stable/std/cell/struct.LazyCell.html)\nA value which is initialized on the first access\n\n## [UnsafeCell](https://doc.rust-lang.org/stable/std/cell/struct.UnsafeCell.html#)\n`UnsafeCell` opts-out of the immutability guarantee for `&T`: a shared reference `&UnsafeCell` may point to data that is being mutated. This is called `interior mutability`.\nAll other types that allow internal mutability, such as `Cell` and `RefCell`, internally use `UnsafeCell` to wrap their data.\nNote that only the immutability guarantee for shared references is affected by `UnsafeCell`. The uniqueness guarantee for mutable references is unaffected (only one mutable reference at one time, or multiple immutable reference). \n\n### methods\n- `pub const fn get(&self) -> *mut T`\nGets a mutable pointer to the wrapped value.\n- `pub fn get_mut(&mut self) -> &mut T`\nReturns a mutable reference to the underlying data\n- `pub const fn raw_get(this: *const UnsafeCell) -> *mut T`\nGets a mutable pointer to the wrapped value. The difference from get is that this function accepts a raw pointer, which is useful to avoid the creation of temporary references. e.g. Gradual initialization of an UnsafeCell requires raw_get, as calling get would require creating a reference to uninitialized data:\n```rust\nuse std::cell::UnsafeCell;\nuse std::mem::MaybeUninit;\n\nlet m = MaybeUninit::>::uninit();\nunsafe { UnsafeCell::raw_get(m.as_ptr()).write(5); }\nlet uc = unsafe { m.assume_init() };\n\nassert_eq!(uc.into_inner(), 5);\n```\n- `fn into_inner(self) -> T`\nUnwraps the value, consuming the cell.\n\n## [SyncUnsafeCell](https://doc.rust-lang.org/stable/std/cell/struct.SyncUnsafeCell.html)\nThis is just an `UnsafeCell`, except it implements `Sync` if T implements Sync.\n\n## [std::cell::RefCell](https://doc.rust-lang.org/stable/std/cell/struct.RefCell.html)\nA mutable memory location with **dynamically** checked borrow rules\n- `fn borrow(&self) -> Ref<'_, T>`\n- `fn borrow_mut(&self) -> RefMut<'_, T>`\n- `fn as_ptr(&self) -> *mut T`\n\n# borrow\n## [std::borrow::Cow](https://doc.rust-lang.org/std/borrow/enum.Cow.html)\n","slug":"rust/rust_std/rust-smart-pointer-and-internal-mutibility","published":1,"updated":"2023-11-05T04:21:13.735Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8e3003rqwsje2592e0r","content":"

smart pointer

Rc

A single-threaded reference-counting pointer. The inherent methods of Rc are all associated functions, which means that you have to call them as e.g., Rc::get_mut(&mut value) instead of value.get_mut(). This avoids conflicts with methods of the inner type T.

\n

internal mutibility

Cell

Cell<T> enables mutation inside an immutable value. In other words, it enables interior mutability. It never gives out mutable pointer to the inner value; A Cell can be shared by multiple references.

\n

methods

    \n
  • fn get(&self) -> T
    • \n
  • fn set(&self, val: T)
    • \n
  • fn swap(&self, other: &Cell<T>)
    • \n
  • fn replace(&self, val: T) -> T
    Replaces the contained value with val, and returns the old contained value
    • \n
  • fn into_inner(self) -> T
    • \n
  • const fn as_ptr(&self) -> *mut T
    • \n
  • fn get_mut(&mut self) -> &mut T
    • \n
  • fn from_mut(t: &mut T) -> &Cell<T>
    • \n
\n

traits

1
impl<T> !Sync for Cell<T>  // cannot be used in other threads
\n\n

OnceCell

A cell which can be written to only once.

\n

special methods

    \n
  • fn get_or_init<F>(&self, f: F) -> &T
    • \n
\n

LazyCell

A value which is initialized on the first access

\n

UnsafeCell

UnsafeCell<T> opts-out of the immutability guarantee for &T: a shared reference &UnsafeCell<T> may point to data that is being mutated. This is called interior mutability.
All other types that allow internal mutability, such as Cell<T> and RefCell<T>, internally use UnsafeCell to wrap their data.
Note that only the immutability guarantee for shared references is affected by UnsafeCell. The uniqueness guarantee for mutable references is unaffected (only one mutable reference at one time, or multiple immutable reference).

\n

methods

    \n
  • pub const fn get(&self) -> *mut T
    Gets a mutable pointer to the wrapped value.
    • \n
  • pub fn get_mut(&mut self) -> &mut T
    Returns a mutable reference to the underlying data
    • \n
  • pub const fn raw_get(this: *const UnsafeCell<T>) -> *mut T
    Gets a mutable pointer to the wrapped value. The difference from get is that this function accepts a raw pointer, which is useful to avoid the creation of temporary references. e.g. Gradual initialization of an UnsafeCell requires raw_get, as calling get would require creating a reference to uninitialized data:
    1
    2
    3
    4
    5
    6
    7
    8
    use std::cell::UnsafeCell;
    use std::mem::MaybeUninit;

    let m = MaybeUninit::<UnsafeCell<i32>>::uninit();
    unsafe { UnsafeCell::raw_get(m.as_ptr()).write(5); }
    let uc = unsafe { m.assume_init() };

    assert_eq!(uc.into_inner(), 5);
    • \n
  • fn into_inner(self) -> T
    Unwraps the value, consuming the cell.
    • \n
\n

SyncUnsafeCell

This is just an UnsafeCell, except it implements Sync if T implements Sync.

\n

std::cell::RefCell

A mutable memory location with dynamically checked borrow rules

\n
    \n
  • fn borrow(&self) -> Ref<'_, T>
    • \n
  • fn borrow_mut(&self) -> RefMut<'_, T>
    • \n
  • fn as_ptr(&self) -> *mut T
    • \n
\n

borrow

std::borrow::Cow

","site":{"data":{}},"excerpt":"","more":"

smart pointer

Rc

A single-threaded reference-counting pointer. The inherent methods of Rc are all associated functions, which means that you have to call them as e.g., Rc::get_mut(&mut value) instead of value.get_mut(). This avoids conflicts with methods of the inner type T.

\n

internal mutibility

Cell

Cell<T> enables mutation inside an immutable value. In other words, it enables interior mutability. It never gives out mutable pointer to the inner value; A Cell can be shared by multiple references.

\n

methods

    \n
  • fn get(&self) -> T
    • \n
  • fn set(&self, val: T)
    • \n
  • fn swap(&self, other: &Cell<T>)
    • \n
  • fn replace(&self, val: T) -> T
    Replaces the contained value with val, and returns the old contained value
    • \n
  • fn into_inner(self) -> T
    • \n
  • const fn as_ptr(&self) -> *mut T
    • \n
  • fn get_mut(&mut self) -> &mut T
    • \n
  • fn from_mut(t: &mut T) -> &Cell<T>
    • \n
\n

traits

1
impl<T> !Sync for Cell<T>  // cannot be used in other threads
\n\n

OnceCell

A cell which can be written to only once.

\n

special methods

    \n
  • fn get_or_init<F>(&self, f: F) -> &T
    • \n
\n

LazyCell

A value which is initialized on the first access

\n

UnsafeCell

UnsafeCell<T> opts-out of the immutability guarantee for &T: a shared reference &UnsafeCell<T> may point to data that is being mutated. This is called interior mutability.
All other types that allow internal mutability, such as Cell<T> and RefCell<T>, internally use UnsafeCell to wrap their data.
Note that only the immutability guarantee for shared references is affected by UnsafeCell. The uniqueness guarantee for mutable references is unaffected (only one mutable reference at one time, or multiple immutable reference).

\n

methods

    \n
  • pub const fn get(&self) -> *mut T
    Gets a mutable pointer to the wrapped value.
    • \n
  • pub fn get_mut(&mut self) -> &mut T
    Returns a mutable reference to the underlying data
    • \n
  • pub const fn raw_get(this: *const UnsafeCell<T>) -> *mut T
    Gets a mutable pointer to the wrapped value. The difference from get is that this function accepts a raw pointer, which is useful to avoid the creation of temporary references. e.g. Gradual initialization of an UnsafeCell requires raw_get, as calling get would require creating a reference to uninitialized data:
    1
    2
    3
    4
    5
    6
    7
    8
    use std::cell::UnsafeCell;
    use std::mem::MaybeUninit;

    let m = MaybeUninit::<UnsafeCell<i32>>::uninit();
    unsafe { UnsafeCell::raw_get(m.as_ptr()).write(5); }
    let uc = unsafe { m.assume_init() };

    assert_eq!(uc.into_inner(), 5);
    • \n
  • fn into_inner(self) -> T
    Unwraps the value, consuming the cell.
    • \n
\n

SyncUnsafeCell

This is just an UnsafeCell, except it implements Sync if T implements Sync.

\n

std::cell::RefCell

A mutable memory location with dynamically checked borrow rules

\n
    \n
  • fn borrow(&self) -> Ref<'_, T>
    • \n
  • fn borrow_mut(&self) -> RefMut<'_, T>
    • \n
  • fn as_ptr(&self) -> *mut T
    • \n
\n

borrow

std::borrow::Cow

"},{"title":"rust std data structure (2D)","date":"2023-05-02T14:04:38.000Z","_content":"\n## collections\n### BTreeMap\n- `clear(&mut self)` Clears the map, removing all elements.\n- `get(&self, key: &Q)` Returns a reference to the value corresponding to the key.\n- `get_key_value(&self, k: &Q)`\n- `first_key_value(&self)` eturns the first key-value pair in the map.\n- `first_entry(&mut self)` Returns the first entry in the map for in-place manipulation\n- `pop_first(&mut self)` \n- `last_key_value(&self)`\n- `last_entry(&mut self)`\n- `pop_last(&mut self)`\n- `contains_key(&self, key: &Q)`\n- `get_mut(&mut self, key: &Q)` Returns a mutable reference to the value corresponding to the key\n- `insert(&mut self, key: K, value: V)`\n- `try_insert(&mut self, key: K, value: V)` If the map already had this key present, nothing is updated, and an error containing the occupied entry and the value is returned.\n- `remove(&mut self, key: &Q)`\n- `remove_entry(&mut self, key: &Q)`\n- `retain(&mut self, mut f: F)` Retains only the elements specified by the predicate.\n```rust\nuse std::collections::BTreeMap;\nlet mut map: BTreeMap = (0..8).map(|x| (x, x*10)).collect();\n// Keep only the elements with even-numbered keys.\nmap.retain(|&k, _| k % 2 == 0);\nassert!(map.into_iter().eq(vec![(0, 0), (2, 20), (4, 40), (6, 60)]));\n```\n- `append(&mut self, other: &mut Self)` Moves all elements from `other` into `self`, leaving `other` empty.\n- `range(&self, range: R) -> Range<'_, K, V>` Constructs a double-ended iterator over a sub-range of elements in the map.\n```rust\nuse std::collections::BTreeMap;\nuse std::ops::Bound::Included;\nlet mut map = BTreeMap::new();\nmap.insert(3, \"a\");\nmap.insert(5, \"b\");\nmap.insert(8, \"c\");\nfor (&key, &value) in map.range((Included(&4), Included(&8))) {\n println!(\"{key}: {value}\");\n}\nassert_eq!(Some((&5, &\"b\")), map.range(4..).next());\n```\n- `range_mut(&mut self, range: R) -> RangeMut<'_, K, V>` \n- `entry(&mut self, key: K)` Gets the given key's corresponding entry in the map for in-place manipulation.\n```rust\nuse std::collections::BTreeMap;\nlet mut count: BTreeMap<&str, usize> = BTreeMap::new();\n// count the number of occurrences of letters in the vec\nfor x in [\"a\", \"b\", \"a\", \"c\", \"a\", \"b\"] {\n count.entry(x).and_modify(|curr| *curr += 1).or_insert(1);\n}\nassert_eq!(count[\"a\"], 3);\nassert_eq!(count[\"b\"], 2);\nassert_eq!(count[\"c\"], 1);\n```\n- `split_off(&mut self, key: &Q)` Splits the collection into two at the given key. Returns everything after the given key,\n- `drain_filter(&mut self, pred: F)` Creates an iterator that visits all elements (key-value pairs) in ascending key order and uses a closure to determine if an element should be removed. If the closure returns `true`, the element is removed from the map and yielded. If the closure returns `false`, or panics, the element remains in the map and will not be yielded\n- `into_keys(self)` Creates a consuming iterator visiting all the keys, in sorted order. The map cannot be used after calling this\n- `into_values(self)`\n","source":"_posts/rust/rust_std/rust-std-data-structure-2.md","raw":"---\ntitle: rust std data structure (2D)\ndate: 2023-05-02 22:04:38\ntags: [rust-std]\n---\n\n## collections\n### BTreeMap\n- `clear(&mut self)` Clears the map, removing all elements.\n- `get(&self, key: &Q)` Returns a reference to the value corresponding to the key.\n- `get_key_value(&self, k: &Q)`\n- `first_key_value(&self)` eturns the first key-value pair in the map.\n- `first_entry(&mut self)` Returns the first entry in the map for in-place manipulation\n- `pop_first(&mut self)` \n- `last_key_value(&self)`\n- `last_entry(&mut self)`\n- `pop_last(&mut self)`\n- `contains_key(&self, key: &Q)`\n- `get_mut(&mut self, key: &Q)` Returns a mutable reference to the value corresponding to the key\n- `insert(&mut self, key: K, value: V)`\n- `try_insert(&mut self, key: K, value: V)` If the map already had this key present, nothing is updated, and an error containing the occupied entry and the value is returned.\n- `remove(&mut self, key: &Q)`\n- `remove_entry(&mut self, key: &Q)`\n- `retain(&mut self, mut f: F)` Retains only the elements specified by the predicate.\n```rust\nuse std::collections::BTreeMap;\nlet mut map: BTreeMap = (0..8).map(|x| (x, x*10)).collect();\n// Keep only the elements with even-numbered keys.\nmap.retain(|&k, _| k % 2 == 0);\nassert!(map.into_iter().eq(vec![(0, 0), (2, 20), (4, 40), (6, 60)]));\n```\n- `append(&mut self, other: &mut Self)` Moves all elements from `other` into `self`, leaving `other` empty.\n- `range(&self, range: R) -> Range<'_, K, V>` Constructs a double-ended iterator over a sub-range of elements in the map.\n```rust\nuse std::collections::BTreeMap;\nuse std::ops::Bound::Included;\nlet mut map = BTreeMap::new();\nmap.insert(3, \"a\");\nmap.insert(5, \"b\");\nmap.insert(8, \"c\");\nfor (&key, &value) in map.range((Included(&4), Included(&8))) {\n println!(\"{key}: {value}\");\n}\nassert_eq!(Some((&5, &\"b\")), map.range(4..).next());\n```\n- `range_mut(&mut self, range: R) -> RangeMut<'_, K, V>` \n- `entry(&mut self, key: K)` Gets the given key's corresponding entry in the map for in-place manipulation.\n```rust\nuse std::collections::BTreeMap;\nlet mut count: BTreeMap<&str, usize> = BTreeMap::new();\n// count the number of occurrences of letters in the vec\nfor x in [\"a\", \"b\", \"a\", \"c\", \"a\", \"b\"] {\n count.entry(x).and_modify(|curr| *curr += 1).or_insert(1);\n}\nassert_eq!(count[\"a\"], 3);\nassert_eq!(count[\"b\"], 2);\nassert_eq!(count[\"c\"], 1);\n```\n- `split_off(&mut self, key: &Q)` Splits the collection into two at the given key. Returns everything after the given key,\n- `drain_filter(&mut self, pred: F)` Creates an iterator that visits all elements (key-value pairs) in ascending key order and uses a closure to determine if an element should be removed. If the closure returns `true`, the element is removed from the map and yielded. If the closure returns `false`, or panics, the element remains in the map and will not be yielded\n- `into_keys(self)` Creates a consuming iterator visiting all the keys, in sorted order. The map cannot be used after calling this\n- `into_values(self)`\n","slug":"rust/rust_std/rust-std-data-structure-2","published":1,"updated":"2023-11-05T04:21:13.735Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8e3003tqwsj3w7e71hn","content":"

collections

BTreeMap

    \n
  • clear(&mut self) Clears the map, removing all elements.
    • \n
  • get(&self, key: &Q) Returns a reference to the value corresponding to the key.
    • \n
  • get_key_value(&self, k: &Q)
    • \n
  • first_key_value(&self) eturns the first key-value pair in the map.
    • \n
  • first_entry(&mut self) Returns the first entry in the map for in-place manipulation
    • \n
  • pop_first(&mut self)
    • \n
  • last_key_value(&self)
    • \n
  • last_entry(&mut self)
    • \n
  • pop_last(&mut self)
    • \n
  • contains_key(&self, key: &Q)
    • \n
  • get_mut(&mut self, key: &Q) Returns a mutable reference to the value corresponding to the key
    • \n
  • insert(&mut self, key: K, value: V)
    • \n
  • try_insert(&mut self, key: K, value: V) If the map already had this key present, nothing is updated, and an error containing the occupied entry and the value is returned.
    • \n
  • remove(&mut self, key: &Q)
    • \n
  • remove_entry(&mut self, key: &Q)
    • \n
  • retain<F>(&mut self, mut f: F) Retains only the elements specified by the predicate.
    1
    2
    3
    4
    5
    use std::collections::BTreeMap;
    let mut map: BTreeMap<i32, i32> = (0..8).map(|x| (x, x*10)).collect();
    // Keep only the elements with even-numbered keys.
    map.retain(|&k, _| k % 2 == 0);
    assert!(map.into_iter().eq(vec![(0, 0), (2, 20), (4, 40), (6, 60)]));
    • \n
  • append(&mut self, other: &mut Self) Moves all elements from other into self, leaving other empty.
    • \n
  • range<T: ?Sized, R>(&self, range: R) -> Range<'_, K, V> Constructs a double-ended iterator over a sub-range of elements in the map.
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    use std::collections::BTreeMap;
    use std::ops::Bound::Included;
    let mut map = BTreeMap::new();
    map.insert(3, "a");
    map.insert(5, "b");
    map.insert(8, "c");
    for (&key, &value) in map.range((Included(&4), Included(&8))) {
        println!("{key}: {value}");
    }
    assert_eq!(Some((&5, &"b")), map.range(4..).next());
    • \n
  • range_mut<T: ?Sized, R>(&mut self, range: R) -> RangeMut<'_, K, V>
    • \n
  • entry(&mut self, key: K) Gets the given key’s corresponding entry in the map for in-place manipulation.
    1
    2
    3
    4
    5
    6
    7
    8
    9
    use std::collections::BTreeMap;
    let mut count: BTreeMap<&str, usize> = BTreeMap::new();
    // count the number of occurrences of letters in the vec
    for x in ["a", "b", "a", "c", "a", "b"] {
        count.entry(x).and_modify(|curr| *curr += 1).or_insert(1);
    }
    assert_eq!(count["a"], 3);
    assert_eq!(count["b"], 2);
    assert_eq!(count["c"], 1);
    • \n
  • split_off<Q: ?Sized + Ord>(&mut self, key: &Q) Splits the collection into two at the given key. Returns everything after the given key,
    • \n
  • drain_filter<F>(&mut self, pred: F) Creates an iterator that visits all elements (key-value pairs) in ascending key order and uses a closure to determine if an element should be removed. If the closure returns true, the element is removed from the map and yielded. If the closure returns false, or panics, the element remains in the map and will not be yielded
    • \n
  • into_keys(self) Creates a consuming iterator visiting all the keys, in sorted order. The map cannot be used after calling this
    • \n
  • into_values(self)
    • \n
\n","site":{"data":{}},"excerpt":"","more":"

collections

BTreeMap

    \n
  • clear(&mut self) Clears the map, removing all elements.
    • \n
  • get(&self, key: &Q) Returns a reference to the value corresponding to the key.
    • \n
  • get_key_value(&self, k: &Q)
    • \n
  • first_key_value(&self) eturns the first key-value pair in the map.
    • \n
  • first_entry(&mut self) Returns the first entry in the map for in-place manipulation
    • \n
  • pop_first(&mut self)
    • \n
  • last_key_value(&self)
    • \n
  • last_entry(&mut self)
    • \n
  • pop_last(&mut self)
    • \n
  • contains_key(&self, key: &Q)
    • \n
  • get_mut(&mut self, key: &Q) Returns a mutable reference to the value corresponding to the key
    • \n
  • insert(&mut self, key: K, value: V)
    • \n
  • try_insert(&mut self, key: K, value: V) If the map already had this key present, nothing is updated, and an error containing the occupied entry and the value is returned.
    • \n
  • remove(&mut self, key: &Q)
    • \n
  • remove_entry(&mut self, key: &Q)
    • \n
  • retain<F>(&mut self, mut f: F) Retains only the elements specified by the predicate.
    1
    2
    3
    4
    5
    use std::collections::BTreeMap;
    let mut map: BTreeMap<i32, i32> = (0..8).map(|x| (x, x*10)).collect();
    // Keep only the elements with even-numbered keys.
    map.retain(|&k, _| k % 2 == 0);
    assert!(map.into_iter().eq(vec![(0, 0), (2, 20), (4, 40), (6, 60)]));
    • \n
  • append(&mut self, other: &mut Self) Moves all elements from other into self, leaving other empty.
    • \n
  • range<T: ?Sized, R>(&self, range: R) -> Range<'_, K, V> Constructs a double-ended iterator over a sub-range of elements in the map.
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    use std::collections::BTreeMap;
    use std::ops::Bound::Included;
    let mut map = BTreeMap::new();
    map.insert(3, "a");
    map.insert(5, "b");
    map.insert(8, "c");
    for (&key, &value) in map.range((Included(&4), Included(&8))) {
        println!("{key}: {value}");
    }
    assert_eq!(Some((&5, &"b")), map.range(4..).next());
    • \n
  • range_mut<T: ?Sized, R>(&mut self, range: R) -> RangeMut<'_, K, V>
    • \n
  • entry(&mut self, key: K) Gets the given key’s corresponding entry in the map for in-place manipulation.
    1
    2
    3
    4
    5
    6
    7
    8
    9
    use std::collections::BTreeMap;
    let mut count: BTreeMap<&str, usize> = BTreeMap::new();
    // count the number of occurrences of letters in the vec
    for x in ["a", "b", "a", "c", "a", "b"] {
        count.entry(x).and_modify(|curr| *curr += 1).or_insert(1);
    }
    assert_eq!(count["a"], 3);
    assert_eq!(count["b"], 2);
    assert_eq!(count["c"], 1);
    • \n
  • split_off<Q: ?Sized + Ord>(&mut self, key: &Q) Splits the collection into two at the given key. Returns everything after the given key,
    • \n
  • drain_filter<F>(&mut self, pred: F) Creates an iterator that visits all elements (key-value pairs) in ascending key order and uses a closure to determine if an element should be removed. If the closure returns true, the element is removed from the map and yielded. If the closure returns false, or panics, the element remains in the map and will not be yielded
    • \n
  • into_keys(self) Creates a consuming iterator visiting all the keys, in sorted order. The map cannot be used after calling this
    • \n
  • into_values(self)
    • \n
\n"},{"title":"rust std sync","date":"2023-06-08T14:04:38.000Z","_content":"\n## [lock free & wait free](https://en.wikipedia.org/wiki/Non-blocking_algorithm)\n\"Lock-free\" and \"wait-free\" are two different approaches to designing concurrent algorithms and data structures. Both aim to provide efficient and non-blocking synchronization in concurrent environments.\n- **lock-free** A lock-free algorithm or data structure guarantees progress for at least one thread, regardless of the behavior or state of other threads. In a lock-free design, threads can independently perform their operations without being blocked by other threads. If one thread gets delayed or suspended, other threads can continue to make progress. Lock-free algorithms typically use low-level synchronization primitives such as atomic operations to ensure progress and prevent data races.\n- **wait-free** A wait-free algorithm or data structure guarantees progress for every thread, regardless of the behavior or state of other threads. In a wait-free design, every thread executing an operation completes its operation within a finite number of steps, without being delayed by other threads. Wait-free algorithms are more stringent in their requirements compared to lock-free algorithms and often require more complex synchronization mechanisms.\n\nIt's important to note that both lock-free and wait-free designs aim to avoid traditional locks or blocking synchronization mechanisms (such as mutexes or condition variables) that can lead to contention and thread blocking. Instead, they rely on techniques like atomic operations, compare-and-swap (CAS), or memory fences to ensure progress and prevent data races in concurrent execution.\n\n## [atomic](https://doc.rust-lang.org/stable/std/sync/atomic/index.html)\nRust atomics currently follow the same rules as [C++20 atomics](https://en.cppreference.com/w/cpp/atomic), specifically `atomic_ref`. Basically, creating a shared reference to one of the Rust atomic types corresponds to creating an `atomic_ref` in C++; the atomic_ref is destroyed when the lifetime of the shared reference ends. \nEach method takes an `Ordering` which represents the strength of the memory barrier for that operation. These orderings are the same as the [C++20 atomic orderings](https://en.cppreference.com/w/cpp/atomic/memory_order). For more information see the [nomicon](https://doc.rust-lang.org/stable/nomicon/atomics.html)\nAtomic variables are safe to share between threads (they implement Sync) but they do not themselves provide the mechanism for sharing and follow the threading model of Rust. The most common way to share an atomic variable is to put it into an Arc (an atomically-reference-counted shared pointer).\n\n### Compiler Reordering\nCompilers may change the actual order of events, or make events never occur! If we write something like\n```rust\nx = 1;\ny = 3;\nx = 2;\n```\nThe compiler may conclude that it would be best if your program did:\n```rust\nx = 2;\ny = 3;\n```\nThis has inverted the order of events and completely eliminated one event. But if our program is multi-threaded, we may have been relying on x to actually be assigned to 1 before y was assigned. \n\n### Hardware Reordering\nhere is indeed a global shared memory space somewhere in your hardware, but from the perspective of each CPU core it is so very far away and so very slow. Each CPU would rather work with its local cache of the data and only go through all the anguish of talking to shared memory only when it doesn't actually have that memory in cache. The end result is that the hardware doesn't guarantee that events that occur in some order on one thread, occur in the same order on another thread. To guarantee this, we must issue special instructions to the CPU telling it to be a bit less smart.\nFor instance, say we convince the compiler to emit this logic:\n```\ninitial state: x = 0, y = 1\n\nTHREAD 1 THREAD2\ny = 3; if x == 1 {\nx = 1; y *= 2;\n }\n```\nIdeally this program has 2 possible final states:\n- y = 3: (thread 2 did the check before thread 1 completed)\n- y = 6: (thread 2 did the check after thread 1 completed)\nHowever there's a third potential state that the hardware enables:\n- y = 2: (thread 2 saw x = 1, but not y = 3, and then overwrote y = 3)\nIt's worth noting that different kinds of CPU provide different guarantees. It is common to separate hardware into two categories: strongly-ordered and weakly-ordered. Most notably x86/64 provides strong ordering guarantees, while ARM provides weak ordering guarantees. \n\n### Data Accesses\nAtomic accesses are how we tell the hardware and compiler that our program is multi-threaded. Each atomic access can be marked with an ordering that specifies what kind of relationship it establishes with other accesses. For the compiler, this largely revolves around re-ordering of instructions. For the hardware, this largely revolves around how writes are propagated to other threads. The set of orderings Rust exposes are:\n- Sequentially Consistent (SeqCst)\n- Release\n- Acquire\n- Relaxed\n\n### Sequentially Consistent\nSequentially Consistent is the most powerful of all, implying the restrictions of all other orderings. Intuitively, a sequentially consistent operation cannot be reordered: all accesses on one thread that happen before and after a SeqCst access stay before and after it.\n\n### Acquire-Release\nAcquire and Release are largely intended to be paired. they're perfectly suited for acquiring and releasing locks. \nIntuitively, an acquire access ensures that every access after it stays after it. However operations that occur before an acquire are free to be reordered to occur after it. Similarly, a release access ensures that every access before it stays before it. However operations that occur after a release are free to be reordered to occur before it.\n```rust\nuse std::sync::Arc;\nuse std::sync::atomic::{AtomicBool, Ordering};\nuse std::thread;\n\nfn main() {\n let lock = Arc::new(AtomicBool::new(false)); // value answers \"am I locked?\"\n\n // ... distribute lock to threads somehow ...\n\n // Try to acquire the lock by setting it to true\n while lock.compare_and_swap(false, true, Ordering::Acquire) { }\n // broke out of the loop, so we successfully acquired the lock!\n\n // ... scary data accesses ...\n\n // ok we're done, release the lock\n lock.store(false, Ordering::Release);\n}\n```\n### Relaxed\nRelaxed accesses are the absolute weakest. They can be freely re-ordered and provide no happens-before relationship. Still, relaxed operations are still atomic. That is, they don't count as data accesses and any read-modify-write operations done to them occur atomically. For instance, incrementing a counter can be safely done by multiple threads using a relaxed `fetch_add` if you're not using the counter to synchronize any other accesses.\n\n## an example (spinlock)\n```rust\nuse std::sync::Arc;\nuse std::sync::atomic::{AtomicUsize, Ordering};\nuse std::{hint, thread};\n\nfn main() {\n let spinlock = Arc::new(AtomicUsize::new(1));\n\n let spinlock_clone = Arc::clone(&spinlock);\n let thread = thread::spawn(move|| {\n spinlock_clone.store(0, Ordering::SeqCst);\n });\n\n // Wait for the other thread to release the lock\n while spinlock.load(Ordering::SeqCst) != 0 {\n hint::spin_loop();\n }\n\n if let Err(panic) = thread.join() {\n println!(\"Thread had an error: {panic:?}\");\n }\n}\n```\n\n## usual structs\n1. [AtomicBool](https://doc.rust-lang.org/stable/std/sync/atomic/struct.AtomicBool.html)\n### methods\n- `fn get_mut(&mut self) -> &mut bool`\n- `fn into_inner(self) -> bool`\n- `fn load(&self, order: Ordering) -> bool`\n- `fn store(&self, val: bool, order: Ordering)`\n- `fn compare_exchange(&self, current: bool,new: bool,success: Ordering,failure: Ordering) -> Result`\nStores a value into the bool if the current value is the same as the current value.\ncompare_exchange takes two Ordering arguments to describe the memory ordering of this operation. success describes the required ordering for the read-modify-write operation that takes place if the comparison with current succeeds. failure describes the required ordering for the load operation that takes place when the comparison fails. \n- `fn fetch_and(&self, val: bool, order: Ordering) -> bool`\nLogical “and” with a boolean value.\nPerforms a logical “and” operation on the current value and the argument val, and sets the new value to the result.\n- `const fn as_ptr(&self) -> *mut bool`\nReturns a mutable pointer to the underlying bool.\nDoing non-atomic reads and writes on the resulting integer can be a data race. This method is mostly useful for FFI, where the function signature may use *mut bool instead of &AtomicBool.\n\n2. [AtomicUsize](https://doc.rust-lang.org/stable/std/sync/atomic/struct.AtomicUsize.html)\n2. [AtomicPtr](https://doc.rust-lang.org/stable/std/sync/atomic/struct.AtomicPtr.html)\nA raw pointer type which can be safely shared between threads.\nThis type has the same in-memory representation as a *mut T\n\n\n## Higher-level synchronization objects\nMost of the low-level synchronization primitives are quite error-prone and inconvenient to use, which is why the standard library also exposes some higher-level synchronization objects.\n- **Arc**: Atomically Reference-Counted pointer, which can be used in multithreaded environments to prolong the lifetime of some data until all the threads have finished using it.\n- **Barrier**: Ensures multiple threads will wait for each other to reach a point in the program, before continuing execution all together.\n- **Condvar**: Condition Variable, providing the ability to block a thread while waiting for an event to occur.\n- **mpsc**: Multi-producer, single-consumer queues, used for message-based communication. Can provide a lightweight inter-thread synchronisation mechanism, at the cost of some extra memory.\n- **Mutex**: Mutual Exclusion mechanism, which ensures that at most one thread at a time is able to access some data.\n- **Once**: Used for a thread-safe, one-time global initialization routine\n- **OnceLock**: Used for thread-safe, one-time initialization of a global variable.\n- **RwLock**: Provides a mutual exclusion mechanism which allows multiple readers at the same time, while allowing only one writer at a time. In some cases, this can be more efficient than a mutex.\n\n## mpsc\nThis module provides message-based communication over channels, concretely defined among three types:\n- Sender\n- SyncSender\n- Receiver","source":"_posts/rust/rust_std/rust-std-sync.md","raw":"---\ntitle: rust std sync\ndate: 2023-06-08 22:04:38\ntags: [rust-std]\n---\n\n## [lock free & wait free](https://en.wikipedia.org/wiki/Non-blocking_algorithm)\n\"Lock-free\" and \"wait-free\" are two different approaches to designing concurrent algorithms and data structures. Both aim to provide efficient and non-blocking synchronization in concurrent environments.\n- **lock-free** A lock-free algorithm or data structure guarantees progress for at least one thread, regardless of the behavior or state of other threads. In a lock-free design, threads can independently perform their operations without being blocked by other threads. If one thread gets delayed or suspended, other threads can continue to make progress. Lock-free algorithms typically use low-level synchronization primitives such as atomic operations to ensure progress and prevent data races.\n- **wait-free** A wait-free algorithm or data structure guarantees progress for every thread, regardless of the behavior or state of other threads. In a wait-free design, every thread executing an operation completes its operation within a finite number of steps, without being delayed by other threads. Wait-free algorithms are more stringent in their requirements compared to lock-free algorithms and often require more complex synchronization mechanisms.\n\nIt's important to note that both lock-free and wait-free designs aim to avoid traditional locks or blocking synchronization mechanisms (such as mutexes or condition variables) that can lead to contention and thread blocking. Instead, they rely on techniques like atomic operations, compare-and-swap (CAS), or memory fences to ensure progress and prevent data races in concurrent execution.\n\n## [atomic](https://doc.rust-lang.org/stable/std/sync/atomic/index.html)\nRust atomics currently follow the same rules as [C++20 atomics](https://en.cppreference.com/w/cpp/atomic), specifically `atomic_ref`. Basically, creating a shared reference to one of the Rust atomic types corresponds to creating an `atomic_ref` in C++; the atomic_ref is destroyed when the lifetime of the shared reference ends. \nEach method takes an `Ordering` which represents the strength of the memory barrier for that operation. These orderings are the same as the [C++20 atomic orderings](https://en.cppreference.com/w/cpp/atomic/memory_order). For more information see the [nomicon](https://doc.rust-lang.org/stable/nomicon/atomics.html)\nAtomic variables are safe to share between threads (they implement Sync) but they do not themselves provide the mechanism for sharing and follow the threading model of Rust. The most common way to share an atomic variable is to put it into an Arc (an atomically-reference-counted shared pointer).\n\n### Compiler Reordering\nCompilers may change the actual order of events, or make events never occur! If we write something like\n```rust\nx = 1;\ny = 3;\nx = 2;\n```\nThe compiler may conclude that it would be best if your program did:\n```rust\nx = 2;\ny = 3;\n```\nThis has inverted the order of events and completely eliminated one event. But if our program is multi-threaded, we may have been relying on x to actually be assigned to 1 before y was assigned. \n\n### Hardware Reordering\nhere is indeed a global shared memory space somewhere in your hardware, but from the perspective of each CPU core it is so very far away and so very slow. Each CPU would rather work with its local cache of the data and only go through all the anguish of talking to shared memory only when it doesn't actually have that memory in cache. The end result is that the hardware doesn't guarantee that events that occur in some order on one thread, occur in the same order on another thread. To guarantee this, we must issue special instructions to the CPU telling it to be a bit less smart.\nFor instance, say we convince the compiler to emit this logic:\n```\ninitial state: x = 0, y = 1\n\nTHREAD 1 THREAD2\ny = 3; if x == 1 {\nx = 1; y *= 2;\n }\n```\nIdeally this program has 2 possible final states:\n- y = 3: (thread 2 did the check before thread 1 completed)\n- y = 6: (thread 2 did the check after thread 1 completed)\nHowever there's a third potential state that the hardware enables:\n- y = 2: (thread 2 saw x = 1, but not y = 3, and then overwrote y = 3)\nIt's worth noting that different kinds of CPU provide different guarantees. It is common to separate hardware into two categories: strongly-ordered and weakly-ordered. Most notably x86/64 provides strong ordering guarantees, while ARM provides weak ordering guarantees. \n\n### Data Accesses\nAtomic accesses are how we tell the hardware and compiler that our program is multi-threaded. Each atomic access can be marked with an ordering that specifies what kind of relationship it establishes with other accesses. For the compiler, this largely revolves around re-ordering of instructions. For the hardware, this largely revolves around how writes are propagated to other threads. The set of orderings Rust exposes are:\n- Sequentially Consistent (SeqCst)\n- Release\n- Acquire\n- Relaxed\n\n### Sequentially Consistent\nSequentially Consistent is the most powerful of all, implying the restrictions of all other orderings. Intuitively, a sequentially consistent operation cannot be reordered: all accesses on one thread that happen before and after a SeqCst access stay before and after it.\n\n### Acquire-Release\nAcquire and Release are largely intended to be paired. they're perfectly suited for acquiring and releasing locks. \nIntuitively, an acquire access ensures that every access after it stays after it. However operations that occur before an acquire are free to be reordered to occur after it. Similarly, a release access ensures that every access before it stays before it. However operations that occur after a release are free to be reordered to occur before it.\n```rust\nuse std::sync::Arc;\nuse std::sync::atomic::{AtomicBool, Ordering};\nuse std::thread;\n\nfn main() {\n let lock = Arc::new(AtomicBool::new(false)); // value answers \"am I locked?\"\n\n // ... distribute lock to threads somehow ...\n\n // Try to acquire the lock by setting it to true\n while lock.compare_and_swap(false, true, Ordering::Acquire) { }\n // broke out of the loop, so we successfully acquired the lock!\n\n // ... scary data accesses ...\n\n // ok we're done, release the lock\n lock.store(false, Ordering::Release);\n}\n```\n### Relaxed\nRelaxed accesses are the absolute weakest. They can be freely re-ordered and provide no happens-before relationship. Still, relaxed operations are still atomic. That is, they don't count as data accesses and any read-modify-write operations done to them occur atomically. For instance, incrementing a counter can be safely done by multiple threads using a relaxed `fetch_add` if you're not using the counter to synchronize any other accesses.\n\n## an example (spinlock)\n```rust\nuse std::sync::Arc;\nuse std::sync::atomic::{AtomicUsize, Ordering};\nuse std::{hint, thread};\n\nfn main() {\n let spinlock = Arc::new(AtomicUsize::new(1));\n\n let spinlock_clone = Arc::clone(&spinlock);\n let thread = thread::spawn(move|| {\n spinlock_clone.store(0, Ordering::SeqCst);\n });\n\n // Wait for the other thread to release the lock\n while spinlock.load(Ordering::SeqCst) != 0 {\n hint::spin_loop();\n }\n\n if let Err(panic) = thread.join() {\n println!(\"Thread had an error: {panic:?}\");\n }\n}\n```\n\n## usual structs\n1. [AtomicBool](https://doc.rust-lang.org/stable/std/sync/atomic/struct.AtomicBool.html)\n### methods\n- `fn get_mut(&mut self) -> &mut bool`\n- `fn into_inner(self) -> bool`\n- `fn load(&self, order: Ordering) -> bool`\n- `fn store(&self, val: bool, order: Ordering)`\n- `fn compare_exchange(&self, current: bool,new: bool,success: Ordering,failure: Ordering) -> Result`\nStores a value into the bool if the current value is the same as the current value.\ncompare_exchange takes two Ordering arguments to describe the memory ordering of this operation. success describes the required ordering for the read-modify-write operation that takes place if the comparison with current succeeds. failure describes the required ordering for the load operation that takes place when the comparison fails. \n- `fn fetch_and(&self, val: bool, order: Ordering) -> bool`\nLogical “and” with a boolean value.\nPerforms a logical “and” operation on the current value and the argument val, and sets the new value to the result.\n- `const fn as_ptr(&self) -> *mut bool`\nReturns a mutable pointer to the underlying bool.\nDoing non-atomic reads and writes on the resulting integer can be a data race. This method is mostly useful for FFI, where the function signature may use *mut bool instead of &AtomicBool.\n\n2. [AtomicUsize](https://doc.rust-lang.org/stable/std/sync/atomic/struct.AtomicUsize.html)\n2. [AtomicPtr](https://doc.rust-lang.org/stable/std/sync/atomic/struct.AtomicPtr.html)\nA raw pointer type which can be safely shared between threads.\nThis type has the same in-memory representation as a *mut T\n\n\n## Higher-level synchronization objects\nMost of the low-level synchronization primitives are quite error-prone and inconvenient to use, which is why the standard library also exposes some higher-level synchronization objects.\n- **Arc**: Atomically Reference-Counted pointer, which can be used in multithreaded environments to prolong the lifetime of some data until all the threads have finished using it.\n- **Barrier**: Ensures multiple threads will wait for each other to reach a point in the program, before continuing execution all together.\n- **Condvar**: Condition Variable, providing the ability to block a thread while waiting for an event to occur.\n- **mpsc**: Multi-producer, single-consumer queues, used for message-based communication. Can provide a lightweight inter-thread synchronisation mechanism, at the cost of some extra memory.\n- **Mutex**: Mutual Exclusion mechanism, which ensures that at most one thread at a time is able to access some data.\n- **Once**: Used for a thread-safe, one-time global initialization routine\n- **OnceLock**: Used for thread-safe, one-time initialization of a global variable.\n- **RwLock**: Provides a mutual exclusion mechanism which allows multiple readers at the same time, while allowing only one writer at a time. In some cases, this can be more efficient than a mutex.\n\n## mpsc\nThis module provides message-based communication over channels, concretely defined among three types:\n- Sender\n- SyncSender\n- Receiver","slug":"rust/rust_std/rust-std-sync","published":1,"updated":"2023-11-05T04:21:13.735Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8e4003vqwsj77aj2fxl","content":"

lock free & wait free

“Lock-free” and “wait-free” are two different approaches to designing concurrent algorithms and data structures. Both aim to provide efficient and non-blocking synchronization in concurrent environments.

\n
    \n
  • lock-free A lock-free algorithm or data structure guarantees progress for at least one thread, regardless of the behavior or state of other threads. In a lock-free design, threads can independently perform their operations without being blocked by other threads. If one thread gets delayed or suspended, other threads can continue to make progress. Lock-free algorithms typically use low-level synchronization primitives such as atomic operations to ensure progress and prevent data races.
    • \n
  • wait-free A wait-free algorithm or data structure guarantees progress for every thread, regardless of the behavior or state of other threads. In a wait-free design, every thread executing an operation completes its operation within a finite number of steps, without being delayed by other threads. Wait-free algorithms are more stringent in their requirements compared to lock-free algorithms and often require more complex synchronization mechanisms.
    • \n
\n

It’s important to note that both lock-free and wait-free designs aim to avoid traditional locks or blocking synchronization mechanisms (such as mutexes or condition variables) that can lead to contention and thread blocking. Instead, they rely on techniques like atomic operations, compare-and-swap (CAS), or memory fences to ensure progress and prevent data races in concurrent execution.

\n

atomic

Rust atomics currently follow the same rules as C++20 atomics, specifically atomic_ref. Basically, creating a shared reference to one of the Rust atomic types corresponds to creating an atomic_ref in C++; the atomic_ref is destroyed when the lifetime of the shared reference ends.
Each method takes an Ordering which represents the strength of the memory barrier for that operation. These orderings are the same as the C++20 atomic orderings. For more information see the nomicon
Atomic variables are safe to share between threads (they implement Sync) but they do not themselves provide the mechanism for sharing and follow the threading model of Rust. The most common way to share an atomic variable is to put it into an Arc (an atomically-reference-counted shared pointer).

\n

Compiler Reordering

Compilers may change the actual order of events, or make events never occur! If we write something like

\n
1
2
3
x = 1;
y = 3;
x = 2;
\n

The compiler may conclude that it would be best if your program did:

\n
1
2
x = 2;
y = 3;
\n

This has inverted the order of events and completely eliminated one event. But if our program is multi-threaded, we may have been relying on x to actually be assigned to 1 before y was assigned.

\n

Hardware Reordering

here is indeed a global shared memory space somewhere in your hardware, but from the perspective of each CPU core it is so very far away and so very slow. Each CPU would rather work with its local cache of the data and only go through all the anguish of talking to shared memory only when it doesn’t actually have that memory in cache. The end result is that the hardware doesn’t guarantee that events that occur in some order on one thread, occur in the same order on another thread. To guarantee this, we must issue special instructions to the CPU telling it to be a bit less smart.
For instance, say we convince the compiler to emit this logic:

\n
1
2
3
4
5
6
initial state: x = 0, y = 1

THREAD 1        THREAD2
y = 3;          if x == 1 {
x = 1;              y *= 2;
                }
\n

Ideally this program has 2 possible final states:

\n
    \n
  • y = 3: (thread 2 did the check before thread 1 completed)
    • \n
  • y = 6: (thread 2 did the check after thread 1 completed)
    However there’s a third potential state that the hardware enables:
    • \n
  • y = 2: (thread 2 saw x = 1, but not y = 3, and then overwrote y = 3)
    It’s worth noting that different kinds of CPU provide different guarantees. It is common to separate hardware into two categories: strongly-ordered and weakly-ordered. Most notably x86/64 provides strong ordering guarantees, while ARM provides weak ordering guarantees.
    • \n
\n

Data Accesses

Atomic accesses are how we tell the hardware and compiler that our program is multi-threaded. Each atomic access can be marked with an ordering that specifies what kind of relationship it establishes with other accesses. For the compiler, this largely revolves around re-ordering of instructions. For the hardware, this largely revolves around how writes are propagated to other threads. The set of orderings Rust exposes are:

\n
    \n
  • Sequentially Consistent (SeqCst)
    • \n
  • Release
    • \n
  • Acquire
    • \n
  • Relaxed
    • \n
\n

Sequentially Consistent

Sequentially Consistent is the most powerful of all, implying the restrictions of all other orderings. Intuitively, a sequentially consistent operation cannot be reordered: all accesses on one thread that happen before and after a SeqCst access stay before and after it.

\n

Acquire-Release

Acquire and Release are largely intended to be paired. they’re perfectly suited for acquiring and releasing locks.
Intuitively, an acquire access ensures that every access after it stays after it. However operations that occur before an acquire are free to be reordered to occur after it. Similarly, a release access ensures that every access before it stays before it. However operations that occur after a release are free to be reordered to occur before it.

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::thread;

fn main() {
    let lock = Arc::new(AtomicBool::new(false)); // value answers "am I locked?"

    // ... distribute lock to threads somehow ...

    // Try to acquire the lock by setting it to true
    while lock.compare_and_swap(false, true, Ordering::Acquire) { }
    // broke out of the loop, so we successfully acquired the lock!

    // ... scary data accesses ...

    // ok we're done, release the lock
    lock.store(false, Ordering::Release);
}
\n

Relaxed

Relaxed accesses are the absolute weakest. They can be freely re-ordered and provide no happens-before relationship. Still, relaxed operations are still atomic. That is, they don’t count as data accesses and any read-modify-write operations done to them occur atomically. For instance, incrementing a counter can be safely done by multiple threads using a relaxed fetch_add if you’re not using the counter to synchronize any other accesses.

\n

an example (spinlock)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::{hint, thread};

fn main() {
    let spinlock = Arc::new(AtomicUsize::new(1));

    let spinlock_clone = Arc::clone(&spinlock);
    let thread = thread::spawn(move|| {
        spinlock_clone.store(0, Ordering::SeqCst);
    });

    // Wait for the other thread to release the lock
    while spinlock.load(Ordering::SeqCst) != 0 {
        hint::spin_loop();
    }

    if let Err(panic) = thread.join() {
        println!("Thread had an error: {panic:?}");
    }
}
\n\n

usual structs

    \n
  1. AtomicBool
    2. \n
\n

methods

    \n
  • fn get_mut(&mut self) -> &mut bool
    • \n
  • fn into_inner(self) -> bool
    • \n
  • fn load(&self, order: Ordering) -> bool
    • \n
  • fn store(&self, val: bool, order: Ordering)
    • \n
  • fn compare_exchange(&self, current: bool,new: bool,success: Ordering,failure: Ordering) -> Result<bool, bool>
    Stores a value into the bool if the current value is the same as the current value.
    compare_exchange takes two Ordering arguments to describe the memory ordering of this operation. success describes the required ordering for the read-modify-write operation that takes place if the comparison with current succeeds. failure describes the required ordering for the load operation that takes place when the comparison fails.
    • \n
  • fn fetch_and(&self, val: bool, order: Ordering) -> bool
    Logical “and” with a boolean value.
    Performs a logical “and” operation on the current value and the argument val, and sets the new value to the result.
    • \n
  • const fn as_ptr(&self) -> *mut bool
    Returns a mutable pointer to the underlying bool.
    Doing non-atomic reads and writes on the resulting integer can be a data race. This method is mostly useful for FFI, where the function signature may use *mut bool instead of &AtomicBool.
    • \n
\n
    \n
  1. AtomicUsize
    2. \n
  2. AtomicPtr
    A raw pointer type which can be safely shared between threads.
    This type has the same in-memory representation as a *mut T
    3. \n
\n

Higher-level synchronization objects

Most of the low-level synchronization primitives are quite error-prone and inconvenient to use, which is why the standard library also exposes some higher-level synchronization objects.

\n
    \n
  • Arc: Atomically Reference-Counted pointer, which can be used in multithreaded environments to prolong the lifetime of some data until all the threads have finished using it.
    • \n
  • Barrier: Ensures multiple threads will wait for each other to reach a point in the program, before continuing execution all together.
    • \n
  • Condvar: Condition Variable, providing the ability to block a thread while waiting for an event to occur.
    • \n
  • mpsc: Multi-producer, single-consumer queues, used for message-based communication. Can provide a lightweight inter-thread synchronisation mechanism, at the cost of some extra memory.
    • \n
  • Mutex: Mutual Exclusion mechanism, which ensures that at most one thread at a time is able to access some data.
    • \n
  • Once: Used for a thread-safe, one-time global initialization routine
    • \n
  • OnceLock: Used for thread-safe, one-time initialization of a global variable.
    • \n
  • RwLock: Provides a mutual exclusion mechanism which allows multiple readers at the same time, while allowing only one writer at a time. In some cases, this can be more efficient than a mutex.
    • \n
\n

mpsc

This module provides message-based communication over channels, concretely defined among three types:

\n
    \n
  • Sender
    • \n
  • SyncSender
    • \n
  • Receiver
    • \n
\n","site":{"data":{}},"excerpt":"","more":"

lock free & wait free

“Lock-free” and “wait-free” are two different approaches to designing concurrent algorithms and data structures. Both aim to provide efficient and non-blocking synchronization in concurrent environments.

\n
    \n
  • lock-free A lock-free algorithm or data structure guarantees progress for at least one thread, regardless of the behavior or state of other threads. In a lock-free design, threads can independently perform their operations without being blocked by other threads. If one thread gets delayed or suspended, other threads can continue to make progress. Lock-free algorithms typically use low-level synchronization primitives such as atomic operations to ensure progress and prevent data races.
    • \n
  • wait-free A wait-free algorithm or data structure guarantees progress for every thread, regardless of the behavior or state of other threads. In a wait-free design, every thread executing an operation completes its operation within a finite number of steps, without being delayed by other threads. Wait-free algorithms are more stringent in their requirements compared to lock-free algorithms and often require more complex synchronization mechanisms.
    • \n
\n

It’s important to note that both lock-free and wait-free designs aim to avoid traditional locks or blocking synchronization mechanisms (such as mutexes or condition variables) that can lead to contention and thread blocking. Instead, they rely on techniques like atomic operations, compare-and-swap (CAS), or memory fences to ensure progress and prevent data races in concurrent execution.

\n

atomic

Rust atomics currently follow the same rules as C++20 atomics, specifically atomic_ref. Basically, creating a shared reference to one of the Rust atomic types corresponds to creating an atomic_ref in C++; the atomic_ref is destroyed when the lifetime of the shared reference ends.
Each method takes an Ordering which represents the strength of the memory barrier for that operation. These orderings are the same as the C++20 atomic orderings. For more information see the nomicon
Atomic variables are safe to share between threads (they implement Sync) but they do not themselves provide the mechanism for sharing and follow the threading model of Rust. The most common way to share an atomic variable is to put it into an Arc (an atomically-reference-counted shared pointer).

\n

Compiler Reordering

Compilers may change the actual order of events, or make events never occur! If we write something like

\n
1
2
3
x = 1;
y = 3;
x = 2;
\n

The compiler may conclude that it would be best if your program did:

\n
1
2
x = 2;
y = 3;
\n

This has inverted the order of events and completely eliminated one event. But if our program is multi-threaded, we may have been relying on x to actually be assigned to 1 before y was assigned.

\n

Hardware Reordering

here is indeed a global shared memory space somewhere in your hardware, but from the perspective of each CPU core it is so very far away and so very slow. Each CPU would rather work with its local cache of the data and only go through all the anguish of talking to shared memory only when it doesn’t actually have that memory in cache. The end result is that the hardware doesn’t guarantee that events that occur in some order on one thread, occur in the same order on another thread. To guarantee this, we must issue special instructions to the CPU telling it to be a bit less smart.
For instance, say we convince the compiler to emit this logic:

\n
1
2
3
4
5
6
initial state: x = 0, y = 1

THREAD 1        THREAD2
y = 3;          if x == 1 {
x = 1;              y *= 2;
                }
\n

Ideally this program has 2 possible final states:

\n
    \n
  • y = 3: (thread 2 did the check before thread 1 completed)
    • \n
  • y = 6: (thread 2 did the check after thread 1 completed)
    However there’s a third potential state that the hardware enables:
    • \n
  • y = 2: (thread 2 saw x = 1, but not y = 3, and then overwrote y = 3)
    It’s worth noting that different kinds of CPU provide different guarantees. It is common to separate hardware into two categories: strongly-ordered and weakly-ordered. Most notably x86/64 provides strong ordering guarantees, while ARM provides weak ordering guarantees.
    • \n
\n

Data Accesses

Atomic accesses are how we tell the hardware and compiler that our program is multi-threaded. Each atomic access can be marked with an ordering that specifies what kind of relationship it establishes with other accesses. For the compiler, this largely revolves around re-ordering of instructions. For the hardware, this largely revolves around how writes are propagated to other threads. The set of orderings Rust exposes are:

\n
    \n
  • Sequentially Consistent (SeqCst)
    • \n
  • Release
    • \n
  • Acquire
    • \n
  • Relaxed
    • \n
\n

Sequentially Consistent

Sequentially Consistent is the most powerful of all, implying the restrictions of all other orderings. Intuitively, a sequentially consistent operation cannot be reordered: all accesses on one thread that happen before and after a SeqCst access stay before and after it.

\n

Acquire-Release

Acquire and Release are largely intended to be paired. they’re perfectly suited for acquiring and releasing locks.
Intuitively, an acquire access ensures that every access after it stays after it. However operations that occur before an acquire are free to be reordered to occur after it. Similarly, a release access ensures that every access before it stays before it. However operations that occur after a release are free to be reordered to occur before it.

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::thread;

fn main() {
    let lock = Arc::new(AtomicBool::new(false)); // value answers "am I locked?"

    // ... distribute lock to threads somehow ...

    // Try to acquire the lock by setting it to true
    while lock.compare_and_swap(false, true, Ordering::Acquire) { }
    // broke out of the loop, so we successfully acquired the lock!

    // ... scary data accesses ...

    // ok we're done, release the lock
    lock.store(false, Ordering::Release);
}
\n

Relaxed

Relaxed accesses are the absolute weakest. They can be freely re-ordered and provide no happens-before relationship. Still, relaxed operations are still atomic. That is, they don’t count as data accesses and any read-modify-write operations done to them occur atomically. For instance, incrementing a counter can be safely done by multiple threads using a relaxed fetch_add if you’re not using the counter to synchronize any other accesses.

\n

an example (spinlock)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::{hint, thread};

fn main() {
    let spinlock = Arc::new(AtomicUsize::new(1));

    let spinlock_clone = Arc::clone(&spinlock);
    let thread = thread::spawn(move|| {
        spinlock_clone.store(0, Ordering::SeqCst);
    });

    // Wait for the other thread to release the lock
    while spinlock.load(Ordering::SeqCst) != 0 {
        hint::spin_loop();
    }

    if let Err(panic) = thread.join() {
        println!("Thread had an error: {panic:?}");
    }
}
\n\n

usual structs

    \n
  1. AtomicBool
    2. \n
\n

methods

    \n
  • fn get_mut(&mut self) -> &mut bool
    • \n
  • fn into_inner(self) -> bool
    • \n
  • fn load(&self, order: Ordering) -> bool
    • \n
  • fn store(&self, val: bool, order: Ordering)
    • \n
  • fn compare_exchange(&self, current: bool,new: bool,success: Ordering,failure: Ordering) -> Result<bool, bool>
    Stores a value into the bool if the current value is the same as the current value.
    compare_exchange takes two Ordering arguments to describe the memory ordering of this operation. success describes the required ordering for the read-modify-write operation that takes place if the comparison with current succeeds. failure describes the required ordering for the load operation that takes place when the comparison fails.
    • \n
  • fn fetch_and(&self, val: bool, order: Ordering) -> bool
    Logical “and” with a boolean value.
    Performs a logical “and” operation on the current value and the argument val, and sets the new value to the result.
    • \n
  • const fn as_ptr(&self) -> *mut bool
    Returns a mutable pointer to the underlying bool.
    Doing non-atomic reads and writes on the resulting integer can be a data race. This method is mostly useful for FFI, where the function signature may use *mut bool instead of &AtomicBool.
    • \n
\n
    \n
  1. AtomicUsize
    2. \n
  2. AtomicPtr
    A raw pointer type which can be safely shared between threads.
    This type has the same in-memory representation as a *mut T
    3. \n
\n

Higher-level synchronization objects

Most of the low-level synchronization primitives are quite error-prone and inconvenient to use, which is why the standard library also exposes some higher-level synchronization objects.

\n
    \n
  • Arc: Atomically Reference-Counted pointer, which can be used in multithreaded environments to prolong the lifetime of some data until all the threads have finished using it.
    • \n
  • Barrier: Ensures multiple threads will wait for each other to reach a point in the program, before continuing execution all together.
    • \n
  • Condvar: Condition Variable, providing the ability to block a thread while waiting for an event to occur.
    • \n
  • mpsc: Multi-producer, single-consumer queues, used for message-based communication. Can provide a lightweight inter-thread synchronisation mechanism, at the cost of some extra memory.
    • \n
  • Mutex: Mutual Exclusion mechanism, which ensures that at most one thread at a time is able to access some data.
    • \n
  • Once: Used for a thread-safe, one-time global initialization routine
    • \n
  • OnceLock: Used for thread-safe, one-time initialization of a global variable.
    • \n
  • RwLock: Provides a mutual exclusion mechanism which allows multiple readers at the same time, while allowing only one writer at a time. In some cases, this can be more efficient than a mutex.
    • \n
\n

mpsc

This module provides message-based communication over channels, concretely defined among three types:

\n
    \n
  • Sender
    • \n
  • SyncSender
    • \n
  • Receiver
    • \n
\n"},{"title":"inline ptx assembly in cuda","date":"2023-11-15T06:47:28.000Z","_content":"# introduction\nPTX, a low-level parallel thread execution virtual machine and instruction set architecture (ISA). PTX exposes the GPU as a data-parallel computing device.\nThe PTX Instruction Set Architecture (ISA) is lower-level compared to CUDA C++, but the programming model is entirely the same, with only some terminology differences. For example: CTA (Cooperative Thread Array): Equivalent to the Block in the CUDA thread model.\n\n# assembler statements\n## parameters\n```cpp\nasm(\"template-string\" : \"constraint\"(output) : \"constraint\"(input));\n```\nan example\n```cpp\nasm(\"add.s32 %0, %1, %2;\" : \"=r\"(i) : \"r\"(j), \"r\"(k));\n```\nEach `%n` in the template string is an index into the following list of operands, in text order. So `%0` refers to the first operand, `%1` to the second operand, and so on. Since the output operands are always listed ahead of the input operands, they are assigned the smallest indices. \nNote that the numbered references in the string can be in arbitrary order. The following is equivalent to the above example:\n```cpp\nasm(\"add.s32 %0, %2, %1;\" : \"=r\"(i) : \"r\"(k), \"r\"(j));\n```\nYou can also repeat a reference, e.g.:\n```cpp\nasm(\"add.s32 %0, %1, %1;\" : \"=r\"(i) : \"r\"(k));\n```\nIf there is no input operand, you can drop the final colon, e.g.:\n```cpp\nasm(\"mov.s32 %0, 2;\" : \"=r\"(i));\n```\n\nIf there is no output operand, the colon separators are adjacent, e.g.:\n\n\n```cpp\nasm(\"mov.s32 r1, %0;\" :: \"r\"(i));\n```\nthe “r” constraint refers to a 32bit integer register. The “=” modifier in “=r” specifies that the register is written to. There is also available a “+” modifier that specifies the register is both read and written\n\nMultiple instructions can be combined into a single asm() statement; \nam example\n```cpp\n__device__ int cube (int x)\n{\n int y;\n asm(\".reg .u32 t1;\\n\\t\" // temp reg t1\n \" mul.lo.u32 t1, %1, %1;\\n\\t\" // t1 = x * x\n \" mul.lo.u32 %0, t1, %1;\" // y = t1 * x\n : \"=r\"(y) : \"r\" (x));\n return y;\n}\n```\n\n## constraints\nThere is a separate constraint letter for each PTX register type:\n```cpp\n\"h\" = .u16 reg\n\"r\" = .u32 reg\n\"l\" = .u64 reg\n\"f\" = .f32 reg\n\"d\" = .f64 reg\n```\nThe constraint \"n\" may be used for immediate integer operands with a known value. Example:\n```cpp\nasm(\"add.u32 %0, %0, %1;\" : \"=r\"(x) : \"n\"(42));\n```\n\n\n# State Space + Type + Identifier\n## state spaces\nA state space is a storage area with particular characteristics. All variables reside in some state space. \n| Name | Description |\n| ----------- | ----------- |\n| .reg | Registers, fast. |\n| .sreg | Special registers. Read-only; pre-defined; platform-specific. |\n|.const|Shared, read-only memory.|\n|.global|Global memory, shared by all threads.|\n|.local|Local memory, private to each thread.|\n|.param|Kernel parameters, defined per-grid; or Function or local parameters, defined per-thread.|\n|.shared|Addressable memory, defined per CTA, accessible to all threads in the cluster throughout the lifetime of the CTA that defines it.|\n\n\n## types\n### fundamental types\nIn PTX, the fundamental types reflect the native data types supported by the target architectures.\n| Basic Type | Fundamental Type Specifiers |\n| ----------- | ----------- |\n| Signed integer | .s8, .s16, .s32, .s64 |\n|Unsigned integer|.u8, .u16, .u32, .u64|\n|Floating-point|.f16, .f16x2, .f32, .f64|\n|Bits (untyped)|.b8, .b16, .b32, .b64, .b128|\n|Predicate|.pred|\n\n## variables\nIn PTX, a variable declaration describes both the variable’s type and its state space. In addition to fundamental types, PTX supports types for simple aggregate objects such as vectors and arrays.\n**examples**\n```cpp\n.global .v4 .f32 V; // a length-4 vector of floats\n.shared .v2 .u16 uv; // a length-2 vector of unsigned ints\n.global .v4 .b8 v; // a length-4 vector of bytes\n```\n\n# references\n- [nvidia docs inlline ptx assembly](https://docs.nvidia.com/cuda/inline-ptx-assembly/index.html)\n- [IBM Open XL C/C++ Inline Assembly](https://www.ibm.com/docs/en/openxl-c-and-cpp-aix/17.1.0?topic=features-inline-assembly-statements)\n- [parallel thread execution ISA guide](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#)","source":"_posts/cuda/inline-ptx-assembly-in-cuda.md","raw":"---\ntitle: inline ptx assembly in cuda\ndate: 2023-11-15 14:47:28\ntags: [cuda]\n---\n# introduction\nPTX, a low-level parallel thread execution virtual machine and instruction set architecture (ISA). PTX exposes the GPU as a data-parallel computing device.\nThe PTX Instruction Set Architecture (ISA) is lower-level compared to CUDA C++, but the programming model is entirely the same, with only some terminology differences. For example: CTA (Cooperative Thread Array): Equivalent to the Block in the CUDA thread model.\n\n# assembler statements\n## parameters\n```cpp\nasm(\"template-string\" : \"constraint\"(output) : \"constraint\"(input));\n```\nan example\n```cpp\nasm(\"add.s32 %0, %1, %2;\" : \"=r\"(i) : \"r\"(j), \"r\"(k));\n```\nEach `%n` in the template string is an index into the following list of operands, in text order. So `%0` refers to the first operand, `%1` to the second operand, and so on. Since the output operands are always listed ahead of the input operands, they are assigned the smallest indices. \nNote that the numbered references in the string can be in arbitrary order. The following is equivalent to the above example:\n```cpp\nasm(\"add.s32 %0, %2, %1;\" : \"=r\"(i) : \"r\"(k), \"r\"(j));\n```\nYou can also repeat a reference, e.g.:\n```cpp\nasm(\"add.s32 %0, %1, %1;\" : \"=r\"(i) : \"r\"(k));\n```\nIf there is no input operand, you can drop the final colon, e.g.:\n```cpp\nasm(\"mov.s32 %0, 2;\" : \"=r\"(i));\n```\n\nIf there is no output operand, the colon separators are adjacent, e.g.:\n\n\n```cpp\nasm(\"mov.s32 r1, %0;\" :: \"r\"(i));\n```\nthe “r” constraint refers to a 32bit integer register. The “=” modifier in “=r” specifies that the register is written to. There is also available a “+” modifier that specifies the register is both read and written\n\nMultiple instructions can be combined into a single asm() statement; \nam example\n```cpp\n__device__ int cube (int x)\n{\n int y;\n asm(\".reg .u32 t1;\\n\\t\" // temp reg t1\n \" mul.lo.u32 t1, %1, %1;\\n\\t\" // t1 = x * x\n \" mul.lo.u32 %0, t1, %1;\" // y = t1 * x\n : \"=r\"(y) : \"r\" (x));\n return y;\n}\n```\n\n## constraints\nThere is a separate constraint letter for each PTX register type:\n```cpp\n\"h\" = .u16 reg\n\"r\" = .u32 reg\n\"l\" = .u64 reg\n\"f\" = .f32 reg\n\"d\" = .f64 reg\n```\nThe constraint \"n\" may be used for immediate integer operands with a known value. Example:\n```cpp\nasm(\"add.u32 %0, %0, %1;\" : \"=r\"(x) : \"n\"(42));\n```\n\n\n# State Space + Type + Identifier\n## state spaces\nA state space is a storage area with particular characteristics. All variables reside in some state space. \n| Name | Description |\n| ----------- | ----------- |\n| .reg | Registers, fast. |\n| .sreg | Special registers. Read-only; pre-defined; platform-specific. |\n|.const|Shared, read-only memory.|\n|.global|Global memory, shared by all threads.|\n|.local|Local memory, private to each thread.|\n|.param|Kernel parameters, defined per-grid; or Function or local parameters, defined per-thread.|\n|.shared|Addressable memory, defined per CTA, accessible to all threads in the cluster throughout the lifetime of the CTA that defines it.|\n\n\n## types\n### fundamental types\nIn PTX, the fundamental types reflect the native data types supported by the target architectures.\n| Basic Type | Fundamental Type Specifiers |\n| ----------- | ----------- |\n| Signed integer | .s8, .s16, .s32, .s64 |\n|Unsigned integer|.u8, .u16, .u32, .u64|\n|Floating-point|.f16, .f16x2, .f32, .f64|\n|Bits (untyped)|.b8, .b16, .b32, .b64, .b128|\n|Predicate|.pred|\n\n## variables\nIn PTX, a variable declaration describes both the variable’s type and its state space. In addition to fundamental types, PTX supports types for simple aggregate objects such as vectors and arrays.\n**examples**\n```cpp\n.global .v4 .f32 V; // a length-4 vector of floats\n.shared .v2 .u16 uv; // a length-2 vector of unsigned ints\n.global .v4 .b8 v; // a length-4 vector of bytes\n```\n\n# references\n- [nvidia docs inlline ptx assembly](https://docs.nvidia.com/cuda/inline-ptx-assembly/index.html)\n- [IBM Open XL C/C++ Inline Assembly](https://www.ibm.com/docs/en/openxl-c-and-cpp-aix/17.1.0?topic=features-inline-assembly-statements)\n- [parallel thread execution ISA guide](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#)","slug":"cuda/inline-ptx-assembly-in-cuda","published":1,"updated":"2023-11-22T02:15:56.342Z","_id":"clp3h29tl0000p97ug63eajx3","comments":1,"layout":"post","photos":[],"link":"","content":"

introduction

PTX, a low-level parallel thread execution virtual machine and instruction set architecture (ISA). PTX exposes the GPU as a data-parallel computing device.
The PTX Instruction Set Architecture (ISA) is lower-level compared to CUDA C++, but the programming model is entirely the same, with only some terminology differences. For example: CTA (Cooperative Thread Array): Equivalent to the Block in the CUDA thread model.

\n

assembler statements

parameters

1
asm("template-string" : "constraint"(output) : "constraint"(input));
\n

an example

\n
1
asm("add.s32 %0, %1, %2;" : "=r"(i) : "r"(j), "r"(k));
\n

Each %n in the template string is an index into the following list of operands, in text order. So %0 refers to the first operand, %1 to the second operand, and so on. Since the output operands are always listed ahead of the input operands, they are assigned the smallest indices.
Note that the numbered references in the string can be in arbitrary order. The following is equivalent to the above example:

\n
1
asm("add.s32 %0, %2, %1;" : "=r"(i) : "r"(k), "r"(j));
\n

You can also repeat a reference, e.g.:

\n
1
asm("add.s32 %0, %1, %1;" : "=r"(i) : "r"(k));
\n

If there is no input operand, you can drop the final colon, e.g.:

\n
1
asm("mov.s32 %0, 2;" : "=r"(i));
\n\n

If there is no output operand, the colon separators are adjacent, e.g.:

\n
1
asm("mov.s32 r1, %0;" :: "r"(i));
\n

the “r” constraint refers to a 32bit integer register. The “=” modifier in “=r” specifies that the register is written to. There is also available a “+” modifier that specifies the register is both read and written

\n

Multiple instructions can be combined into a single asm() statement;
am example

\n
1
2
3
4
5
6
7
8
9
__device__ int cube (int x)
{
  int y;
  asm(".reg .u32 t1;\\n\\t"              // temp reg t1
      " mul.lo.u32 t1, %1, %1;\\n\\t"    // t1 = x * x
      " mul.lo.u32 %0, t1, %1;"        // y = t1 * x
      : "=r"(y) : "r" (x));
  return y;
}
\n\n

constraints

There is a separate constraint letter for each PTX register type:

\n
1
2
3
4
5
"h" = .u16 reg
"r" = .u32 reg
"l" = .u64 reg
"f" = .f32 reg
"d" = .f64 reg
\n

The constraint “n” may be used for immediate integer operands with a known value. Example:

\n
1
asm("add.u32 %0, %0, %1;" : "=r"(x) : "n"(42));
\n\n\n

State Space + Type + Identifier

state spaces

A state space is a storage area with particular characteristics. All variables reside in some state space.

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
NameDescription
.regRegisters, fast.
.sregSpecial registers. Read-only; pre-defined; platform-specific.
.constShared, read-only memory.
.globalGlobal memory, shared by all threads.
.localLocal memory, private to each thread.
.paramKernel parameters, defined per-grid; or Function or local parameters, defined per-thread.
.sharedAddressable memory, defined per CTA, accessible to all threads in the cluster throughout the lifetime of the CTA that defines it.
\n

types

fundamental types

In PTX, the fundamental types reflect the native data types supported by the target architectures.

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
Basic TypeFundamental Type Specifiers
Signed integer.s8, .s16, .s32, .s64
Unsigned integer.u8, .u16, .u32, .u64
Floating-point.f16, .f16x2, .f32, .f64
Bits (untyped).b8, .b16, .b32, .b64, .b128
Predicate.pred
\n

variables

In PTX, a variable declaration describes both the variable’s type and its state space. In addition to fundamental types, PTX supports types for simple aggregate objects such as vectors and arrays.
examples

\n
1
2
3
.global .v4 .f32 V;   // a length-4 vector of floats
.shared .v2 .u16 uv;  // a length-2 vector of unsigned ints
.global .v4 .b8  v;   // a length-4 vector of bytes
\n\n

references

\n","site":{"data":{}},"excerpt":"","more":"

introduction

PTX, a low-level parallel thread execution virtual machine and instruction set architecture (ISA). PTX exposes the GPU as a data-parallel computing device.
The PTX Instruction Set Architecture (ISA) is lower-level compared to CUDA C++, but the programming model is entirely the same, with only some terminology differences. For example: CTA (Cooperative Thread Array): Equivalent to the Block in the CUDA thread model.

\n

assembler statements

parameters

1
asm("template-string" : "constraint"(output) : "constraint"(input));
\n

an example

\n
1
asm("add.s32 %0, %1, %2;" : "=r"(i) : "r"(j), "r"(k));
\n

Each %n in the template string is an index into the following list of operands, in text order. So %0 refers to the first operand, %1 to the second operand, and so on. Since the output operands are always listed ahead of the input operands, they are assigned the smallest indices.
Note that the numbered references in the string can be in arbitrary order. The following is equivalent to the above example:

\n
1
asm("add.s32 %0, %2, %1;" : "=r"(i) : "r"(k), "r"(j));
\n

You can also repeat a reference, e.g.:

\n
1
asm("add.s32 %0, %1, %1;" : "=r"(i) : "r"(k));
\n

If there is no input operand, you can drop the final colon, e.g.:

\n
1
asm("mov.s32 %0, 2;" : "=r"(i));
\n\n

If there is no output operand, the colon separators are adjacent, e.g.:

\n
1
asm("mov.s32 r1, %0;" :: "r"(i));
\n

the “r” constraint refers to a 32bit integer register. The “=” modifier in “=r” specifies that the register is written to. There is also available a “+” modifier that specifies the register is both read and written

\n

Multiple instructions can be combined into a single asm() statement;
am example

\n
1
2
3
4
5
6
7
8
9
__device__ int cube (int x)
{
  int y;
  asm(".reg .u32 t1;\\n\\t"              // temp reg t1
      " mul.lo.u32 t1, %1, %1;\\n\\t"    // t1 = x * x
      " mul.lo.u32 %0, t1, %1;"        // y = t1 * x
      : "=r"(y) : "r" (x));
  return y;
}
\n\n

constraints

There is a separate constraint letter for each PTX register type:

\n
1
2
3
4
5
"h" = .u16 reg
"r" = .u32 reg
"l" = .u64 reg
"f" = .f32 reg
"d" = .f64 reg
\n

The constraint “n” may be used for immediate integer operands with a known value. Example:

\n
1
asm("add.u32 %0, %0, %1;" : "=r"(x) : "n"(42));
\n\n\n

State Space + Type + Identifier

state spaces

A state space is a storage area with particular characteristics. All variables reside in some state space.

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
NameDescription
.regRegisters, fast.
.sregSpecial registers. Read-only; pre-defined; platform-specific.
.constShared, read-only memory.
.globalGlobal memory, shared by all threads.
.localLocal memory, private to each thread.
.paramKernel parameters, defined per-grid; or Function or local parameters, defined per-thread.
.sharedAddressable memory, defined per CTA, accessible to all threads in the cluster throughout the lifetime of the CTA that defines it.
\n

types

fundamental types

In PTX, the fundamental types reflect the native data types supported by the target architectures.

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
Basic TypeFundamental Type Specifiers
Signed integer.s8, .s16, .s32, .s64
Unsigned integer.u8, .u16, .u32, .u64
Floating-point.f16, .f16x2, .f32, .f64
Bits (untyped).b8, .b16, .b32, .b64, .b128
Predicate.pred
\n

variables

In PTX, a variable declaration describes both the variable’s type and its state space. In addition to fundamental types, PTX supports types for simple aggregate objects such as vectors and arrays.
examples

\n
1
2
3
.global .v4 .f32 V;   // a length-4 vector of floats
.shared .v2 .u16 uv;  // a length-2 vector of unsigned ints
.global .v4 .b8  v;   // a length-4 vector of bytes
\n\n

references

\n"},{"title":"cpp generics","date":"2023-11-21T03:35:20.000Z","_content":"\n# generics\n## variadic template parameter\n```cpp\ntemplate\nvoid printValues(const Types&... values) {\n // Fold expression (C++17 and later) to print all values\n (std::cout << ... << values) << std::endl;\n}\n```\n\n# macros\n## examples\n```cpp\n#if defined(_WIN32)\n#else\n#endif\n```\n- `__cplusplus` macro is a predefined macro in C++ that is used to indicate the version of the C++ standard being used by the compiler. The value of `__cplusplus` is an integer that represents the version.\n- __FILE__\n`__FILE__` is a predefined macro in C and C++ that expands to the name of the current source file as a string literal.\n- __LINE__\n`__LINE__` is a predefined macro in C and C++ that expands to the current line number in the source code.","source":"_posts/cpp/generics_and_macros.md","raw":"---\ntitle: cpp generics\ndate: 2023-11-21 11:35:20\ntags: [cpp]\n---\n\n# generics\n## variadic template parameter\n```cpp\ntemplate\nvoid printValues(const Types&... values) {\n // Fold expression (C++17 and later) to print all values\n (std::cout << ... << values) << std::endl;\n}\n```\n\n# macros\n## examples\n```cpp\n#if defined(_WIN32)\n#else\n#endif\n```\n- `__cplusplus` macro is a predefined macro in C++ that is used to indicate the version of the C++ standard being used by the compiler. The value of `__cplusplus` is an integer that represents the version.\n- __FILE__\n`__FILE__` is a predefined macro in C and C++ that expands to the name of the current source file as a string literal.\n- __LINE__\n`__LINE__` is a predefined macro in C and C++ that expands to the current line number in the source code.","slug":"cpp/generics_and_macros","published":1,"updated":"2023-11-23T02:30:16.168Z","_id":"clp8cm0vx0000l57u7eqtbi6s","comments":1,"layout":"post","photos":[],"link":"","content":"

generics

variadic template parameter

1
2
3
4
5
template<typename... Types>
void printValues(const Types&... values) {
    // Fold expression (C++17 and later) to print all values
    (std::cout << ... << values) << std::endl;
}
\n\n

macros

examples

1
2
3
#if defined(_WIN32)
#else
#endif
\n
    \n
  • __cplusplus macro is a predefined macro in C++ that is used to indicate the version of the C++ standard being used by the compiler. The value of __cplusplus is an integer that represents the version.
    • \n
  • FILE
    __FILE__ is a predefined macro in C and C++ that expands to the name of the current source file as a string literal.
    • \n
  • LINE
    __LINE__ is a predefined macro in C and C++ that expands to the current line number in the source code.
    • \n
\n","site":{"data":{}},"excerpt":"","more":"

generics

variadic template parameter

1
2
3
4
5
template<typename... Types>
void printValues(const Types&... values) {
    // Fold expression (C++17 and later) to print all values
    (std::cout << ... << values) << std::endl;
}
\n\n

macros

examples

1
2
3
#if defined(_WIN32)
#else
#endif
\n
    \n
  • __cplusplus macro is a predefined macro in C++ that is used to indicate the version of the C++ standard being used by the compiler. The value of __cplusplus is an integer that represents the version.
    • \n
  • FILE
    __FILE__ is a predefined macro in C and C++ that expands to the name of the current source file as a string literal.
    • \n
  • LINE
    __LINE__ is a predefined macro in C and C++ that expands to the current line number in the source code.
    • \n
\n"},{"title":"cpp key workds","date":"2023-11-21T03:35:20.000Z","_content":"\n- `decltype`\n`decltype` is a keyword that is used to obtain the type of an expression or an entity\n```cpp\n#include \n\nint main() {\n int x = 42;\n double y = 3.14;\n\n // Using decltype to declare a variable with the same type as another variable\n decltype(x) result1 = x; // result1 has the type int\n decltype(y) result2 = y; // result2 has the type double\n\n // Using decltype with an expression\n int a = 10;\n int b = 20;\n decltype(a + b) sum = a + b; // sum has the type int\n\n return 0;\n}\n```\n\n- `friend`\nIn C++, the friend keyword is used to grant non-member functions or other classes access to the private and protected members of a class. When a function or class is declared as a friend of another class, it is allowed to access private and protected members of that class as if it were a member of that class.","source":"_posts/cpp/key_words.md","raw":"---\ntitle: cpp key workds\ndate: 2023-11-21 11:35:20\ntags: [cpp]\n---\n\n- `decltype`\n`decltype` is a keyword that is used to obtain the type of an expression or an entity\n```cpp\n#include \n\nint main() {\n int x = 42;\n double y = 3.14;\n\n // Using decltype to declare a variable with the same type as another variable\n decltype(x) result1 = x; // result1 has the type int\n decltype(y) result2 = y; // result2 has the type double\n\n // Using decltype with an expression\n int a = 10;\n int b = 20;\n decltype(a + b) sum = a + b; // sum has the type int\n\n return 0;\n}\n```\n\n- `friend`\nIn C++, the friend keyword is used to grant non-member functions or other classes access to the private and protected members of a class. When a function or class is declared as a friend of another class, it is allowed to access private and protected members of that class as if it were a member of that class.","slug":"cpp/key_words","published":1,"updated":"2023-11-23T07:05:38.353Z","_id":"clp8cm0w50003l57ubl7e5d6y","comments":1,"layout":"post","photos":[],"link":"","content":"
    \n

  • decltype
    decltype is a keyword that is used to obtain the type of an expression or an entity

    \n
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    #include <iostream>

    int main() {
        int x = 42;
        double y = 3.14;

        // Using decltype to declare a variable with the same type as another variable
        decltype(x) result1 = x;  // result1 has the type int
        decltype(y) result2 = y;  // result2 has the type double

        // Using decltype with an expression
        int a = 10;
        int b = 20;
        decltype(a + b) sum = a + b;  // sum has the type int

        return 0;
    }
    \n
    • \n

  • friend
    In C++, the friend keyword is used to grant non-member functions or other classes access to the private and protected members of a class. When a function or class is declared as a friend of another class, it is allowed to access private and protected members of that class as if it were a member of that class.

    \n
    • \n
\n","site":{"data":{}},"excerpt":"","more":"
    \n

  • decltype
    decltype is a keyword that is used to obtain the type of an expression or an entity

    \n
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    #include <iostream>

    int main() {
        int x = 42;
        double y = 3.14;

        // Using decltype to declare a variable with the same type as another variable
        decltype(x) result1 = x;  // result1 has the type int
        decltype(y) result2 = y;  // result2 has the type double

        // Using decltype with an expression
        int a = 10;
        int b = 20;
        decltype(a + b) sum = a + b;  // sum has the type int

        return 0;
    }
    \n
    • \n

  • friend
    In C++, the friend keyword is used to grant non-member functions or other classes access to the private and protected members of a class. When a function or class is declared as a friend of another class, it is allowed to access private and protected members of that class as if it were a member of that class.

    \n
    • \n
\n"},{"title":"cpp std","date":"2023-11-21T03:35:20.000Z","_content":"\n\n# snprintf\n`snprintf` is a function that formats a string and writes the resulting characters to a character array.\n```cpp\nint snprintf(char *str, size_t size, const char *format, ...);\n```\n- `str`: A pointer to the destination buffer where the resulting string is stored.\n- `size`: The size of the destination buffer (including the null-terminating character).\n- `format`: A format string that specifies the desired format of the output.\n- `...`: Additional arguments corresponding to the placeholders in the format string.\n```cpp\nint main() {\n size_t n = std::snprintf(nullptr, 0, \"%s\", \"hello, world\");\n std::cout << n << std::endl; // print 12\n return 0;\n}\n```\n\n# string\n## `std::string(n, '\\0')` \ncreates a string with a length of `n` and fills it with null characters ('\\0').\n```cpp\nsize_t n = 10;\nstd::string myString(n, '\\0');\n```\n## `strerror_r` \nis a function that is commonly used for thread-safe retrieval of error messages. The purpose of this function is to convert an error number (an integer typically set by a system call or library function to indicate an error) into a human-readable error message\n```cpp\n#include \n\nint strerror_r(int errnum, char *buf, size_t buflen);\n```\n- `errnum`: The error number for which you want to retrieve the error message.\n- `buf`: A pointer to the buffer where the error message will be stored.\n- `buflen`: The size of the buffer pointed to by buf.\n\n## others\n- `strncpy`: Copy no more than N characters of SRC to DEST.\n- `std::strlen`: Return the length of String.\n- `std::strstr` is a standard C library function that is used to find the first occurrence of a substring within a string. The function returns a pointer to the first occurrence of the substring in the given string, or NULL if the substring is not found.","source":"_posts/cpp/std.md","raw":"---\ntitle: cpp std\ndate: 2023-11-21 11:35:20\ntags: [cpp]\n---\n\n\n# snprintf\n`snprintf` is a function that formats a string and writes the resulting characters to a character array.\n```cpp\nint snprintf(char *str, size_t size, const char *format, ...);\n```\n- `str`: A pointer to the destination buffer where the resulting string is stored.\n- `size`: The size of the destination buffer (including the null-terminating character).\n- `format`: A format string that specifies the desired format of the output.\n- `...`: Additional arguments corresponding to the placeholders in the format string.\n```cpp\nint main() {\n size_t n = std::snprintf(nullptr, 0, \"%s\", \"hello, world\");\n std::cout << n << std::endl; // print 12\n return 0;\n}\n```\n\n# string\n## `std::string(n, '\\0')` \ncreates a string with a length of `n` and fills it with null characters ('\\0').\n```cpp\nsize_t n = 10;\nstd::string myString(n, '\\0');\n```\n## `strerror_r` \nis a function that is commonly used for thread-safe retrieval of error messages. The purpose of this function is to convert an error number (an integer typically set by a system call or library function to indicate an error) into a human-readable error message\n```cpp\n#include \n\nint strerror_r(int errnum, char *buf, size_t buflen);\n```\n- `errnum`: The error number for which you want to retrieve the error message.\n- `buf`: A pointer to the buffer where the error message will be stored.\n- `buflen`: The size of the buffer pointed to by buf.\n\n## others\n- `strncpy`: Copy no more than N characters of SRC to DEST.\n- `std::strlen`: Return the length of String.\n- `std::strstr` is a standard C library function that is used to find the first occurrence of a substring within a string. The function returns a pointer to the first occurrence of the substring in the given string, or NULL if the substring is not found.","slug":"cpp/std","published":1,"updated":"2023-11-21T13:05:50.461Z","_id":"clp8cm0w60004l57uckd04dmz","comments":1,"layout":"post","photos":[],"link":"","content":"

snprintf

snprintf is a function that formats a string and writes the resulting characters to a character array.

\n
1
int snprintf(char *str, size_t size, const char *format, ...);
\n
    \n
  • str: A pointer to the destination buffer where the resulting string is stored.
    • \n
  • size: The size of the destination buffer (including the null-terminating character).
    • \n
  • format: A format string that specifies the desired format of the output.
    • \n
  • ...: Additional arguments corresponding to the placeholders in the format string.
    1
    2
    3
    4
    5
    int main() {
        size_t n = std::snprintf(nullptr, 0, "%s", "hello, world");
        std::cout << n << std::endl;  // print 12
        return 0;
    }
    • \n
\n

string

std::string(n, '\\0')

creates a string with a length of n and fills it with null characters (‘\\0’).

\n
1
2
size_t n = 10;
std::string myString(n, '\\0');
\n

strerror_r

is a function that is commonly used for thread-safe retrieval of error messages. The purpose of this function is to convert an error number (an integer typically set by a system call or library function to indicate an error) into a human-readable error message

\n
1
2
3
#include <cstring>

int strerror_r(int errnum, char *buf, size_t buflen);
\n
    \n
  • errnum: The error number for which you want to retrieve the error message.
    • \n
  • buf: A pointer to the buffer where the error message will be stored.
    • \n
  • buflen: The size of the buffer pointed to by buf.
    • \n
\n

others

    \n
  • strncpy: Copy no more than N characters of SRC to DEST.
    • \n
  • std::strlen: Return the length of String.
    • \n
  • std::strstr is a standard C library function that is used to find the first occurrence of a substring within a string. The function returns a pointer to the first occurrence of the substring in the given string, or NULL if the substring is not found.
    • \n
\n","site":{"data":{}},"excerpt":"","more":"

snprintf

snprintf is a function that formats a string and writes the resulting characters to a character array.

\n
1
int snprintf(char *str, size_t size, const char *format, ...);
\n
    \n
  • str: A pointer to the destination buffer where the resulting string is stored.
    • \n
  • size: The size of the destination buffer (including the null-terminating character).
    • \n
  • format: A format string that specifies the desired format of the output.
    • \n
  • ...: Additional arguments corresponding to the placeholders in the format string.
    1
    2
    3
    4
    5
    int main() {
        size_t n = std::snprintf(nullptr, 0, "%s", "hello, world");
        std::cout << n << std::endl;  // print 12
        return 0;
    }
    • \n
\n

string

std::string(n, '\\0')

creates a string with a length of n and fills it with null characters (‘\\0’).

\n
1
2
size_t n = 10;
std::string myString(n, '\\0');
\n

strerror_r

is a function that is commonly used for thread-safe retrieval of error messages. The purpose of this function is to convert an error number (an integer typically set by a system call or library function to indicate an error) into a human-readable error message

\n
1
2
3
#include <cstring>

int strerror_r(int errnum, char *buf, size_t buflen);
\n
    \n
  • errnum: The error number for which you want to retrieve the error message.
    • \n
  • buf: A pointer to the buffer where the error message will be stored.
    • \n
  • buflen: The size of the buffer pointed to by buf.
    • \n
\n

others

    \n
  • strncpy: Copy no more than N characters of SRC to DEST.
    • \n
  • std::strlen: Return the length of String.
    • \n
  • std::strstr is a standard C library function that is used to find the first occurrence of a substring within a string. The function returns a pointer to the first occurrence of the substring in the given string, or NULL if the substring is not found.
    • \n
\n"},{"title":"cpp types","date":"2023-11-15T03:35:20.000Z","_content":"\n## Conversion Operator\n```cpp\nclass MyIntegerWrapper {\nprivate:\n int value;\n\npublic:\n // Constructor\n MyIntegerWrapper(int val) : value(val) {}\n\n // Conversion operator to int\n operator int() const {\n return value;\n }\n};\n\nint main() {\n MyIntegerWrapper myObj(42);\n\n // Implicit conversion using the conversion operator\n int intValue = myObj;\n std::cout << \"Implicit Conversion: \" << intValue << std::endl;\n\n // Explicit conversion using static_cast\n double doubleValue = static_cast(myObj);\n std::cout << \"Explicit Conversion: \" << doubleValue << std::endl;\n\n return 0;\n}\n```","source":"_posts/cpp/types.md","raw":"---\ntitle: cpp types\ndate: 2023-11-15 11:35:20\ntags: [cpp]\n---\n\n## Conversion Operator\n```cpp\nclass MyIntegerWrapper {\nprivate:\n int value;\n\npublic:\n // Constructor\n MyIntegerWrapper(int val) : value(val) {}\n\n // Conversion operator to int\n operator int() const {\n return value;\n }\n};\n\nint main() {\n MyIntegerWrapper myObj(42);\n\n // Implicit conversion using the conversion operator\n int intValue = myObj;\n std::cout << \"Implicit Conversion: \" << intValue << std::endl;\n\n // Explicit conversion using static_cast\n double doubleValue = static_cast(myObj);\n std::cout << \"Explicit Conversion: \" << doubleValue << std::endl;\n\n return 0;\n}\n```","slug":"cpp/types","published":1,"updated":"2023-11-21T12:26:56.654Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clp8cm0w60006l57u0fvme5s2","content":"

Conversion Operator

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
class MyIntegerWrapper {
private:
    int value;

public:
    // Constructor
    MyIntegerWrapper(int val) : value(val) {}

    // Conversion operator to int
    operator int() const {
        return value;
    }
};

int main() {
    MyIntegerWrapper myObj(42);

    // Implicit conversion using the conversion operator
    int intValue = myObj;
    std::cout << "Implicit Conversion: " << intValue << std::endl;

    // Explicit conversion using static_cast
    double doubleValue = static_cast<double>(myObj);
    std::cout << "Explicit Conversion: " << doubleValue << std::endl;

    return 0;
}
","site":{"data":{}},"excerpt":"","more":"

Conversion Operator

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
class MyIntegerWrapper {
private:
    int value;

public:
    // Constructor
    MyIntegerWrapper(int val) : value(val) {}

    // Conversion operator to int
    operator int() const {
        return value;
    }
};

int main() {
    MyIntegerWrapper myObj(42);

    // Implicit conversion using the conversion operator
    int intValue = myObj;
    std::cout << "Implicit Conversion: " << intValue << std::endl;

    // Explicit conversion using static_cast
    double doubleValue = static_cast<double>(myObj);
    std::cout << "Explicit Conversion: " << doubleValue << std::endl;

    return 0;
}
"},{"title":"key concepts in cuda","date":"2023-11-22T06:47:28.000Z","_content":"\n## key concepts\n- **Warps**: In CUDA, a warp is a fundamental execution unit in the GPU. It represents a group of threads that are executed together in a SIMD (Single Instruction, Multiple Data) fashion on a single GPU core. In CUDA, each group of 32 consecutive threads is called a warp. A Warp is the primary unit of execution in an SM. Once a thread block is allocated to an SM, it will be further divided into a set of warps for execution.\n- **Stream Multiprocessor** (SM)\nThe CUDA architecture is built around a scalable array of multithreaded Streaming Multiprocessors (SMs). When a CUDA program on the host CPU invokes a kernel grid, the blocks of the grid are enumerated and distributed to multiprocessors with available execution capacity. The threads of a thread block execute concurrently on one multiprocessor, and multiple thread blocks can execute concurrently on one multiprocessor. As thread blocks terminate, new blocks are launched on the vacated multiprocessors\nEach SM contains 8 CUDA cores, and at any one time they're executing a single warp of 32 threads - so it takes 4 clock cycles to issue a single instruction for the whole warp. You can assume that threads in any given warp execute in lock-step, but to synchronise across warps, you need to use `__syncthreads()`\n\n## memory\n- shared_memory: https://developer.nvidia.com/blog/using-shared-memory-cuda-cc/\n- [Page-Locked Host Memory](https://leimao.github.io/blog/Page-Locked-Host-Memory-Data-Transfer/)\n\n## stream\nA sequence of operations that execute in issue-order on the GPU\n- CUDA operations in different streams may run concurrently\n- CUDA operations from different streams may be interleaved\n![cuda stream example](images/cuda/cuda_stream_example.png)\n\n## event\nevents are synchronization mechanisms used to measure and manage time, particularly for timing purposes and performance profiling. \n- event creation\n```cpp\n#include \n\ncudaEvent_t startEvent, stopEvent;\ncudaEventCreate(&startEvent);\ncudaEventCreate(&stopEvent);\n```\n- record event\n```cpp\ncudaEventRecord(startEvent, 0);\n// ... GPU code to be timed ...\ncudaEventRecord(stopEvent, 0);\n```\nHere, `cudaEventRecord` records the current time in the events startEvent and stopEvent. The second argument, `0`, specifies the stream in which to record the event (0 means the default stream).\n- synchonize events\n```cpp\ncudaEventSynchronize(stopEvent);\n```\nThis function call makes the CPU wait until the GPU has completed all tasks associated with the stopEvent.\n- Calculate Elapsed Time\n```cpp\nfloat milliseconds = 0;\ncudaEventElapsedTime(&milliseconds, startEvent, stopEvent);\n```\n- destroy event\n```cpp\ncudaEventDestroy(startEvent);\ncudaEventDestroy(stopEvent);\n```\n## keywords\n- __global__ functions that are executed on the GPU, called from CPU\n- __device__ functions that are executed on the GPU, called within GPU\n- __shared__ \n- __align__ is a compiler directive that can be used to specify alignment requirements for variables in device code.\n\n- `__launch_bounds__`\nthe __launch_bounds__ attribute is used to specify launch bounds for a kernel function. The launch bounds control the number of threads per block and the maximum number of blocks per multiprocessor for a specific kernel.\n```cpp\n__launch_bounds__(maxThreadsPerBlock, minBlocksPerMultiprocessor)\n```\n\n## macros\n- __CUDACC__ is a predefined macro in CUDA C/C++ that is used to determine whether the code is being compiled by the NVIDIA CUDA C/C++ compiler (nvcc). \n- __NVCC__ is a predefined macro that is automatically defined by the NVIDIA CUDA compiler (nvcc). It allows you to conditionally compile code based on whether the code is being processed by the CUDA compiler or a regular C++ compiler\n\n## usage\nthe triple-chevron notation `<<<...>>>` is used to specify the execution configuration of a CUDA kernel launch. \n```cpp\nkernel<<>>(...);\n```\n- `gridDim`: Specifies the dimensions of the grid (number of blocks). It can be a 1D, 2D, or 3D configuration, specified as (gridDimX, gridDimY, gridDimZ).\n\n- `blockDim`: Specifies the dimensions of each block (number of threads per block). It can also be a 1D, 2D, or 3D configuration, specified as (blockDimX, blockDimY, blockDimZ).\n\n- `sharedMem`: Optional parameter specifying the dynamic shared memory per block in bytes. If not specified, it defaults to 0.\n\n- `stream`: Optional parameter specifying the CUDA stream in which the kernel should be executed. If not specified, it defaults to the default stream (0).","source":"_posts/cuda/concepts_and_keywords.md","raw":"---\ntitle: key concepts in cuda\ndate: 2023-11-22 14:47:28\ntags: [cuda]\n---\n\n## key concepts\n- **Warps**: In CUDA, a warp is a fundamental execution unit in the GPU. It represents a group of threads that are executed together in a SIMD (Single Instruction, Multiple Data) fashion on a single GPU core. In CUDA, each group of 32 consecutive threads is called a warp. A Warp is the primary unit of execution in an SM. Once a thread block is allocated to an SM, it will be further divided into a set of warps for execution.\n- **Stream Multiprocessor** (SM)\nThe CUDA architecture is built around a scalable array of multithreaded Streaming Multiprocessors (SMs). When a CUDA program on the host CPU invokes a kernel grid, the blocks of the grid are enumerated and distributed to multiprocessors with available execution capacity. The threads of a thread block execute concurrently on one multiprocessor, and multiple thread blocks can execute concurrently on one multiprocessor. As thread blocks terminate, new blocks are launched on the vacated multiprocessors\nEach SM contains 8 CUDA cores, and at any one time they're executing a single warp of 32 threads - so it takes 4 clock cycles to issue a single instruction for the whole warp. You can assume that threads in any given warp execute in lock-step, but to synchronise across warps, you need to use `__syncthreads()`\n\n## memory\n- shared_memory: https://developer.nvidia.com/blog/using-shared-memory-cuda-cc/\n- [Page-Locked Host Memory](https://leimao.github.io/blog/Page-Locked-Host-Memory-Data-Transfer/)\n\n## stream\nA sequence of operations that execute in issue-order on the GPU\n- CUDA operations in different streams may run concurrently\n- CUDA operations from different streams may be interleaved\n![cuda stream example](images/cuda/cuda_stream_example.png)\n\n## event\nevents are synchronization mechanisms used to measure and manage time, particularly for timing purposes and performance profiling. \n- event creation\n```cpp\n#include \n\ncudaEvent_t startEvent, stopEvent;\ncudaEventCreate(&startEvent);\ncudaEventCreate(&stopEvent);\n```\n- record event\n```cpp\ncudaEventRecord(startEvent, 0);\n// ... GPU code to be timed ...\ncudaEventRecord(stopEvent, 0);\n```\nHere, `cudaEventRecord` records the current time in the events startEvent and stopEvent. The second argument, `0`, specifies the stream in which to record the event (0 means the default stream).\n- synchonize events\n```cpp\ncudaEventSynchronize(stopEvent);\n```\nThis function call makes the CPU wait until the GPU has completed all tasks associated with the stopEvent.\n- Calculate Elapsed Time\n```cpp\nfloat milliseconds = 0;\ncudaEventElapsedTime(&milliseconds, startEvent, stopEvent);\n```\n- destroy event\n```cpp\ncudaEventDestroy(startEvent);\ncudaEventDestroy(stopEvent);\n```\n## keywords\n- __global__ functions that are executed on the GPU, called from CPU\n- __device__ functions that are executed on the GPU, called within GPU\n- __shared__ \n- __align__ is a compiler directive that can be used to specify alignment requirements for variables in device code.\n\n- `__launch_bounds__`\nthe __launch_bounds__ attribute is used to specify launch bounds for a kernel function. The launch bounds control the number of threads per block and the maximum number of blocks per multiprocessor for a specific kernel.\n```cpp\n__launch_bounds__(maxThreadsPerBlock, minBlocksPerMultiprocessor)\n```\n\n## macros\n- __CUDACC__ is a predefined macro in CUDA C/C++ that is used to determine whether the code is being compiled by the NVIDIA CUDA C/C++ compiler (nvcc). \n- __NVCC__ is a predefined macro that is automatically defined by the NVIDIA CUDA compiler (nvcc). It allows you to conditionally compile code based on whether the code is being processed by the CUDA compiler or a regular C++ compiler\n\n## usage\nthe triple-chevron notation `<<<...>>>` is used to specify the execution configuration of a CUDA kernel launch. \n```cpp\nkernel<<>>(...);\n```\n- `gridDim`: Specifies the dimensions of the grid (number of blocks). It can be a 1D, 2D, or 3D configuration, specified as (gridDimX, gridDimY, gridDimZ).\n\n- `blockDim`: Specifies the dimensions of each block (number of threads per block). It can also be a 1D, 2D, or 3D configuration, specified as (blockDimX, blockDimY, blockDimZ).\n\n- `sharedMem`: Optional parameter specifying the dynamic shared memory per block in bytes. If not specified, it defaults to 0.\n\n- `stream`: Optional parameter specifying the CUDA stream in which the kernel should be executed. If not specified, it defaults to the default stream (0).","slug":"cuda/concepts_and_keywords","published":1,"updated":"2023-11-23T07:30:03.676Z","_id":"clp95py9p0000e37u3ivq1as7","comments":1,"layout":"post","photos":[],"link":"","content":"

key concepts

    \n
  • Warps: In CUDA, a warp is a fundamental execution unit in the GPU. It represents a group of threads that are executed together in a SIMD (Single Instruction, Multiple Data) fashion on a single GPU core. In CUDA, each group of 32 consecutive threads is called a warp. A Warp is the primary unit of execution in an SM. Once a thread block is allocated to an SM, it will be further divided into a set of warps for execution.
    • \n
  • Stream Multiprocessor (SM)
    The CUDA architecture is built around a scalable array of multithreaded Streaming Multiprocessors (SMs). When a CUDA program on the host CPU invokes a kernel grid, the blocks of the grid are enumerated and distributed to multiprocessors with available execution capacity. The threads of a thread block execute concurrently on one multiprocessor, and multiple thread blocks can execute concurrently on one multiprocessor. As thread blocks terminate, new blocks are launched on the vacated multiprocessors
    Each SM contains 8 CUDA cores, and at any one time they’re executing a single warp of 32 threads - so it takes 4 clock cycles to issue a single instruction for the whole warp. You can assume that threads in any given warp execute in lock-step, but to synchronise across warps, you need to use __syncthreads()
    • \n
\n

memory

\n

stream

A sequence of operations that execute in issue-order on the GPU

\n
    \n
  • CUDA operations in different streams may run concurrently
    • \n
  • CUDA operations from different streams may be interleaved
    \"cuda
    • \n
\n

event

events are synchronization mechanisms used to measure and manage time, particularly for timing purposes and performance profiling.

\n
    \n
  • event creation
    1
    2
    3
    4
    5
    #include <cuda_runtime.h>

    cudaEvent_t startEvent, stopEvent;
    cudaEventCreate(&startEvent);
    cudaEventCreate(&stopEvent);
    • \n
  • record event
    1
    2
    3
    cudaEventRecord(startEvent, 0);
    // ... GPU code to be timed ...
    cudaEventRecord(stopEvent, 0);
    \nHere, cudaEventRecord records the current time in the events startEvent and stopEvent. The second argument, 0, specifies the stream in which to record the event (0 means the default stream).
    • \n
  • synchonize events
    1
    cudaEventSynchronize(stopEvent);
    \nThis function call makes the CPU wait until the GPU has completed all tasks associated with the stopEvent.
    • \n
  • Calculate Elapsed Time
    1
    2
    float milliseconds = 0;
    cudaEventElapsedTime(&milliseconds, startEvent, stopEvent);
    • \n
  • destroy event
    1
    2
    cudaEventDestroy(startEvent);
    cudaEventDestroy(stopEvent);
    • \n
\n

keywords

    \n

  • global functions that are executed on the GPU, called from CPU

    \n
    • \n

  • device functions that are executed on the GPU, called within GPU

    \n
    • \n

  • shared

    \n
    • \n

  • align is a compiler directive that can be used to specify alignment requirements for variables in device code.

    \n
    • \n

  • __launch_bounds__
    the launch_bounds attribute is used to specify launch bounds for a kernel function. The launch bounds control the number of threads per block and the maximum number of blocks per multiprocessor for a specific kernel.

    \n
    1
    __launch_bounds__(maxThreadsPerBlock, minBlocksPerMultiprocessor)
    • \n
\n

macros

    \n
  • CUDACC is a predefined macro in CUDA C/C++ that is used to determine whether the code is being compiled by the NVIDIA CUDA C/C++ compiler (nvcc).
    • \n
  • NVCC is a predefined macro that is automatically defined by the NVIDIA CUDA compiler (nvcc). It allows you to conditionally compile code based on whether the code is being processed by the CUDA compiler or a regular C++ compiler
    • \n
\n

usage

the triple-chevron notation <<<...>>> is used to specify the execution configuration of a CUDA kernel launch.

\n
1
kernel<<<gridDim, blockDim, sharedMem, stream>>>(...);
\n
    \n

  • gridDim: Specifies the dimensions of the grid (number of blocks). It can be a 1D, 2D, or 3D configuration, specified as (gridDimX, gridDimY, gridDimZ).

    \n
    • \n

  • blockDim: Specifies the dimensions of each block (number of threads per block). It can also be a 1D, 2D, or 3D configuration, specified as (blockDimX, blockDimY, blockDimZ).

    \n
    • \n

  • sharedMem: Optional parameter specifying the dynamic shared memory per block in bytes. If not specified, it defaults to 0.

    \n
    • \n

  • stream: Optional parameter specifying the CUDA stream in which the kernel should be executed. If not specified, it defaults to the default stream (0).

    \n
    • \n
\n","site":{"data":{}},"excerpt":"","more":"

key concepts

    \n
  • Warps: In CUDA, a warp is a fundamental execution unit in the GPU. It represents a group of threads that are executed together in a SIMD (Single Instruction, Multiple Data) fashion on a single GPU core. In CUDA, each group of 32 consecutive threads is called a warp. A Warp is the primary unit of execution in an SM. Once a thread block is allocated to an SM, it will be further divided into a set of warps for execution.
    • \n
  • Stream Multiprocessor (SM)
    The CUDA architecture is built around a scalable array of multithreaded Streaming Multiprocessors (SMs). When a CUDA program on the host CPU invokes a kernel grid, the blocks of the grid are enumerated and distributed to multiprocessors with available execution capacity. The threads of a thread block execute concurrently on one multiprocessor, and multiple thread blocks can execute concurrently on one multiprocessor. As thread blocks terminate, new blocks are launched on the vacated multiprocessors
    Each SM contains 8 CUDA cores, and at any one time they’re executing a single warp of 32 threads - so it takes 4 clock cycles to issue a single instruction for the whole warp. You can assume that threads in any given warp execute in lock-step, but to synchronise across warps, you need to use __syncthreads()
    • \n
\n

memory

\n

stream

A sequence of operations that execute in issue-order on the GPU

\n
    \n
  • CUDA operations in different streams may run concurrently
    • \n
  • CUDA operations from different streams may be interleaved
    \"cuda
    • \n
\n

event

events are synchronization mechanisms used to measure and manage time, particularly for timing purposes and performance profiling.

\n
    \n
  • event creation
    1
    2
    3
    4
    5
    #include <cuda_runtime.h>

    cudaEvent_t startEvent, stopEvent;
    cudaEventCreate(&startEvent);
    cudaEventCreate(&stopEvent);
    • \n
  • record event
    1
    2
    3
    cudaEventRecord(startEvent, 0);
    // ... GPU code to be timed ...
    cudaEventRecord(stopEvent, 0);
    \nHere, cudaEventRecord records the current time in the events startEvent and stopEvent. The second argument, 0, specifies the stream in which to record the event (0 means the default stream).
    • \n
  • synchonize events
    1
    cudaEventSynchronize(stopEvent);
    \nThis function call makes the CPU wait until the GPU has completed all tasks associated with the stopEvent.
    • \n
  • Calculate Elapsed Time
    1
    2
    float milliseconds = 0;
    cudaEventElapsedTime(&milliseconds, startEvent, stopEvent);
    • \n
  • destroy event
    1
    2
    cudaEventDestroy(startEvent);
    cudaEventDestroy(stopEvent);
    • \n
\n

keywords

    \n

  • global functions that are executed on the GPU, called from CPU

    \n
    • \n

  • device functions that are executed on the GPU, called within GPU

    \n
    • \n

  • shared

    \n
    • \n

  • align is a compiler directive that can be used to specify alignment requirements for variables in device code.

    \n
    • \n

  • __launch_bounds__
    the launch_bounds attribute is used to specify launch bounds for a kernel function. The launch bounds control the number of threads per block and the maximum number of blocks per multiprocessor for a specific kernel.

    \n
    1
    __launch_bounds__(maxThreadsPerBlock, minBlocksPerMultiprocessor)
    • \n
\n

macros

    \n
  • CUDACC is a predefined macro in CUDA C/C++ that is used to determine whether the code is being compiled by the NVIDIA CUDA C/C++ compiler (nvcc).
    • \n
  • NVCC is a predefined macro that is automatically defined by the NVIDIA CUDA compiler (nvcc). It allows you to conditionally compile code based on whether the code is being processed by the CUDA compiler or a regular C++ compiler
    • \n
\n

usage

the triple-chevron notation <<<...>>> is used to specify the execution configuration of a CUDA kernel launch.

\n
1
kernel<<<gridDim, blockDim, sharedMem, stream>>>(...);
\n
    \n

  • gridDim: Specifies the dimensions of the grid (number of blocks). It can be a 1D, 2D, or 3D configuration, specified as (gridDimX, gridDimY, gridDimZ).

    \n
    • \n

  • blockDim: Specifies the dimensions of each block (number of threads per block). It can also be a 1D, 2D, or 3D configuration, specified as (blockDimX, blockDimY, blockDimZ).

    \n
    • \n

  • sharedMem: Optional parameter specifying the dynamic shared memory per block in bytes. If not specified, it defaults to 0.

    \n
    • \n

  • stream: Optional parameter specifying the CUDA stream in which the kernel should be executed. If not specified, it defaults to the default stream (0).

    \n
    • \n
\n"},{"title":"cuda standard api & tools","date":"2023-11-22T06:47:28.000Z","_content":"\n# API\n- `__syncthreads()`: A thread’s execution can only proceed past a __syncthreads() after all threads in its block have executed the __syncthreads()\n- `__syncwarp()` is an intrinsic function that provides a warp-level synchronization point within a warp. A warp is a group of threads in CUDA that execute instructions in lockstep.\n\n\n# Tool\n## cmd\n- nvidia-smi\n\n\n\n## references\n- [cuda runtime api](https://docs.nvidia.com/cuda/cuda-runtime-api/index.html)","source":"_posts/cuda/cuda_api_and_tools.md","raw":"---\ntitle: cuda standard api & tools\ndate: 2023-11-22 14:47:28\ntags: [cuda]\n---\n\n# API\n- `__syncthreads()`: A thread’s execution can only proceed past a __syncthreads() after all threads in its block have executed the __syncthreads()\n- `__syncwarp()` is an intrinsic function that provides a warp-level synchronization point within a warp. A warp is a group of threads in CUDA that execute instructions in lockstep.\n\n\n# Tool\n## cmd\n- nvidia-smi\n\n\n\n## references\n- [cuda runtime api](https://docs.nvidia.com/cuda/cuda-runtime-api/index.html)","slug":"cuda/cuda_api_and_tools","published":1,"updated":"2023-11-23T07:41:25.289Z","_id":"clp95py9w0003e37uhwx0baye","comments":1,"layout":"post","photos":[],"link":"","content":"

API

    \n
  • __syncthreads(): A thread’s execution can only proceed past a __syncthreads() after all threads in its block have executed the __syncthreads()
    • \n
  • __syncwarp() is an intrinsic function that provides a warp-level synchronization point within a warp. A warp is a group of threads in CUDA that execute instructions in lockstep.
    • \n
\n

Tool

cmd

    \n
  • nvidia-smi
    • \n
\n

references

\n","site":{"data":{}},"excerpt":"","more":"

API

    \n
  • __syncthreads(): A thread’s execution can only proceed past a __syncthreads() after all threads in its block have executed the __syncthreads()
    • \n
  • __syncwarp() is an intrinsic function that provides a warp-level synchronization point within a warp. A warp is a group of threads in CUDA that execute instructions in lockstep.
    • \n
\n

Tool

cmd

    \n
  • nvidia-smi
    • \n
\n

references

\n"},{"title":"cryptography arithmetic tricks","date":"2023-11-22T06:47:28.000Z","_content":"\n\n\n\n- get the next m*SIZE\n```cpp\n# define WARP_SZ 32\nsize_t n = (nelems+WARP_SZ-1) & ((size_t)0-WARP_SZ)\n```\n0-WARP_SIZE = 0xffffffffffffffe0\neo = 0b11100000\nit will select multiples of WARP_SZ\nnelems+WARP_SZ-1 will counter for the removed numbers by 00000\n\nn will be [32, 64, 96, ...]","source":"_posts/arithmatic/tricks.md","raw":"---\ntitle: cryptography arithmetic tricks\ndate: 2023-11-22 14:47:28\ntags: [arithmatic]\n---\n\n\n\n\n- get the next m*SIZE\n```cpp\n# define WARP_SZ 32\nsize_t n = (nelems+WARP_SZ-1) & ((size_t)0-WARP_SZ)\n```\n0-WARP_SIZE = 0xffffffffffffffe0\neo = 0b11100000\nit will select multiples of WARP_SZ\nnelems+WARP_SZ-1 will counter for the removed numbers by 00000\n\nn will be [32, 64, 96, ...]","slug":"arithmatic/tricks","published":1,"updated":"2023-12-08T06:18:33.813Z","_id":"clp96plbb0006e37uei4tcdkw","comments":1,"layout":"post","photos":[],"link":"","content":"\n\n\n
    \n
  • get the next m*SIZE
    1
    2
    # define WARP_SZ 32
    size_t n = (nelems+WARP_SZ-1) & ((size_t)0-WARP_SZ)
    \n0-WARP_SIZE = 0xffffffffffffffe0
    eo = 0b11100000
    it will select multiples of WARP_SZ
    nelems+WARP_SZ-1 will counter for the removed numbers by 00000
    • \n
\n

n will be [32, 64, 96, …]

\n","site":{"data":{}},"excerpt":"","more":"\n\n\n
    \n
  • get the next m*SIZE
    1
    2
    # define WARP_SZ 32
    size_t n = (nelems+WARP_SZ-1) & ((size_t)0-WARP_SZ)
    \n0-WARP_SIZE = 0xffffffffffffffe0
    eo = 0b11100000
    it will select multiples of WARP_SZ
    nelems+WARP_SZ-1 will counter for the removed numbers by 00000
    • \n
\n

n will be [32, 64, 96, …]

\n"},{"title":"classic implementations of arithematic of multi precision big integers","date":"2023-11-22T06:47:28.000Z","_content":"\n\n\n\n\n## multiple-precision integer arithmetic\nIf \\\\(b ≥ 2\\\\) is an integer, then any positive integer \\\\(a\\\\) can be expressed uniquely as \\\\(a = a_n \\cdot b^n+a_{n−1}\\cdot b^{n−1}+···+a_1 \\cdot b+a_0\\\\), where \\\\(a_i\\\\) is an integer with \\\\(0≤a_i \n\n\n\n## multiple-precision integer arithmetic\nIf \\\\(b ≥ 2\\\\) is an integer, then any positive integer \\\\(a\\\\) can be expressed uniquely as \\\\(a = a_n \\cdot b^n+a_{n−1}\\cdot b^{n−1}+···+a_1 \\cdot b+a_0\\\\), where \\\\(a_i\\\\) is an integer with \\\\(0≤a_i \n\n\n\n

multiple-precision integer arithmetic

If \\(b ≥ 2\\) is an integer, then any positive integer \\(a\\) can be expressed uniquely as \\(a = a_n \\cdot b^n+a_{n−1}\\cdot b^{n−1}+···+a_1 \\cdot b+a_0\\), where \\(a_i\\) is an integer with \\(0≤a_i <b \\) for \\(0≤ i ≤ n \\), and \\(a_n \\ne 0\\).

\n

The representation of a positive integer \\(a\\) as a sum of multiples of powers of \\(b\\), as given above, is called the base \\(b\\) or \\(radix b\\) representation of \\(a\\), which is usually written as
\\(a = (a_n a_{n−1} ···a_1 a_0)_b\\)

\n

If \\( (a_n a_{n−1} ···a_1 a_0)_b\\) is the base b representation of \\(a\\) and \\(a_n \\ne 0\\), then the precision
or length of a is n + 1. If n = 0, then a is called a single-precision integer; otherwise, a is a multiple-precision integer.

\n

The division algorithm for integers provides an efficient method for determining the base b representation of a non-negative integer
\"radix_b_repesentation\"

\n

if \\( (a_{n} a_{n−1} ···a_1 a_0)_b \\) is the base b representation of a and k is a positive integer, then

\n

\\( (u_{l} u_{l−1} ···u_1 u_0)_{b^k} \\) is the base \\( b^k \\) representation of a, where \\( l =\\lceil (n+1)/k \\rceil -1 \\),

\n

\\( u_i = \\sum_{j=0}^{k-1} a_{ik+j}b^j \\) for \\( 0 \\le i \\le l-1 \\), and
\\( u_l = \\sum_{j=0}^{n-lk} a_{lk+j}b^j\\)

\n

Representing negative numbers

complement representation.

\n

addition and subtraction

\"multiple_precision_addition\"
subraction is quite similar to addition. (omitted here)

\n

multiplication

Let x and y be integers expressed in radix b representation:
\\( x = (x_n x_{n−1} ··· x_1 x_0)_b \\) and

\n

\\( y = (y_t y_{t−1} ··· y_1 y_0)_b \\). The product x · y will have at most (n + t + 2) base b digits.

\n

modular reduction

Barrett

Mongtgomery

references

\n","site":{"data":{}},"excerpt":"","more":"\n\n\n\n

multiple-precision integer arithmetic

If \\(b ≥ 2\\) is an integer, then any positive integer \\(a\\) can be expressed uniquely as \\(a = a_n \\cdot b^n+a_{n−1}\\cdot b^{n−1}+···+a_1 \\cdot b+a_0\\), where \\(a_i\\) is an integer with \\(0≤a_i <b \\) for \\(0≤ i ≤ n \\), and \\(a_n \\ne 0\\).

\n

The representation of a positive integer \\(a\\) as a sum of multiples of powers of \\(b\\), as given above, is called the base \\(b\\) or \\(radix b\\) representation of \\(a\\), which is usually written as
\\(a = (a_n a_{n−1} ···a_1 a_0)_b\\)

\n

If \\( (a_n a_{n−1} ···a_1 a_0)_b\\) is the base b representation of \\(a\\) and \\(a_n \\ne 0\\), then the precision
or length of a is n + 1. If n = 0, then a is called a single-precision integer; otherwise, a is a multiple-precision integer.

\n

The division algorithm for integers provides an efficient method for determining the base b representation of a non-negative integer
\"radix_b_repesentation\"

\n

if \\( (a_{n} a_{n−1} ···a_1 a_0)_b \\) is the base b representation of a and k is a positive integer, then

\n

\\( (u_{l} u_{l−1} ···u_1 u_0)_{b^k} \\) is the base \\( b^k \\) representation of a, where \\( l =\\lceil (n+1)/k \\rceil -1 \\),

\n

\\( u_i = \\sum_{j=0}^{k-1} a_{ik+j}b^j \\) for \\( 0 \\le i \\le l-1 \\), and
\\( u_l = \\sum_{j=0}^{n-lk} a_{lk+j}b^j\\)

\n

Representing negative numbers

complement representation.

\n

addition and subtraction

\"multiple_precision_addition\"
subraction is quite similar to addition. (omitted here)

\n

multiplication

Let x and y be integers expressed in radix b representation:
\\( x = (x_n x_{n−1} ··· x_1 x_0)_b \\) and

\n

\\( y = (y_t y_{t−1} ··· y_1 y_0)_b \\). The product x · y will have at most (n + t + 2) base b digits.

\n

modular reduction

Barrett

Mongtgomery

references

\n"},{"title":"montgomery multiplication","date":"2023-12-07T11:10:13.000Z","_content":"\n\n\n\n## montgomery space\nMontgomery multiplication works by first transforming the multipliers into Montgomery space, where modular multiplication can be performed cheaply, and then transforming them back when their actual values are needed. Unlike general integer division methods, Montgomery multiplication is not efficient for performing just one modular reduction and only becomes worthwhile when there is a chain of modular operations. \n\nThe space is defined by the modulo `n` and a positive integer `r ≥ n` coprime to `n`. The algorithm involves modulo and division by `r`, so in practice, `r` is chosen to be `2^32` or `2^64`, so that these operations can be done with a right-shift and a bitwise `AND` respectively.\n\n\nDefinition. the representative \\\\(\\bar{x}\\\\) of a number \\\\(x\\\\) in the Montgomery space is defined as \n\\\\[ \\bar{x} = x \\cdot r \\mod n \\\\]\n\nComputing this transformation involves a multiplication and a modulo — an expensive operation that we wanted to optimize away in the first place — which is why we only use this method when the overhead of transforming numbers to and from the Montgomery space is worth it and not for general modular multiplication.\n\nInside the Montgomery space, addition, substraction, and checking for equality is performed as usual:\n\n\n\\\\[ x \\cdot r + y \\cdot r \\equiv (x+y) \\cdot r \\mod n \\\\]\n\nHowever, this is not the case for multiplication. Denoting multiplication in the Montgomery space as \\\\(∗\\\\) and the “normal” multiplication as \\\\(\\cdot\\\\), we expect the result to be:\n\\\\[ \\bar{x} * \\bar{y} = \\overline{x\\cdot y} = (x\\cdot y)\\cdot r \\mod n \\\\]\n\nBut the normal multiplication in the Montgomery space yields:\n\\\\[ \\bar{x} \\cdot \\bar{y}=(x\\cdot y)\\cdot r \\cdot r \\mod n \\\\]\n\nTherefore, the multiplication in the Montgomery space is defined as\n\\\\[ \\bar{x} * \\bar{y} = \\bar{x} \\cdot \\bar{y} \\cdot r^{-1} \\mod n \\\\]\n\nThis means that, after we normally multiply two numbers in the Montgomery space, we need to reduce the result by multiplying it by \\\\(r^{-1}\\\\) and taking the modulo — and there is an efficent way to do this particular operation.\n\n\n## Montgomery reduction\nassume that \\\\(r=2^{32}\\\\), the modulo \\\\(n\\\\) is 32-bit, and the number \\\\(x\\\\) we need to reduce is 64-bit(the product of two 32-bit numbers). The goal is to calculate \\\\(y=x\\cdot r^{-1} \\mod n\\\\).\n\nSince \\\\(r\\\\) is coprime with \\\\(n\\\\), we know that there are two numbers \\\\(r^{-1}\\\\) and \\\\(n'\\\\) in the \\\\([0, n)\\\\) range such that\n\\\\[ r \\cdot r^{-1} + n \\cdot n' = 1 \\\\]\nboth \\\\(r^{-1} and \\quad n'\\\\) can be computed using extended Euclidean algorithm **(\\\\(n' = n^{-1} \\mod r\\\\))**.\nUsing this identiy, we can express \\\\(r \\cdot r^{-1}\\\\) as \\\\(1-n\\cdot n'\\\\) and write \\\\(x \\cdot r^{-1}\\\\) as\n\n\\\\[\n \\displaylines{\n x\\cdot r^{-1} = \\frac{x \\cdot r \\cdot r^{-1}}{r} \\\\\\\\\n = \\frac{x \\cdot (1-n\\cdot n')}{r} \\\\\\\\\n = \\frac{x - x \\cdot n \\cdot n'}{r} \\\\\\\\\n = \\frac{x - x \\cdot n \\cdot n' + k\\cdot r \\cdot n}{r} \\quad (\\mod n) (for \\hspace{0.2cm}any\\hspace{0.2cm} integer\\hspace{0.2cm} k) \\\\\\\\\n = \\frac{x - (x\\cdot n' - k\\cdot r)\\cdot n}{r}\n}\n \\\\]\n\nNow, if we choose \\\\(k\\\\) to be \\\\(\\lfloor x\\cdot n'/r\\rfloor\\\\) (the upper 64 bits of \\\\(x\\cdot n'\\\\), giving \\\\(x\\\\) is 64 bits and \\\\(n'\\\\) is 32 bits ), it will cancel out, and \\\\(k\\cdot r - x\\cdot n'\\\\) will simply be equal to \\\\(x \\cdot n' \\mod r\\\\) (the lower 32 bits of \\\\(x\\cdot n'\\\\)), implying:\n\\\\[ x\\cdot r^{-1} = (x - (x\\cdot n' \\mod r )\\cdot n)/r\\\\]\n\nthe algorithm itself just evaluates this formula, performing two multiplications to calculate \\\\(q = x\\cdot n' \\mod r\\\\) and \\\\(m = q\\cdot n\\\\), and then subtracts it from \\\\(x\\\\) and right shifts the result to divide it by r.\n\nThe only remaining thing to handle is that the result may not be in the \\\\([0,n)\\\\) range; but since\n\\\\[x < n\\cdot n < r\\cdot n => x/r < n \\\\]\nand\n\\\\[m = q\\cdot n < r\\cdot n => m/r < n\\\\]\nit is guaranted that \n\\\\[ -n < (x-m)/r < n \\\\]\n\nTherefore, we can simply check if the result is negative and in that case, add \\\\(n\\\\) to it, giving the following algorithm:\n\n```cpp\ntypedef __uint32_t u32;\ntypedef __uint64_t u64;\n\nconst u32 n = 1e9 + 7, nr = inverse(n, 1ull << 32); // nr is n' in the above equations, r is 1<<32\n\nu32 reduce(u64 x) {\n u32 q = u32(x) * nr; // q = x * n' mod r\n u64 m = (u64) q * n; // m = q * n\n u32 y = (x - m) >> 32; // y = (x - m) / r\n return x < m ? y + n : y; // if y < 0, add n to make it be in the [0, n) range\n}\n```\nThis last check is relatively cheap, but it is still on the critical path. If we are fine with the result being in the \\\\([0,2\\cdot n−2]\\\\) range instead of \\\\([0, n)\\\\), we can remove it and add \\\\(n\\\\) to the result unconditionally:\n\n```cpp\nu32 reduce(u64 x) {\n u32 q = u32(x) * nr;\n u64 m = (u64) q * n;\n u32 y = (x - m) >> 32;\n return y + n\n}\n```\n\nWe can also move the `>> 32` operation one step earlier in the computation graph and compute \\\\(\\lfloor x/r \\rfloor - \\lfloor m/r \\rfloor\\\\) instead of \\\\( (x-m)/r\\\\). This is correct because the lower 32 bits of \\\\(x\\\\) and \\\\(m\\\\) are equal anyway since \n\\\\[ m = x\\cdot n' \\cdot n = x \\mod r\\\\] (mod r is same as taking the lower 32 bits).\n\nBut why would we voluntarily choose to perfom two right-shifts instead of just one? This is beneficial because for `((u64) q * n) >> 32` we need to do a 32-by-32 multiplication and take the upper 32 bits of the result (which the x86 mul instruction already writes in a separate register, so it doesn’t cost anything), and the other right-shift `x >> 32` is not on the critical path.\n```cpp\nu32 reduce(u64 x) {\n u32 q = u32(x) * nr;\n u32 m = ((u64) q * n) >> 32;\n return (x >> 32) + n - m;\n}\n```\nif to reduce a u128 number（u64 * u64). it is as below\n```cpp\ntypedef __uint128_t u128;\n\nu64 reduce(u128 x) const {\n u64 q = u64(x) * nr;\n u64 m = ((u128) q * n) >> 64;\n return (x >> 64) + n - m;\n}\n```\n\n# Faster Inverse\nMontgomery multiplication itself is fast, but it requires some precomputation:\n- inverting \\\\(n\\\\) modulo \\\\(r\\\\) to compute \\\\(n'\\\\)\n- transforming a number to the Montgomery space,\n- transforming a number from the Montgomery space.\n\nThe last operation is already efficiently performed with the reduce procedure we just implemented, but the first `inverting` can be slightly optimized.\nComputing the inverse \\\\(n' = n^{-1} \\mod r\\\\) can be done faster than with the extended Euclidean algorith by taking advantage of the fact that \\\\(r\\\\) is a power of two and using the following identity:\n\\\\[ a\\cdot x = 1 \\mod 2^k => a\\cdot x\\cdot (2-a\\cdot x) = 1 \\mod 2^{2k} \\\\]\nProof: \n \\\\[\n \\displaylines{\n a\\cdot x \\cdot (2-a\\cdot x) = 2\\cdot a\\cdot x - (a\\cdot x)^2 \\\\\\\\\n = 2\\cdot(1+m\\cdot 2^k) - (1+m\\cdot 2^k)^2 \\\\\\\\\n = 2 + 2\\cdot m\\cdot 2^k - 1-2\\cdot m \\cdot 2^k -m^2\\cdot 2^{2k} \\\\\\\\\n = 1-m^2\\cdot 2^{2k} \\\\\\\\\n = 1 \\mod 2^{2k}\n}\n \\\\]\n**note**\n- \\\\(a\\cdot x \\\\) coudl be represented by \\\\(1+m\\cdot 2^k\\\\) as \\\\(a\\cdot x = 1 \\mod 2^k\\\\)\n\n\nas for our case, \\\\(x\\\\) is \\\\(nr\\\\), \\\\(a\\\\) is \\\\(n\\\\). i.e \\\\(n\\cdot nr = 1 \\mod 2^k\\\\). Since \\\\(n \\\\) is an odd number, we can start with \\\\(nr=1\\\\) as the inverse of \\\\(n\\\\) modulo \\\\(2^1\\\\). then applying this identity one time, i.e \\\\(n\\cdot 1 \\cdot (2-n\\cdot 1) = 1 \\mod 2^2\\\\). Let \\\\(nr = 1\\cdot(2-n\\cdot 1) \\\\), we apply this identity again, and get \\\\(n\\cdot nr \\cdot (2-a\\cdot nr) = 1 \\mod 2^4\\\\). Following this approach, and apply this identity exactly \\\\(log_2^m\\\\) times, where \\\\(m\\\\) is the bit length of \\\\(r\\\\)\n\n## Complement Implementation\n```cpp\n\n\n```\n## references\n- https://en.algorithmica.org/hpc/number-theory/montgomery/\n- [csdn blog on montgomery reduction](https://blog.csdn.net/mutourend/article/details/95613967)\n- [nayuki montgomery reduction](https://www.nayuki.io/page/montgomery-reduction-algorithm)\n","source":"_posts/arithmatic/montgomery-multiplication.md","raw":"---\ntitle: montgomery multiplication\ndate: 2023-12-07 19:10:13\ntags: [number_theory]\n---\n\n\n\n\n## montgomery space\nMontgomery multiplication works by first transforming the multipliers into Montgomery space, where modular multiplication can be performed cheaply, and then transforming them back when their actual values are needed. Unlike general integer division methods, Montgomery multiplication is not efficient for performing just one modular reduction and only becomes worthwhile when there is a chain of modular operations. \n\nThe space is defined by the modulo `n` and a positive integer `r ≥ n` coprime to `n`. The algorithm involves modulo and division by `r`, so in practice, `r` is chosen to be `2^32` or `2^64`, so that these operations can be done with a right-shift and a bitwise `AND` respectively.\n\n\nDefinition. the representative \\\\(\\bar{x}\\\\) of a number \\\\(x\\\\) in the Montgomery space is defined as \n\\\\[ \\bar{x} = x \\cdot r \\mod n \\\\]\n\nComputing this transformation involves a multiplication and a modulo — an expensive operation that we wanted to optimize away in the first place — which is why we only use this method when the overhead of transforming numbers to and from the Montgomery space is worth it and not for general modular multiplication.\n\nInside the Montgomery space, addition, substraction, and checking for equality is performed as usual:\n\n\n\\\\[ x \\cdot r + y \\cdot r \\equiv (x+y) \\cdot r \\mod n \\\\]\n\nHowever, this is not the case for multiplication. Denoting multiplication in the Montgomery space as \\\\(∗\\\\) and the “normal” multiplication as \\\\(\\cdot\\\\), we expect the result to be:\n\\\\[ \\bar{x} * \\bar{y} = \\overline{x\\cdot y} = (x\\cdot y)\\cdot r \\mod n \\\\]\n\nBut the normal multiplication in the Montgomery space yields:\n\\\\[ \\bar{x} \\cdot \\bar{y}=(x\\cdot y)\\cdot r \\cdot r \\mod n \\\\]\n\nTherefore, the multiplication in the Montgomery space is defined as\n\\\\[ \\bar{x} * \\bar{y} = \\bar{x} \\cdot \\bar{y} \\cdot r^{-1} \\mod n \\\\]\n\nThis means that, after we normally multiply two numbers in the Montgomery space, we need to reduce the result by multiplying it by \\\\(r^{-1}\\\\) and taking the modulo — and there is an efficent way to do this particular operation.\n\n\n## Montgomery reduction\nassume that \\\\(r=2^{32}\\\\), the modulo \\\\(n\\\\) is 32-bit, and the number \\\\(x\\\\) we need to reduce is 64-bit(the product of two 32-bit numbers). The goal is to calculate \\\\(y=x\\cdot r^{-1} \\mod n\\\\).\n\nSince \\\\(r\\\\) is coprime with \\\\(n\\\\), we know that there are two numbers \\\\(r^{-1}\\\\) and \\\\(n'\\\\) in the \\\\([0, n)\\\\) range such that\n\\\\[ r \\cdot r^{-1} + n \\cdot n' = 1 \\\\]\nboth \\\\(r^{-1} and \\quad n'\\\\) can be computed using extended Euclidean algorithm **(\\\\(n' = n^{-1} \\mod r\\\\))**.\nUsing this identiy, we can express \\\\(r \\cdot r^{-1}\\\\) as \\\\(1-n\\cdot n'\\\\) and write \\\\(x \\cdot r^{-1}\\\\) as\n\n\\\\[\n \\displaylines{\n x\\cdot r^{-1} = \\frac{x \\cdot r \\cdot r^{-1}}{r} \\\\\\\\\n = \\frac{x \\cdot (1-n\\cdot n')}{r} \\\\\\\\\n = \\frac{x - x \\cdot n \\cdot n'}{r} \\\\\\\\\n = \\frac{x - x \\cdot n \\cdot n' + k\\cdot r \\cdot n}{r} \\quad (\\mod n) (for \\hspace{0.2cm}any\\hspace{0.2cm} integer\\hspace{0.2cm} k) \\\\\\\\\n = \\frac{x - (x\\cdot n' - k\\cdot r)\\cdot n}{r}\n}\n \\\\]\n\nNow, if we choose \\\\(k\\\\) to be \\\\(\\lfloor x\\cdot n'/r\\rfloor\\\\) (the upper 64 bits of \\\\(x\\cdot n'\\\\), giving \\\\(x\\\\) is 64 bits and \\\\(n'\\\\) is 32 bits ), it will cancel out, and \\\\(k\\cdot r - x\\cdot n'\\\\) will simply be equal to \\\\(x \\cdot n' \\mod r\\\\) (the lower 32 bits of \\\\(x\\cdot n'\\\\)), implying:\n\\\\[ x\\cdot r^{-1} = (x - (x\\cdot n' \\mod r )\\cdot n)/r\\\\]\n\nthe algorithm itself just evaluates this formula, performing two multiplications to calculate \\\\(q = x\\cdot n' \\mod r\\\\) and \\\\(m = q\\cdot n\\\\), and then subtracts it from \\\\(x\\\\) and right shifts the result to divide it by r.\n\nThe only remaining thing to handle is that the result may not be in the \\\\([0,n)\\\\) range; but since\n\\\\[x < n\\cdot n < r\\cdot n => x/r < n \\\\]\nand\n\\\\[m = q\\cdot n < r\\cdot n => m/r < n\\\\]\nit is guaranted that \n\\\\[ -n < (x-m)/r < n \\\\]\n\nTherefore, we can simply check if the result is negative and in that case, add \\\\(n\\\\) to it, giving the following algorithm:\n\n```cpp\ntypedef __uint32_t u32;\ntypedef __uint64_t u64;\n\nconst u32 n = 1e9 + 7, nr = inverse(n, 1ull << 32); // nr is n' in the above equations, r is 1<<32\n\nu32 reduce(u64 x) {\n u32 q = u32(x) * nr; // q = x * n' mod r\n u64 m = (u64) q * n; // m = q * n\n u32 y = (x - m) >> 32; // y = (x - m) / r\n return x < m ? y + n : y; // if y < 0, add n to make it be in the [0, n) range\n}\n```\nThis last check is relatively cheap, but it is still on the critical path. If we are fine with the result being in the \\\\([0,2\\cdot n−2]\\\\) range instead of \\\\([0, n)\\\\), we can remove it and add \\\\(n\\\\) to the result unconditionally:\n\n```cpp\nu32 reduce(u64 x) {\n u32 q = u32(x) * nr;\n u64 m = (u64) q * n;\n u32 y = (x - m) >> 32;\n return y + n\n}\n```\n\nWe can also move the `>> 32` operation one step earlier in the computation graph and compute \\\\(\\lfloor x/r \\rfloor - \\lfloor m/r \\rfloor\\\\) instead of \\\\( (x-m)/r\\\\). This is correct because the lower 32 bits of \\\\(x\\\\) and \\\\(m\\\\) are equal anyway since \n\\\\[ m = x\\cdot n' \\cdot n = x \\mod r\\\\] (mod r is same as taking the lower 32 bits).\n\nBut why would we voluntarily choose to perfom two right-shifts instead of just one? This is beneficial because for `((u64) q * n) >> 32` we need to do a 32-by-32 multiplication and take the upper 32 bits of the result (which the x86 mul instruction already writes in a separate register, so it doesn’t cost anything), and the other right-shift `x >> 32` is not on the critical path.\n```cpp\nu32 reduce(u64 x) {\n u32 q = u32(x) * nr;\n u32 m = ((u64) q * n) >> 32;\n return (x >> 32) + n - m;\n}\n```\nif to reduce a u128 number（u64 * u64). it is as below\n```cpp\ntypedef __uint128_t u128;\n\nu64 reduce(u128 x) const {\n u64 q = u64(x) * nr;\n u64 m = ((u128) q * n) >> 64;\n return (x >> 64) + n - m;\n}\n```\n\n# Faster Inverse\nMontgomery multiplication itself is fast, but it requires some precomputation:\n- inverting \\\\(n\\\\) modulo \\\\(r\\\\) to compute \\\\(n'\\\\)\n- transforming a number to the Montgomery space,\n- transforming a number from the Montgomery space.\n\nThe last operation is already efficiently performed with the reduce procedure we just implemented, but the first `inverting` can be slightly optimized.\nComputing the inverse \\\\(n' = n^{-1} \\mod r\\\\) can be done faster than with the extended Euclidean algorith by taking advantage of the fact that \\\\(r\\\\) is a power of two and using the following identity:\n\\\\[ a\\cdot x = 1 \\mod 2^k => a\\cdot x\\cdot (2-a\\cdot x) = 1 \\mod 2^{2k} \\\\]\nProof: \n \\\\[\n \\displaylines{\n a\\cdot x \\cdot (2-a\\cdot x) = 2\\cdot a\\cdot x - (a\\cdot x)^2 \\\\\\\\\n = 2\\cdot(1+m\\cdot 2^k) - (1+m\\cdot 2^k)^2 \\\\\\\\\n = 2 + 2\\cdot m\\cdot 2^k - 1-2\\cdot m \\cdot 2^k -m^2\\cdot 2^{2k} \\\\\\\\\n = 1-m^2\\cdot 2^{2k} \\\\\\\\\n = 1 \\mod 2^{2k}\n}\n \\\\]\n**note**\n- \\\\(a\\cdot x \\\\) coudl be represented by \\\\(1+m\\cdot 2^k\\\\) as \\\\(a\\cdot x = 1 \\mod 2^k\\\\)\n\n\nas for our case, \\\\(x\\\\) is \\\\(nr\\\\), \\\\(a\\\\) is \\\\(n\\\\). i.e \\\\(n\\cdot nr = 1 \\mod 2^k\\\\). Since \\\\(n \\\\) is an odd number, we can start with \\\\(nr=1\\\\) as the inverse of \\\\(n\\\\) modulo \\\\(2^1\\\\). then applying this identity one time, i.e \\\\(n\\cdot 1 \\cdot (2-n\\cdot 1) = 1 \\mod 2^2\\\\). Let \\\\(nr = 1\\cdot(2-n\\cdot 1) \\\\), we apply this identity again, and get \\\\(n\\cdot nr \\cdot (2-a\\cdot nr) = 1 \\mod 2^4\\\\). Following this approach, and apply this identity exactly \\\\(log_2^m\\\\) times, where \\\\(m\\\\) is the bit length of \\\\(r\\\\)\n\n## Complement Implementation\n```cpp\n\n\n```\n## references\n- https://en.algorithmica.org/hpc/number-theory/montgomery/\n- [csdn blog on montgomery reduction](https://blog.csdn.net/mutourend/article/details/95613967)\n- [nayuki montgomery reduction](https://www.nayuki.io/page/montgomery-reduction-algorithm)\n","slug":"arithmatic/montgomery-multiplication","published":1,"updated":"2023-12-08T02:45:44.046Z","_id":"clphpvdof0000aj7u9gaz5qfy","comments":1,"layout":"post","photos":[],"link":"","content":"\n\n\n

montgomery space

Montgomery multiplication works by first transforming the multipliers into Montgomery space, where modular multiplication can be performed cheaply, and then transforming them back when their actual values are needed. Unlike general integer division methods, Montgomery multiplication is not efficient for performing just one modular reduction and only becomes worthwhile when there is a chain of modular operations.

\n

The space is defined by the modulo n and a positive integer r ≥ n coprime to n. The algorithm involves modulo and division by r, so in practice, r is chosen to be 2^32 or 2^64, so that these operations can be done with a right-shift and a bitwise AND respectively.

\n

Definition. the representative \\(\\bar{x}\\) of a number \\(x\\) in the Montgomery space is defined as
\\[ \\bar{x} = x \\cdot r \\mod n \\]

\n

Computing this transformation involves a multiplication and a modulo — an expensive operation that we wanted to optimize away in the first place — which is why we only use this method when the overhead of transforming numbers to and from the Montgomery space is worth it and not for general modular multiplication.

\n

Inside the Montgomery space, addition, substraction, and checking for equality is performed as usual:

\n

\\[ x \\cdot r + y \\cdot r \\equiv (x+y) \\cdot r \\mod n \\]

\n

However, this is not the case for multiplication. Denoting multiplication in the Montgomery space as \\(∗\\) and the “normal” multiplication as \\(\\cdot\\), we expect the result to be:
\\[ \\bar{x} * \\bar{y} = \\overline{x\\cdot y} = (x\\cdot y)\\cdot r \\mod n \\]

\n

But the normal multiplication in the Montgomery space yields:
\\[ \\bar{x} \\cdot \\bar{y}=(x\\cdot y)\\cdot r \\cdot r \\mod n \\]

\n

Therefore, the multiplication in the Montgomery space is defined as
\\[ \\bar{x} * \\bar{y} = \\bar{x} \\cdot \\bar{y} \\cdot r^{-1} \\mod n \\]

\n

This means that, after we normally multiply two numbers in the Montgomery space, we need to reduce the result by multiplying it by \\(r^{-1}\\) and taking the modulo — and there is an efficent way to do this particular operation.

\n

Montgomery reduction

assume that \\(r=2^{32}\\), the modulo \\(n\\) is 32-bit, and the number \\(x\\) we need to reduce is 64-bit(the product of two 32-bit numbers). The goal is to calculate \\(y=x\\cdot r^{-1} \\mod n\\).

\n

Since \\(r\\) is coprime with \\(n\\), we know that there are two numbers \\(r^{-1}\\) and \\(n’\\) in the \\([0, n)\\) range such that
\\[ r \\cdot r^{-1} + n \\cdot n’ = 1 \\]
both \\(r^{-1} and \\quad n’\\) can be computed using extended Euclidean algorithm (\\(n’ = n^{-1} \\mod r\\)).
Using this identiy, we can express \\(r \\cdot r^{-1}\\) as \\(1-n\\cdot n’\\) and write \\(x \\cdot r^{-1}\\) as

\n

\\[
\\displaylines{
x\\cdot r^{-1} = \\frac{x \\cdot r \\cdot r^{-1}}{r} \\\\
= \\frac{x \\cdot (1-n\\cdot n’)}{r} \\\\
= \\frac{x - x \\cdot n \\cdot n’}{r} \\\\
= \\frac{x - x \\cdot n \\cdot n’ + k\\cdot r \\cdot n}{r} \\quad (\\mod n) (for \\hspace{0.2cm}any\\hspace{0.2cm} integer\\hspace{0.2cm} k) \\\\
= \\frac{x - (x\\cdot n’ - k\\cdot r)\\cdot n}{r}
}
\\]

\n

Now, if we choose \\(k\\) to be \\(\\lfloor x\\cdot n’/r\\rfloor\\) (the upper 64 bits of \\(x\\cdot n’\\), giving \\(x\\) is 64 bits and \\(n’\\) is 32 bits ), it will cancel out, and \\(k\\cdot r - x\\cdot n’\\) will simply be equal to \\(x \\cdot n’ \\mod r\\) (the lower 32 bits of \\(x\\cdot n’\\)), implying:
\\[ x\\cdot r^{-1} = (x - (x\\cdot n’ \\mod r )\\cdot n)/r\\]

\n

the algorithm itself just evaluates this formula, performing two multiplications to calculate \\(q = x\\cdot n’ \\mod r\\) and \\(m = q\\cdot n\\), and then subtracts it from \\(x\\) and right shifts the result to divide it by r.

\n

The only remaining thing to handle is that the result may not be in the \\([0,n)\\) range; but since
\\[x < n\\cdot n < r\\cdot n => x/r < n \\]
and
\\[m = q\\cdot n < r\\cdot n => m/r < n\\]
it is guaranted that
\\[ -n < (x-m)/r < n \\]

\n

Therefore, we can simply check if the result is negative and in that case, add \\(n\\) to it, giving the following algorithm:

\n
1
2
3
4
5
6
7
8
9
10
11
typedef __uint32_t u32;
typedef __uint64_t u64;

const u32 n = 1e9 + 7, nr = inverse(n, 1ull << 32); // nr is n' in the above equations, r is 1<<32

u32 reduce(u64 x) {
    u32 q = u32(x) * nr;      // q = x * n' mod r
    u64 m = (u64) q * n;      // m = q * n
    u32 y = (x - m) >> 32;    // y = (x - m) / r
    return x < m ? y + n : y; // if y < 0, add n to make it be in the [0, n) range
}
\n

This last check is relatively cheap, but it is still on the critical path. If we are fine with the result being in the \\([0,2\\cdot n−2]\\) range instead of \\([0, n)\\), we can remove it and add \\(n\\) to the result unconditionally:

\n
1
2
3
4
5
6
u32 reduce(u64 x) {
    u32 q = u32(x) * nr;
    u64 m = (u64) q * n;
    u32 y = (x - m) >> 32;
    return y + n
}
\n\n

We can also move the >> 32 operation one step earlier in the computation graph and compute \\(\\lfloor x/r \\rfloor - \\lfloor m/r \\rfloor\\) instead of \\( (x-m)/r\\). This is correct because the lower 32 bits of \\(x\\) and \\(m\\) are equal anyway since
\\[ m = x\\cdot n’ \\cdot n = x \\mod r\\] (mod r is same as taking the lower 32 bits).

\n

But why would we voluntarily choose to perfom two right-shifts instead of just one? This is beneficial because for ((u64) q * n) >> 32 we need to do a 32-by-32 multiplication and take the upper 32 bits of the result (which the x86 mul instruction already writes in a separate register, so it doesn’t cost anything), and the other right-shift x >> 32 is not on the critical path.

\n
1
2
3
4
5
u32 reduce(u64 x) {
    u32 q = u32(x) * nr;
    u32 m = ((u64) q * n) >> 32;
    return (x >> 32) + n - m;
}
\n

if to reduce a u128 number（u64 * u64). it is as below

\n
1
2
3
4
5
6
7
typedef __uint128_t u128;

u64 reduce(u128 x) const {
    u64 q = u64(x) * nr;
    u64 m = ((u128) q * n) >> 64;
    return (x >> 64) + n - m;
}
\n\n

Faster Inverse

Montgomery multiplication itself is fast, but it requires some precomputation:

\n
    \n
  • inverting \\(n\\) modulo \\(r\\) to compute \\(n’\\)
    • \n
  • transforming a number to the Montgomery space,
    • \n
  • transforming a number from the Montgomery space.
    • \n
\n

The last operation is already efficiently performed with the reduce procedure we just implemented, but the first inverting can be slightly optimized.
Computing the inverse \\(n’ = n^{-1} \\mod r\\) can be done faster than with the extended Euclidean algorith by taking advantage of the fact that \\(r\\) is a power of two and using the following identity:
\\[ a\\cdot x = 1 \\mod 2^k => a\\cdot x\\cdot (2-a\\cdot x) = 1 \\mod 2^{2k} \\]
Proof:
\\[
\\displaylines{
a\\cdot x \\cdot (2-a\\cdot x) = 2\\cdot a\\cdot x - (a\\cdot x)^2 \\\\
= 2\\cdot(1+m\\cdot 2^k) - (1+m\\cdot 2^k)^2 \\\\
= 2 + 2\\cdot m\\cdot 2^k - 1-2\\cdot m \\cdot 2^k -m^2\\cdot 2^{2k} \\\\
= 1-m^2\\cdot 2^{2k} \\\\
= 1 \\mod 2^{2k}
}
\\]
note

\n
    \n
  • \\(a\\cdot x \\) coudl be represented by \\(1+m\\cdot 2^k\\) as \\(a\\cdot x = 1 \\mod 2^k\\)
    • \n
\n

as for our case, \\(x\\) is \\(nr\\), \\(a\\) is \\(n\\). i.e \\(n\\cdot nr = 1 \\mod 2^k\\). Since \\(n \\) is an odd number, we can start with \\(nr=1\\) as the inverse of \\(n\\) modulo \\(2^1\\). then applying this identity one time, i.e \\(n\\cdot 1 \\cdot (2-n\\cdot 1) = 1 \\mod 2^2\\). Let \\(nr = 1\\cdot(2-n\\cdot 1) \\), we apply this identity again, and get \\(n\\cdot nr \\cdot (2-a\\cdot nr) = 1 \\mod 2^4\\). Following this approach, and apply this identity exactly \\(log_2^m\\) times, where \\(m\\) is the bit length of \\(r\\)

\n

Complement Implementation

1
2


\n

references

\n","site":{"data":{}},"excerpt":"","more":"\n\n\n

montgomery space

Montgomery multiplication works by first transforming the multipliers into Montgomery space, where modular multiplication can be performed cheaply, and then transforming them back when their actual values are needed. Unlike general integer division methods, Montgomery multiplication is not efficient for performing just one modular reduction and only becomes worthwhile when there is a chain of modular operations.

\n

The space is defined by the modulo n and a positive integer r ≥ n coprime to n. The algorithm involves modulo and division by r, so in practice, r is chosen to be 2^32 or 2^64, so that these operations can be done with a right-shift and a bitwise AND respectively.

\n

Definition. the representative \\(\\bar{x}\\) of a number \\(x\\) in the Montgomery space is defined as
\\[ \\bar{x} = x \\cdot r \\mod n \\]

\n

Computing this transformation involves a multiplication and a modulo — an expensive operation that we wanted to optimize away in the first place — which is why we only use this method when the overhead of transforming numbers to and from the Montgomery space is worth it and not for general modular multiplication.

\n

Inside the Montgomery space, addition, substraction, and checking for equality is performed as usual:

\n

\\[ x \\cdot r + y \\cdot r \\equiv (x+y) \\cdot r \\mod n \\]

\n

However, this is not the case for multiplication. Denoting multiplication in the Montgomery space as \\(∗\\) and the “normal” multiplication as \\(\\cdot\\), we expect the result to be:
\\[ \\bar{x} * \\bar{y} = \\overline{x\\cdot y} = (x\\cdot y)\\cdot r \\mod n \\]

\n

But the normal multiplication in the Montgomery space yields:
\\[ \\bar{x} \\cdot \\bar{y}=(x\\cdot y)\\cdot r \\cdot r \\mod n \\]

\n

Therefore, the multiplication in the Montgomery space is defined as
\\[ \\bar{x} * \\bar{y} = \\bar{x} \\cdot \\bar{y} \\cdot r^{-1} \\mod n \\]

\n

This means that, after we normally multiply two numbers in the Montgomery space, we need to reduce the result by multiplying it by \\(r^{-1}\\) and taking the modulo — and there is an efficent way to do this particular operation.

\n

Montgomery reduction

assume that \\(r=2^{32}\\), the modulo \\(n\\) is 32-bit, and the number \\(x\\) we need to reduce is 64-bit(the product of two 32-bit numbers). The goal is to calculate \\(y=x\\cdot r^{-1} \\mod n\\).

\n

Since \\(r\\) is coprime with \\(n\\), we know that there are two numbers \\(r^{-1}\\) and \\(n’\\) in the \\([0, n)\\) range such that
\\[ r \\cdot r^{-1} + n \\cdot n’ = 1 \\]
both \\(r^{-1} and \\quad n’\\) can be computed using extended Euclidean algorithm (\\(n’ = n^{-1} \\mod r\\)).
Using this identiy, we can express \\(r \\cdot r^{-1}\\) as \\(1-n\\cdot n’\\) and write \\(x \\cdot r^{-1}\\) as

\n

\\[
\\displaylines{
x\\cdot r^{-1} = \\frac{x \\cdot r \\cdot r^{-1}}{r} \\\\
= \\frac{x \\cdot (1-n\\cdot n’)}{r} \\\\
= \\frac{x - x \\cdot n \\cdot n’}{r} \\\\
= \\frac{x - x \\cdot n \\cdot n’ + k\\cdot r \\cdot n}{r} \\quad (\\mod n) (for \\hspace{0.2cm}any\\hspace{0.2cm} integer\\hspace{0.2cm} k) \\\\
= \\frac{x - (x\\cdot n’ - k\\cdot r)\\cdot n}{r}
}
\\]

\n

Now, if we choose \\(k\\) to be \\(\\lfloor x\\cdot n’/r\\rfloor\\) (the upper 64 bits of \\(x\\cdot n’\\), giving \\(x\\) is 64 bits and \\(n’\\) is 32 bits ), it will cancel out, and \\(k\\cdot r - x\\cdot n’\\) will simply be equal to \\(x \\cdot n’ \\mod r\\) (the lower 32 bits of \\(x\\cdot n’\\)), implying:
\\[ x\\cdot r^{-1} = (x - (x\\cdot n’ \\mod r )\\cdot n)/r\\]

\n

the algorithm itself just evaluates this formula, performing two multiplications to calculate \\(q = x\\cdot n’ \\mod r\\) and \\(m = q\\cdot n\\), and then subtracts it from \\(x\\) and right shifts the result to divide it by r.

\n

The only remaining thing to handle is that the result may not be in the \\([0,n)\\) range; but since
\\[x < n\\cdot n < r\\cdot n => x/r < n \\]
and
\\[m = q\\cdot n < r\\cdot n => m/r < n\\]
it is guaranted that
\\[ -n < (x-m)/r < n \\]

\n

Therefore, we can simply check if the result is negative and in that case, add \\(n\\) to it, giving the following algorithm:

\n
1
2
3
4
5
6
7
8
9
10
11
typedef __uint32_t u32;
typedef __uint64_t u64;

const u32 n = 1e9 + 7, nr = inverse(n, 1ull << 32); // nr is n' in the above equations, r is 1<<32

u32 reduce(u64 x) {
    u32 q = u32(x) * nr;      // q = x * n' mod r
    u64 m = (u64) q * n;      // m = q * n
    u32 y = (x - m) >> 32;    // y = (x - m) / r
    return x < m ? y + n : y; // if y < 0, add n to make it be in the [0, n) range
}
\n

This last check is relatively cheap, but it is still on the critical path. If we are fine with the result being in the \\([0,2\\cdot n−2]\\) range instead of \\([0, n)\\), we can remove it and add \\(n\\) to the result unconditionally:

\n
1
2
3
4
5
6
u32 reduce(u64 x) {
    u32 q = u32(x) * nr;
    u64 m = (u64) q * n;
    u32 y = (x - m) >> 32;
    return y + n
}
\n\n

We can also move the >> 32 operation one step earlier in the computation graph and compute \\(\\lfloor x/r \\rfloor - \\lfloor m/r \\rfloor\\) instead of \\( (x-m)/r\\). This is correct because the lower 32 bits of \\(x\\) and \\(m\\) are equal anyway since
\\[ m = x\\cdot n’ \\cdot n = x \\mod r\\] (mod r is same as taking the lower 32 bits).

\n

But why would we voluntarily choose to perfom two right-shifts instead of just one? This is beneficial because for ((u64) q * n) >> 32 we need to do a 32-by-32 multiplication and take the upper 32 bits of the result (which the x86 mul instruction already writes in a separate register, so it doesn’t cost anything), and the other right-shift x >> 32 is not on the critical path.

\n
1
2
3
4
5
u32 reduce(u64 x) {
    u32 q = u32(x) * nr;
    u32 m = ((u64) q * n) >> 32;
    return (x >> 32) + n - m;
}
\n

if to reduce a u128 number（u64 * u64). it is as below

\n
1
2
3
4
5
6
7
typedef __uint128_t u128;

u64 reduce(u128 x) const {
    u64 q = u64(x) * nr;
    u64 m = ((u128) q * n) >> 64;
    return (x >> 64) + n - m;
}
\n\n

Faster Inverse

Montgomery multiplication itself is fast, but it requires some precomputation:

\n
    \n
  • inverting \\(n\\) modulo \\(r\\) to compute \\(n’\\)
    • \n
  • transforming a number to the Montgomery space,
    • \n
  • transforming a number from the Montgomery space.
    • \n
\n

The last operation is already efficiently performed with the reduce procedure we just implemented, but the first inverting can be slightly optimized.
Computing the inverse \\(n’ = n^{-1} \\mod r\\) can be done faster than with the extended Euclidean algorith by taking advantage of the fact that \\(r\\) is a power of two and using the following identity:
\\[ a\\cdot x = 1 \\mod 2^k => a\\cdot x\\cdot (2-a\\cdot x) = 1 \\mod 2^{2k} \\]
Proof:
\\[
\\displaylines{
a\\cdot x \\cdot (2-a\\cdot x) = 2\\cdot a\\cdot x - (a\\cdot x)^2 \\\\
= 2\\cdot(1+m\\cdot 2^k) - (1+m\\cdot 2^k)^2 \\\\
= 2 + 2\\cdot m\\cdot 2^k - 1-2\\cdot m \\cdot 2^k -m^2\\cdot 2^{2k} \\\\
= 1-m^2\\cdot 2^{2k} \\\\
= 1 \\mod 2^{2k}
}
\\]
note

\n
    \n
  • \\(a\\cdot x \\) coudl be represented by \\(1+m\\cdot 2^k\\) as \\(a\\cdot x = 1 \\mod 2^k\\)
    • \n
\n

as for our case, \\(x\\) is \\(nr\\), \\(a\\) is \\(n\\). i.e \\(n\\cdot nr = 1 \\mod 2^k\\). Since \\(n \\) is an odd number, we can start with \\(nr=1\\) as the inverse of \\(n\\) modulo \\(2^1\\). then applying this identity one time, i.e \\(n\\cdot 1 \\cdot (2-n\\cdot 1) = 1 \\mod 2^2\\). Let \\(nr = 1\\cdot(2-n\\cdot 1) \\), we apply this identity again, and get \\(n\\cdot nr \\cdot (2-a\\cdot nr) = 1 \\mod 2^4\\). Following this approach, and apply this identity exactly \\(log_2^m\\) times, where \\(m\\) is the bit length of \\(r\\)

\n

Complement Implementation

1
2


\n

references

\n"},{"title":"sgx multi prover","date":"2023-11-24T06:29:26.000Z","_content":"\n\n\n## references \n- https://blog.csdn.net/mutourend/article/details/133586549?spm=1001.2014.3001.5502","source":"_posts/cryptography/zkp/sgx_multi_prover.md","raw":"---\ntitle: sgx multi prover\ndate: 2023-11-24 14:29:26\ntags: [cryptography,zkp]\n---\n\n\n\n## references \n- https://blog.csdn.net/mutourend/article/details/133586549?spm=1001.2014.3001.5502","slug":"cryptography/zkp/sgx_multi_prover","published":1,"updated":"2023-11-24T14:05:15.647Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clphpvdoj0003aj7ugctear78","content":"\n\n\n

references

\n","site":{"data":{}},"excerpt":"","more":"\n\n\n

references

\n"},{"title":"stark 101","date":"2023-11-03T09:07:33.000Z","_content":"\n\n\n## trace and low degree extension\nthe objective is to develop a STARK prover for the FibonacciSq sequence over a finite field. The FibonacciSq sequence is defined by the recurrence relation\n\\\\[ a_{n+2} = a_{n+1} ^2 + a_n ^2 \\\\]\n\nthe statement is: **I know a FieldElement \\\\(X\\in \\mathbb{F}\\\\) such that the 1023rd element of the FibonacciSq sequence starting with \\\\(1, X\\\\) is \\\\(2338775057\\\\)**\n\nThe underlying field of this class is \\\\(\\mathbb{F}_{3221225473}\\\\) (\\\\(3221225473 = 3 \\cdot 2^{30} + 1\\\\)), so all operations are done modulo 3221225473.\n\n\n### FibonacciSq Trace\n let's construct a list `a` of length 1023, whose first two elements will be FieldElement objects representing 1 and 3141592, respectively. The next 1021 elements will be the FibonacciSq sequence induced by these two elements. `a` is called the **trace** of FibonacciSq, or, when the context is clear, the trace.\n\n### Thinking of Polynomials\nWe now want to think of the sequence as the evaluation of some polynomial \\\\(f\\\\) of degree 1022.\nWe will choose the domain to be some subgroup \\\\(G \\subseteq \\mathbb{F}^\\times\\\\) of size 1024, for reasons that will become clear later.\n\n(Recall that \\\\(\\mathbb{F}^\\times\\\\) denotes the multiplicative group of \\\\(\\mathbb{F}\\\\), which we get from \\\\(\\mathbb{F}\\\\) by omitting the zero element with the induced multiplication from the field. A subgroup of size 1024 exists because \\\\(\\mathbb{F}^\\times\\\\) is a cyclic group of size \\\\(3\\cdot 2^{30}\\\\), so it contains a subgroup of size \\\\(2^i\\\\) for any \\\\(0 \\leq i \\leq 30\\\\)).\n#### Find a Group of Size 1024\nIf we find an element \\\\(g \\in \\mathbb{F}\\\\) whose (multiplicative) order is 1024, then \\\\(g\\\\) will generate such a group. Create a list called `G` with all the elements of \\\\(G\\\\), such that \\\\(G[i] := g^i\\\\).\n\n\n### Evaluating on a Larger Domain\nthen, interpolating `G` over `a` we get a polynomial `f`. The trace, viewed as evaluations of a polynomial \\\\(f\\\\) on \\\\(G\\\\), can now be extended by evaluating \\\\(f\\\\) over a larger domain, thereby creating a Reed-Solomon error correction code.\n\n#### Cosets\nTo that end, we must decide on a larger domain on which \\\\(f\\\\) will be evaluated. We will work with a domain that is 8 times larger than \\\\(G\\\\).
A natural choice for such a domain is to take some group \\\\(H\\\\) of size 8192 (which exists because 8192 divides \\\\(|\\mathbb{F}^\\times|\\\\)), and shift it by the generator of \\\\(\\mathbb{F}^\\times\\\\), thereby obtaining a [coset](https://en.wikipedia.org/wiki/Coset) of \\\\(H\\\\).\n\nCreate a list called `H` of the elements of \\\\(H\\\\), and multiply each of them by the generator of \\\\(\\mathbb{F}^\\times\\\\) to obtain a list called `eval_domain`. In other words, `eval_domain` = \\\\(\\\\{w\\cdot h^i | 0 \\leq i <8192 \\\\}\\\\) for \\\\(h\\\\) the generator of \\\\(H\\\\) and \\\\(w\\\\) the generator of \\\\(\\mathbb{F}^\\times\\\\).\n\n\n#### Evaluate on a Coset\n```python\nf = interpolate_poly(G[:-1], a)\nf_eval = [f(d) for d in eval_domain]\n```\n\n### Commitments\nWe will use [Sha256](https://en.wikipedia.org/wiki/SHA-2)-based [Merkle Trees](https://en.wikipedia.org/wiki/Merkle_tree) as our commitment scheme.\n```python\nfrom merkle import MerkleTree\nf_merkle = MerkleTree(f_eval)\n```\n\n### Channel\nTheoretically, a STARK proof system is a protocol for interaction between two parties - a prover and a verifier. In practice, we convert this interactive protocol into a non-interactive proof using the [Fiat-Shamir Heuristic](https://en.wikipedia.org/wiki/Fiat%E2%80%93Shamir_heuristic). In this tutorial you will use the `Channel` class, which implements this transformation. This channel replaces the verifier in the sense that the prover (which you are writing) will send data, and receive random numbers or random `FieldElement` instances.\n\n\n\n\n## constraints\nIn this part, we are going to create a set of constraints over the trace `a`. \n### Step 1 - FibonacciSq Constraints\nFor `a` to be a correct trace of a FibonacciSq sequence that proves our claim:\n1. The first element has to be 1, namely \\\\(a[0] = 1\\\\).\n2. The last element has to be 2338775057, namely \\\\(a[1022] = 2338775057\\\\).\n3. The FibonacciSq rule must apply, that is - for every \\\\(i<1021\\\\), \\\\(a[i+2]=a[i+1]^2+a[i]^2\\\\).\n\n\n### Step 2 - Polynomial Constraints\nRecall that `f` is a polynomial over the trace domain, that evaluates exactly to `a` over \\\\(G \\setminus \\{g^{1023}\\}\\\\) where \\\\(G=\\{g^i : 0\\leq i\\leq 1023\\}\\\\) is the \"small\" group generated by \\\\(g\\\\).
\n\nWe now rewrite the above three constraints in a form of polynomial constraints over `f`:\n1. \\\\(a[0] = 1\\\\) is translated to the polynomial \\\\(f(x) - 1\\\\), which evalutes to 0 for \\\\(x = g^0\\\\) (note that \\\\(g^0\\\\) is \\\\(1\\\\)).
\n2. \\\\(a[1022] = 2338775057\\\\) is translated to the polynomial \\\\(f(x) - 2338775057\\\\), which evalutes to 0 for \\\\(x = g^{1022}\\\\).
\n3. \\\\(a[i+2]=a[i+1]^2+a[i]^2\\\\) for every \\\\(i<1021\\\\) is translated to the polynomial \\\\(f(g^2 \\cdot x) - (f(g \\cdot x))^2 - (f(x))^2\\\\), which evaluates to 0 for \\\\(x \\in G \\backslash \\{g^{1021}, g^{1022}, g^{1023}\\}\\\\).

\n\n### Step 3 - Rational Functions (That are in Fact Polynomials)\n\nEach of the constraints above is represented by a polynomial \\\\(u(x)\\\\) that supposedly evaluates to \\\\(0\\\\) on certain elements of the group \\\\(G\\\\). That is, for some \\\\(x_0, \\ldots, x_k \\in G\\\\), we claim that\n\n\\\\[u(x_0) = \\ldots = u(x_k) = 0\\\\]\n\n(note that for the first two constaints, \\\\(k=0\\\\) because they only refer to one point and for the third \\\\(k=1021\\\\)).\n\nThis is equivalent to saying that \\\\(u(x)\\\\) is divisible, as a polynomial, by all of \\\\(\\{(x-x_i)\\}_{i=0}^k\\\\), or, equivalently, by\n\n\\\\[\\prod_{i=0}^k (x-x_i)\\\\]\n\nTherefore, each of the three constraints above can be written as a rational function of the form:\n\n\\\\[\\frac{u(x)}{\\prod_{i=0}^k (x-x_i)}\\\\]\n\nfor the corresponding \\\\(u(x)\\\\) and \\\\(\\{x_i\\}_{i=0}^k\\\\). In this step we will construct these three rational functions and show that they are indeed polynomials.\n\n### The First Constraint:\n\nIn the first constraint, \\\\(f(x) - 1\\\\) and \\\\(\\{x_i\\} = \\{1\\}\\\\).\n\nWe will now construct the **polynomial** \\\\(p_0(x)=\\frac{f(x) - 1}{x - 1}\\\\), making sure that \\\\(f(x) - 1\\\\) is indeed divisible by \\\\((x-1)\\\\).\n\n### The Second Constraint\n\nConstruct the polynomial `p1` representing the second constraint, \\\\(p_1(x)= \\frac{f(x) - 2338775057}{x - g^{1022}}\\\\), similarly:
\n\n### The Third Constraint - Succinctness\n\nThe last constraint's rational function is slightly more complicated:
\n\n\n\\\\[p_2(x) = \\frac{f(g^2 \\cdot x) - (f(g \\cdot x))^2 - (f(x))^2}{\\prod\\limits_{i=0}^{1020} (x-g^i)}\\\\]\n\nwhose denominator can be rewritten, so that the entire expression is easier to compute:
\n\n$$\\frac{f(g^2 \\cdot x) - (f(g \\cdot x))^2 - (f(x))^2}{\\frac{x^{1024} - 1}{(x-g^{1021})(x-g^{1022})(x-g^{1023})}}$$
\n\nThis follows from the equality\n\n$$\\prod\\limits_{i=0}^{1023} (x-g^i) = x^{1024} - 1$$\n\n### Step 4 - Composition Polynomial\nRecall that we're translating a problem of checking the validity of three polynomial constraints to checking that each of the rational functions \\\\(p_0, p_1, p_2\\\\) are polynomials.
\n\nOur protocol uses an algorithm called [FRI](https://eccc.weizmann.ac.il/report/2017/134/) to do so, which will be discussed in the next part.
\nIn order for the proof to be succinct (short), we prefer to work with just one rational function instead of three. For that, we take a random linear combination of \\\\(p_0, p_1, p_2\\\\) called the **compostion polynomial** (CP for short):\n\n$$CP(x) = \\alpha_0 \\cdot p_0(x) + \\alpha_1 \\cdot p_1(x) + \\alpha_2 \\cdot p_2(x)$$
\n\nwhere \\\\(\\alpha_0, \\alpha_1, \\alpha_2 \\\\) are random field elements obtained from the verifier, or in our case - from the channel.\n\nProving that (the rational function) \\\\(CP\\\\) is a polynomial guarantess, with high probability, that each of \\\\(p_0, p_1, p_2\\\\) are themselves polynomials.\n\n### Commit on the Composition Polynomial\nLastly, we evaluate $cp$ over the evaluation domain (`eval_domain`), build a Merkle tree on top of that and send its root over the channel. This is similar to commiting on the LDE trace, as we did at the end of part 1.\n\n![trace to cp](images/zkp/stark/trace_to_CP.png)\n\n## FRI Commitments\n### FRI Folding\n\nOur goal in this part is to construct the FRI layers and commit on them. \n
To obtain each layer we need:\n1. To generate a domain for the layer (from the previous layer's domain).\n2. To generate a polynomial for the layer (from the previous layer's polynomial and domain).\n3. To evaluate said polynomial on said domain - **this is the next FRI layer**.\n\n#### Domain Generation\n\nThe first FRI domain is simply the `eval_domain` that you already generated in Part 1, namely a coset of a group of order 8192. Each subsequent FRI domain is obtained by taking the first half of the previous FRI domain (dropping the second half), and squaring each of its elements.
\n\nFormally - we got `eval_domain` by taking:
\n$$w, w\\cdot h, w\\cdot h^2, ..., w\\cdot h^{8191}$$\n\nThe next layer will therefore be:
\n$$w^2, (w\\cdot h)^2, (w\\cdot h^2)^2, ..., (w\\cdot h^{4095})^2$$\n\nNote that taking the squares of the second half of each elements in `eval_domain` yields exactly\nthe same result as taking the squares of the first half. This is true for the next layers as well.\n\nSimilarly, the domain of the third layer will be:
\n$$w^4, (w\\cdot h)^4, (w\\cdot h^2)^4, ..., (w\\cdot h^{2047})^4$$\n\nAnd so on.\n\n#### FRI Folding Operator\nThe first FRI polynomial is simply the composition polynomial, i.e., `cp`.
\nEach subsequent FRI polynomial is obtained by:\n1. Getting a random field element \\\\(\\beta\\\\) (by calling `Channel.receive_random_field_element`).\n2. Multiplying the odd coefficients of the previous polynomial by \\\\(\\beta\\\\).\n3. Summing together consecutive pairs (even-odd) of coefficients.\n\nFormally, let's say that the k-th polynomial is of degree \\\\(< m\\\\) (for some \\\\(m\\\\) which is a power of 2):\n\n$$p_{k}(x) := \\sum _{i=0} ^{m-1} c_i x^i$$\n\n\nThen the (k+1)-th polynomial, whose degree is \\\\(< \\frac m 2 \\\\) will be:\n\n\\\\[ p_{k+1}(x) := \\sum_{i=0} ^{ m / 2 - 1 } (c_{2i} + \\beta \\cdot c_{2i + 1}) x^i \\\\]
\n\n![fri](images/zkp/stark/fri.png)\n\n![fri example](images/zkp/stark/fri_example.png)\n\n### Generating FRI Commitments\n\nWe have now developed the tools to write the `FriCommit` method, that contains the main FRI commitment loop.
\n\nIt takes the following 5 arguments:\n1. The composition polynomial, that is also the first FRI polynomial, that is - `cp`.\n2. The coset of order 8192 that is also the first FRI domain, that is - `eval_domain`.\n3. The evaluation of the former over the latter, which is also the first FRI layer , that is - `cp_eval`.\n4. The first Merkle tree (we will have one for each FRI layer) constructed from these evaluations, that is - `cp_merkle`.\n5. A channel object, that is `channel`.\n\nThe method accordingly returns 4 lists:\n1. The FRI polynomials.\n2. The FRI domains.\n3. The FRI layers.\n4. The FRI Merkle trees.\n\nThe method contains a loop, in each iteration of which we extend these four lists, using the last element in each.\nThe iteration should stop once the last FRI polynomial is of degree 0, that is - when the last FRI polynomial is just a constant. It should then send over the channel this constant (i.e. - the polynomial's free term).\nThe `Channel` class only supports sending strings, so make sure you convert anything you wish to send over the channel to a string before sending.\n\n![commitment](images/zkp/stark/commitment.png)\n\n## Query Phase\nGet q random elements, provide a valdiation data for each\n\n![decommitment](images/zkp/stark/decommitment.png)\n\n# references\n- https://github.com/starkware-industries/stark101\n- [startk_math](https://medium.com/starkware/tagged/stark-math)\n- [starkEx deep dive](https://medium.com/starkware/starkdex-deep-dive-introduction-7b4ef0dedba8)\n- [coset] https://en.wikipedia.org/wiki/Coset\n- [starkware math](https://medium.com/starkware/arithmetization-ii-403c3b3f4355)","source":"_posts/cryptography/zkp/stark-101.md","raw":"---\ntitle: stark 101\ndate: 2023-11-03 17:07:33\ntags: [cryptography,zkp]\n---\n\n\n\n## trace and low degree extension\nthe objective is to develop a STARK prover for the FibonacciSq sequence over a finite field. The FibonacciSq sequence is defined by the recurrence relation\n\\\\[ a_{n+2} = a_{n+1} ^2 + a_n ^2 \\\\]\n\nthe statement is: **I know a FieldElement \\\\(X\\in \\mathbb{F}\\\\) such that the 1023rd element of the FibonacciSq sequence starting with \\\\(1, X\\\\) is \\\\(2338775057\\\\)**\n\nThe underlying field of this class is \\\\(\\mathbb{F}_{3221225473}\\\\) (\\\\(3221225473 = 3 \\cdot 2^{30} + 1\\\\)), so all operations are done modulo 3221225473.\n\n\n### FibonacciSq Trace\n let's construct a list `a` of length 1023, whose first two elements will be FieldElement objects representing 1 and 3141592, respectively. The next 1021 elements will be the FibonacciSq sequence induced by these two elements. `a` is called the **trace** of FibonacciSq, or, when the context is clear, the trace.\n\n### Thinking of Polynomials\nWe now want to think of the sequence as the evaluation of some polynomial \\\\(f\\\\) of degree 1022.\nWe will choose the domain to be some subgroup \\\\(G \\subseteq \\mathbb{F}^\\times\\\\) of size 1024, for reasons that will become clear later.\n\n(Recall that \\\\(\\mathbb{F}^\\times\\\\) denotes the multiplicative group of \\\\(\\mathbb{F}\\\\), which we get from \\\\(\\mathbb{F}\\\\) by omitting the zero element with the induced multiplication from the field. A subgroup of size 1024 exists because \\\\(\\mathbb{F}^\\times\\\\) is a cyclic group of size \\\\(3\\cdot 2^{30}\\\\), so it contains a subgroup of size \\\\(2^i\\\\) for any \\\\(0 \\leq i \\leq 30\\\\)).\n#### Find a Group of Size 1024\nIf we find an element \\\\(g \\in \\mathbb{F}\\\\) whose (multiplicative) order is 1024, then \\\\(g\\\\) will generate such a group. Create a list called `G` with all the elements of \\\\(G\\\\), such that \\\\(G[i] := g^i\\\\).\n\n\n### Evaluating on a Larger Domain\nthen, interpolating `G` over `a` we get a polynomial `f`. The trace, viewed as evaluations of a polynomial \\\\(f\\\\) on \\\\(G\\\\), can now be extended by evaluating \\\\(f\\\\) over a larger domain, thereby creating a Reed-Solomon error correction code.\n\n#### Cosets\nTo that end, we must decide on a larger domain on which \\\\(f\\\\) will be evaluated. We will work with a domain that is 8 times larger than \\\\(G\\\\).
A natural choice for such a domain is to take some group \\\\(H\\\\) of size 8192 (which exists because 8192 divides \\\\(|\\mathbb{F}^\\times|\\\\)), and shift it by the generator of \\\\(\\mathbb{F}^\\times\\\\), thereby obtaining a [coset](https://en.wikipedia.org/wiki/Coset) of \\\\(H\\\\).\n\nCreate a list called `H` of the elements of \\\\(H\\\\), and multiply each of them by the generator of \\\\(\\mathbb{F}^\\times\\\\) to obtain a list called `eval_domain`. In other words, `eval_domain` = \\\\(\\\\{w\\cdot h^i | 0 \\leq i <8192 \\\\}\\\\) for \\\\(h\\\\) the generator of \\\\(H\\\\) and \\\\(w\\\\) the generator of \\\\(\\mathbb{F}^\\times\\\\).\n\n\n#### Evaluate on a Coset\n```python\nf = interpolate_poly(G[:-1], a)\nf_eval = [f(d) for d in eval_domain]\n```\n\n### Commitments\nWe will use [Sha256](https://en.wikipedia.org/wiki/SHA-2)-based [Merkle Trees](https://en.wikipedia.org/wiki/Merkle_tree) as our commitment scheme.\n```python\nfrom merkle import MerkleTree\nf_merkle = MerkleTree(f_eval)\n```\n\n### Channel\nTheoretically, a STARK proof system is a protocol for interaction between two parties - a prover and a verifier. In practice, we convert this interactive protocol into a non-interactive proof using the [Fiat-Shamir Heuristic](https://en.wikipedia.org/wiki/Fiat%E2%80%93Shamir_heuristic). In this tutorial you will use the `Channel` class, which implements this transformation. This channel replaces the verifier in the sense that the prover (which you are writing) will send data, and receive random numbers or random `FieldElement` instances.\n\n\n\n\n## constraints\nIn this part, we are going to create a set of constraints over the trace `a`. \n### Step 1 - FibonacciSq Constraints\nFor `a` to be a correct trace of a FibonacciSq sequence that proves our claim:\n1. The first element has to be 1, namely \\\\(a[0] = 1\\\\).\n2. The last element has to be 2338775057, namely \\\\(a[1022] = 2338775057\\\\).\n3. The FibonacciSq rule must apply, that is - for every \\\\(i<1021\\\\), \\\\(a[i+2]=a[i+1]^2+a[i]^2\\\\).\n\n\n### Step 2 - Polynomial Constraints\nRecall that `f` is a polynomial over the trace domain, that evaluates exactly to `a` over \\\\(G \\setminus \\{g^{1023}\\}\\\\) where \\\\(G=\\{g^i : 0\\leq i\\leq 1023\\}\\\\) is the \"small\" group generated by \\\\(g\\\\).
\n\nWe now rewrite the above three constraints in a form of polynomial constraints over `f`:\n1. \\\\(a[0] = 1\\\\) is translated to the polynomial \\\\(f(x) - 1\\\\), which evalutes to 0 for \\\\(x = g^0\\\\) (note that \\\\(g^0\\\\) is \\\\(1\\\\)).
\n2. \\\\(a[1022] = 2338775057\\\\) is translated to the polynomial \\\\(f(x) - 2338775057\\\\), which evalutes to 0 for \\\\(x = g^{1022}\\\\).
\n3. \\\\(a[i+2]=a[i+1]^2+a[i]^2\\\\) for every \\\\(i<1021\\\\) is translated to the polynomial \\\\(f(g^2 \\cdot x) - (f(g \\cdot x))^2 - (f(x))^2\\\\), which evaluates to 0 for \\\\(x \\in G \\backslash \\{g^{1021}, g^{1022}, g^{1023}\\}\\\\).

\n\n### Step 3 - Rational Functions (That are in Fact Polynomials)\n\nEach of the constraints above is represented by a polynomial \\\\(u(x)\\\\) that supposedly evaluates to \\\\(0\\\\) on certain elements of the group \\\\(G\\\\). That is, for some \\\\(x_0, \\ldots, x_k \\in G\\\\), we claim that\n\n\\\\[u(x_0) = \\ldots = u(x_k) = 0\\\\]\n\n(note that for the first two constaints, \\\\(k=0\\\\) because they only refer to one point and for the third \\\\(k=1021\\\\)).\n\nThis is equivalent to saying that \\\\(u(x)\\\\) is divisible, as a polynomial, by all of \\\\(\\{(x-x_i)\\}_{i=0}^k\\\\), or, equivalently, by\n\n\\\\[\\prod_{i=0}^k (x-x_i)\\\\]\n\nTherefore, each of the three constraints above can be written as a rational function of the form:\n\n\\\\[\\frac{u(x)}{\\prod_{i=0}^k (x-x_i)}\\\\]\n\nfor the corresponding \\\\(u(x)\\\\) and \\\\(\\{x_i\\}_{i=0}^k\\\\). In this step we will construct these three rational functions and show that they are indeed polynomials.\n\n### The First Constraint:\n\nIn the first constraint, \\\\(f(x) - 1\\\\) and \\\\(\\{x_i\\} = \\{1\\}\\\\).\n\nWe will now construct the **polynomial** \\\\(p_0(x)=\\frac{f(x) - 1}{x - 1}\\\\), making sure that \\\\(f(x) - 1\\\\) is indeed divisible by \\\\((x-1)\\\\).\n\n### The Second Constraint\n\nConstruct the polynomial `p1` representing the second constraint, \\\\(p_1(x)= \\frac{f(x) - 2338775057}{x - g^{1022}}\\\\), similarly:
\n\n### The Third Constraint - Succinctness\n\nThe last constraint's rational function is slightly more complicated:
\n\n\n\\\\[p_2(x) = \\frac{f(g^2 \\cdot x) - (f(g \\cdot x))^2 - (f(x))^2}{\\prod\\limits_{i=0}^{1020} (x-g^i)}\\\\]\n\nwhose denominator can be rewritten, so that the entire expression is easier to compute:
\n\n$$\\frac{f(g^2 \\cdot x) - (f(g \\cdot x))^2 - (f(x))^2}{\\frac{x^{1024} - 1}{(x-g^{1021})(x-g^{1022})(x-g^{1023})}}$$
\n\nThis follows from the equality\n\n$$\\prod\\limits_{i=0}^{1023} (x-g^i) = x^{1024} - 1$$\n\n### Step 4 - Composition Polynomial\nRecall that we're translating a problem of checking the validity of three polynomial constraints to checking that each of the rational functions \\\\(p_0, p_1, p_2\\\\) are polynomials.
\n\nOur protocol uses an algorithm called [FRI](https://eccc.weizmann.ac.il/report/2017/134/) to do so, which will be discussed in the next part.
\nIn order for the proof to be succinct (short), we prefer to work with just one rational function instead of three. For that, we take a random linear combination of \\\\(p_0, p_1, p_2\\\\) called the **compostion polynomial** (CP for short):\n\n$$CP(x) = \\alpha_0 \\cdot p_0(x) + \\alpha_1 \\cdot p_1(x) + \\alpha_2 \\cdot p_2(x)$$
\n\nwhere \\\\(\\alpha_0, \\alpha_1, \\alpha_2 \\\\) are random field elements obtained from the verifier, or in our case - from the channel.\n\nProving that (the rational function) \\\\(CP\\\\) is a polynomial guarantess, with high probability, that each of \\\\(p_0, p_1, p_2\\\\) are themselves polynomials.\n\n### Commit on the Composition Polynomial\nLastly, we evaluate $cp$ over the evaluation domain (`eval_domain`), build a Merkle tree on top of that and send its root over the channel. This is similar to commiting on the LDE trace, as we did at the end of part 1.\n\n![trace to cp](images/zkp/stark/trace_to_CP.png)\n\n## FRI Commitments\n### FRI Folding\n\nOur goal in this part is to construct the FRI layers and commit on them. \n
To obtain each layer we need:\n1. To generate a domain for the layer (from the previous layer's domain).\n2. To generate a polynomial for the layer (from the previous layer's polynomial and domain).\n3. To evaluate said polynomial on said domain - **this is the next FRI layer**.\n\n#### Domain Generation\n\nThe first FRI domain is simply the `eval_domain` that you already generated in Part 1, namely a coset of a group of order 8192. Each subsequent FRI domain is obtained by taking the first half of the previous FRI domain (dropping the second half), and squaring each of its elements.
\n\nFormally - we got `eval_domain` by taking:
\n$$w, w\\cdot h, w\\cdot h^2, ..., w\\cdot h^{8191}$$\n\nThe next layer will therefore be:
\n$$w^2, (w\\cdot h)^2, (w\\cdot h^2)^2, ..., (w\\cdot h^{4095})^2$$\n\nNote that taking the squares of the second half of each elements in `eval_domain` yields exactly\nthe same result as taking the squares of the first half. This is true for the next layers as well.\n\nSimilarly, the domain of the third layer will be:
\n$$w^4, (w\\cdot h)^4, (w\\cdot h^2)^4, ..., (w\\cdot h^{2047})^4$$\n\nAnd so on.\n\n#### FRI Folding Operator\nThe first FRI polynomial is simply the composition polynomial, i.e., `cp`.
\nEach subsequent FRI polynomial is obtained by:\n1. Getting a random field element \\\\(\\beta\\\\) (by calling `Channel.receive_random_field_element`).\n2. Multiplying the odd coefficients of the previous polynomial by \\\\(\\beta\\\\).\n3. Summing together consecutive pairs (even-odd) of coefficients.\n\nFormally, let's say that the k-th polynomial is of degree \\\\(< m\\\\) (for some \\\\(m\\\\) which is a power of 2):\n\n$$p_{k}(x) := \\sum _{i=0} ^{m-1} c_i x^i$$\n\n\nThen the (k+1)-th polynomial, whose degree is \\\\(< \\frac m 2 \\\\) will be:\n\n\\\\[ p_{k+1}(x) := \\sum_{i=0} ^{ m / 2 - 1 } (c_{2i} + \\beta \\cdot c_{2i + 1}) x^i \\\\]
\n\n![fri](images/zkp/stark/fri.png)\n\n![fri example](images/zkp/stark/fri_example.png)\n\n### Generating FRI Commitments\n\nWe have now developed the tools to write the `FriCommit` method, that contains the main FRI commitment loop.
\n\nIt takes the following 5 arguments:\n1. The composition polynomial, that is also the first FRI polynomial, that is - `cp`.\n2. The coset of order 8192 that is also the first FRI domain, that is - `eval_domain`.\n3. The evaluation of the former over the latter, which is also the first FRI layer , that is - `cp_eval`.\n4. The first Merkle tree (we will have one for each FRI layer) constructed from these evaluations, that is - `cp_merkle`.\n5. A channel object, that is `channel`.\n\nThe method accordingly returns 4 lists:\n1. The FRI polynomials.\n2. The FRI domains.\n3. The FRI layers.\n4. The FRI Merkle trees.\n\nThe method contains a loop, in each iteration of which we extend these four lists, using the last element in each.\nThe iteration should stop once the last FRI polynomial is of degree 0, that is - when the last FRI polynomial is just a constant. It should then send over the channel this constant (i.e. - the polynomial's free term).\nThe `Channel` class only supports sending strings, so make sure you convert anything you wish to send over the channel to a string before sending.\n\n![commitment](images/zkp/stark/commitment.png)\n\n## Query Phase\nGet q random elements, provide a valdiation data for each\n\n![decommitment](images/zkp/stark/decommitment.png)\n\n# references\n- https://github.com/starkware-industries/stark101\n- [startk_math](https://medium.com/starkware/tagged/stark-math)\n- [starkEx deep dive](https://medium.com/starkware/starkdex-deep-dive-introduction-7b4ef0dedba8)\n- [coset] https://en.wikipedia.org/wiki/Coset\n- [starkware math](https://medium.com/starkware/arithmetization-ii-403c3b3f4355)","slug":"cryptography/zkp/stark-101","published":1,"updated":"2023-11-05T04:18:25.566Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clphpvdoj0005aj7ug6vbhr5l","content":"\n\n\n

trace and low degree extension

the objective is to develop a STARK prover for the FibonacciSq sequence over a finite field. The FibonacciSq sequence is defined by the recurrence relation
\\[ a_{n+2} = a_{n+1} ^2 + a_n ^2 \\]

\n

the statement is: I know a FieldElement \\(X\\in \\mathbb{F}\\) such that the 1023rd element of the FibonacciSq sequence starting with \\(1, X\\) is \\(2338775057\\)

\n

The underlying field of this class is \\(\\mathbb{F}_{3221225473}\\) (\\(3221225473 = 3 \\cdot 2^{30} + 1\\)), so all operations are done modulo 3221225473.

\n

FibonacciSq Trace

let’s construct a list a of length 1023, whose first two elements will be FieldElement objects representing 1 and 3141592, respectively. The next 1021 elements will be the FibonacciSq sequence induced by these two elements. a is called the trace of FibonacciSq, or, when the context is clear, the trace.

\n

Thinking of Polynomials

We now want to think of the sequence as the evaluation of some polynomial \\(f\\) of degree 1022.
We will choose the domain to be some subgroup \\(G \\subseteq \\mathbb{F}^\\times\\) of size 1024, for reasons that will become clear later.

\n

(Recall that \\(\\mathbb{F}^\\times\\) denotes the multiplicative group of \\(\\mathbb{F}\\), which we get from \\(\\mathbb{F}\\) by omitting the zero element with the induced multiplication from the field. A subgroup of size 1024 exists because \\(\\mathbb{F}^\\times\\) is a cyclic group of size \\(3\\cdot 2^{30}\\), so it contains a subgroup of size \\(2^i\\) for any \\(0 \\leq i \\leq 30\\)).

\n

Find a Group of Size 1024

If we find an element \\(g \\in \\mathbb{F}\\) whose (multiplicative) order is 1024, then \\(g\\) will generate such a group. Create a list called G with all the elements of \\(G\\), such that \\(G[i] := g^i\\).

\n

Evaluating on a Larger Domain

then, interpolating G over a we get a polynomial f. The trace, viewed as evaluations of a polynomial \\(f\\) on \\(G\\), can now be extended by evaluating \\(f\\) over a larger domain, thereby creating a Reed-Solomon error correction code.

\n

Cosets

To that end, we must decide on a larger domain on which \\(f\\) will be evaluated. We will work with a domain that is 8 times larger than \\(G\\).
A natural choice for such a domain is to take some group \\(H\\) of size 8192 (which exists because 8192 divides \\(|\\mathbb{F}^\\times|\\)), and shift it by the generator of \\(\\mathbb{F}^\\times\\), thereby obtaining a coset of \\(H\\).

\n

Create a list called H of the elements of \\(H\\), and multiply each of them by the generator of \\(\\mathbb{F}^\\times\\) to obtain a list called eval_domain. In other words, eval_domain = \\(\\{w\\cdot h^i | 0 \\leq i <8192 \\}\\) for \\(h\\) the generator of \\(H\\) and \\(w\\) the generator of \\(\\mathbb{F}^\\times\\).

\n

Evaluate on a Coset

1
2
f = interpolate_poly(G[:-1], a)
f_eval = [f(d) for d in eval_domain]
\n\n

Commitments

We will use Sha256-based Merkle Trees as our commitment scheme.

\n
1
2
from merkle import MerkleTree
f_merkle = MerkleTree(f_eval)
\n\n

Channel

Theoretically, a STARK proof system is a protocol for interaction between two parties - a prover and a verifier. In practice, we convert this interactive protocol into a non-interactive proof using the Fiat-Shamir Heuristic. In this tutorial you will use the Channel class, which implements this transformation. This channel replaces the verifier in the sense that the prover (which you are writing) will send data, and receive random numbers or random FieldElement instances.

\n

constraints

In this part, we are going to create a set of constraints over the trace a.

\n

Step 1 - FibonacciSq Constraints

For a to be a correct trace of a FibonacciSq sequence that proves our claim:

\n
    \n
  1. The first element has to be 1, namely \\(a[0] = 1\\).
    2. \n
  2. The last element has to be 2338775057, namely \\(a[1022] = 2338775057\\).
    3. \n
  3. The FibonacciSq rule must apply, that is - for every \\(i<1021\\), \\(a[i+2]=a[i+1]^2+a[i]^2\\).
    4. \n
\n

Step 2 - Polynomial Constraints

Recall that f is a polynomial over the trace domain, that evaluates exactly to a over \\(G \\setminus {g^{1023}}\\) where \\(G={g^i : 0\\leq i\\leq 1023}\\) is the “small” group generated by \\(g\\).

\n

We now rewrite the above three constraints in a form of polynomial constraints over f:

\n
    \n
  1. \\(a[0] = 1\\) is translated to the polynomial \\(f(x) - 1\\), which evalutes to 0 for \\(x = g^0\\) (note that \\(g^0\\) is \\(1\\)).
    2. \n
  2. \\(a[1022] = 2338775057\\) is translated to the polynomial \\(f(x) - 2338775057\\), which evalutes to 0 for \\(x = g^{1022}\\).
    3. \n
  3. \\(a[i+2]=a[i+1]^2+a[i]^2\\) for every \\(i<1021\\) is translated to the polynomial \\(f(g^2 \\cdot x) - (f(g \\cdot x))^2 - (f(x))^2\\), which evaluates to 0 for \\(x \\in G \\backslash {g^{1021}, g^{1022}, g^{1023}}\\).

    4. \n
\n

Step 3 - Rational Functions (That are in Fact Polynomials)

Each of the constraints above is represented by a polynomial \\(u(x)\\) that supposedly evaluates to \\(0\\) on certain elements of the group \\(G\\). That is, for some \\(x_0, \\ldots, x_k \\in G\\), we claim that

\n

\\[u(x_0) = \\ldots = u(x_k) = 0\\]

\n

(note that for the first two constaints, \\(k=0\\) because they only refer to one point and for the third \\(k=1021\\)).

\n

This is equivalent to saying that \\(u(x)\\) is divisible, as a polynomial, by all of \\({(x-x_i)}_{i=0}^k\\), or, equivalently, by

\n

\\[\\prod_{i=0}^k (x-x_i)\\]

\n

Therefore, each of the three constraints above can be written as a rational function of the form:

\n

\\[\\frac{u(x)}{\\prod_{i=0}^k (x-x_i)}\\]

\n

for the corresponding \\(u(x)\\) and \\({x_i}_{i=0}^k\\). In this step we will construct these three rational functions and show that they are indeed polynomials.

\n

The First Constraint:

In the first constraint, \\(f(x) - 1\\) and \\({x_i} = {1}\\).

\n

We will now construct the polynomial \\(p_0(x)=\\frac{f(x) - 1}{x - 1}\\), making sure that \\(f(x) - 1\\) is indeed divisible by \\((x-1)\\).

\n

The Second Constraint

Construct the polynomial p1 representing the second constraint, \\(p_1(x)= \\frac{f(x) - 2338775057}{x - g^{1022}}\\), similarly:

\n

The Third Constraint - Succinctness

The last constraint’s rational function is slightly more complicated:

\n

\\[p_2(x) = \\frac{f(g^2 \\cdot x) - (f(g \\cdot x))^2 - (f(x))^2}{\\prod\\limits_{i=0}^{1020} (x-g^i)}\\]

\n

whose denominator can be rewritten, so that the entire expression is easier to compute:

\n

$$\\frac{f(g^2 \\cdot x) - (f(g \\cdot x))^2 - (f(x))^2}{\\frac{x^{1024} - 1}{(x-g^{1021})(x-g^{1022})(x-g^{1023})}}$$

\n

This follows from the equality

\n

$$\\prod\\limits_{i=0}^{1023} (x-g^i) = x^{1024} - 1$$

\n

Step 4 - Composition Polynomial

Recall that we’re translating a problem of checking the validity of three polynomial constraints to checking that each of the rational functions \\(p_0, p_1, p_2\\) are polynomials.

\n

Our protocol uses an algorithm called FRI to do so, which will be discussed in the next part.

In order for the proof to be succinct (short), we prefer to work with just one rational function instead of three. For that, we take a random linear combination of \\(p_0, p_1, p_2\\) called the compostion polynomial (CP for short):

\n

$$CP(x) = \\alpha_0 \\cdot p_0(x) + \\alpha_1 \\cdot p_1(x) + \\alpha_2 \\cdot p_2(x)$$

\n

where \\(\\alpha_0, \\alpha_1, \\alpha_2 \\) are random field elements obtained from the verifier, or in our case - from the channel.

\n

Proving that (the rational function) \\(CP\\) is a polynomial guarantess, with high probability, that each of \\(p_0, p_1, p_2\\) are themselves polynomials.

\n

Commit on the Composition Polynomial

Lastly, we evaluate $cp$ over the evaluation domain (eval_domain), build a Merkle tree on top of that and send its root over the channel. This is similar to commiting on the LDE trace, as we did at the end of part 1.

\n

\"trace

\n

FRI Commitments

FRI Folding

Our goal in this part is to construct the FRI layers and commit on them.

To obtain each layer we need:

\n
    \n
  1. To generate a domain for the layer (from the previous layer’s domain).
    2. \n
  2. To generate a polynomial for the layer (from the previous layer’s polynomial and domain).
    3. \n
  3. To evaluate said polynomial on said domain - this is the next FRI layer.
    4. \n
\n

Domain Generation

The first FRI domain is simply the eval_domain that you already generated in Part 1, namely a coset of a group of order 8192. Each subsequent FRI domain is obtained by taking the first half of the previous FRI domain (dropping the second half), and squaring each of its elements.

\n

Formally - we got eval_domain by taking:

$$w, w\\cdot h, w\\cdot h^2, …, w\\cdot h^{8191}$$

\n

The next layer will therefore be:

$$w^2, (w\\cdot h)^2, (w\\cdot h^2)^2, …, (w\\cdot h^{4095})^2$$

\n

Note that taking the squares of the second half of each elements in eval_domain yields exactly
the same result as taking the squares of the first half. This is true for the next layers as well.

\n

Similarly, the domain of the third layer will be:

$$w^4, (w\\cdot h)^4, (w\\cdot h^2)^4, …, (w\\cdot h^{2047})^4$$

\n

And so on.

\n

FRI Folding Operator

The first FRI polynomial is simply the composition polynomial, i.e., cp.

Each subsequent FRI polynomial is obtained by:

\n
    \n
  1. Getting a random field element \\(\\beta\\) (by calling Channel.receive_random_field_element).
    2. \n
  2. Multiplying the odd coefficients of the previous polynomial by \\(\\beta\\).
    3. \n
  3. Summing together consecutive pairs (even-odd) of coefficients.
    4. \n
\n

Formally, let’s say that the k-th polynomial is of degree \\(< m\\) (for some \\(m\\) which is a power of 2):

\n

$$p_{k}(x) := \\sum _{i=0} ^{m-1} c_i x^i$$

\n

Then the (k+1)-th polynomial, whose degree is \\(< \\frac m 2 \\) will be:

\n

\\[ p_{k+1}(x) := \\sum_{i=0} ^{ m / 2 - 1 } (c_{2i} + \\beta \\cdot c_{2i + 1}) x^i \\]

\n

\"fri\"

\n

\"fri

\n

Generating FRI Commitments

We have now developed the tools to write the FriCommit method, that contains the main FRI commitment loop.

\n

It takes the following 5 arguments:

\n
    \n
  1. The composition polynomial, that is also the first FRI polynomial, that is - cp.
    2. \n
  2. The coset of order 8192 that is also the first FRI domain, that is - eval_domain.
    3. \n
  3. The evaluation of the former over the latter, which is also the first FRI layer , that is - cp_eval.
    4. \n
  4. The first Merkle tree (we will have one for each FRI layer) constructed from these evaluations, that is - cp_merkle.
    5. \n
  5. A channel object, that is channel.
    6. \n
\n

The method accordingly returns 4 lists:

\n
    \n
  1. The FRI polynomials.
    2. \n
  2. The FRI domains.
    3. \n
  3. The FRI layers.
    4. \n
  4. The FRI Merkle trees.
    5. \n
\n

The method contains a loop, in each iteration of which we extend these four lists, using the last element in each.
The iteration should stop once the last FRI polynomial is of degree 0, that is - when the last FRI polynomial is just a constant. It should then send over the channel this constant (i.e. - the polynomial’s free term).
The Channel class only supports sending strings, so make sure you convert anything you wish to send over the channel to a string before sending.

\n

\"commitment\"

\n

Query Phase

Get q random elements, provide a valdiation data for each

\n

\"decommitment\"

\n

references

\n","site":{"data":{}},"excerpt":"","more":"\n\n\n

trace and low degree extension

the objective is to develop a STARK prover for the FibonacciSq sequence over a finite field. The FibonacciSq sequence is defined by the recurrence relation
\\[ a_{n+2} = a_{n+1} ^2 + a_n ^2 \\]

\n

the statement is: I know a FieldElement \\(X\\in \\mathbb{F}\\) such that the 1023rd element of the FibonacciSq sequence starting with \\(1, X\\) is \\(2338775057\\)

\n

The underlying field of this class is \\(\\mathbb{F}_{3221225473}\\) (\\(3221225473 = 3 \\cdot 2^{30} + 1\\)), so all operations are done modulo 3221225473.

\n

FibonacciSq Trace

let’s construct a list a of length 1023, whose first two elements will be FieldElement objects representing 1 and 3141592, respectively. The next 1021 elements will be the FibonacciSq sequence induced by these two elements. a is called the trace of FibonacciSq, or, when the context is clear, the trace.

\n

Thinking of Polynomials

We now want to think of the sequence as the evaluation of some polynomial \\(f\\) of degree 1022.
We will choose the domain to be some subgroup \\(G \\subseteq \\mathbb{F}^\\times\\) of size 1024, for reasons that will become clear later.

\n

(Recall that \\(\\mathbb{F}^\\times\\) denotes the multiplicative group of \\(\\mathbb{F}\\), which we get from \\(\\mathbb{F}\\) by omitting the zero element with the induced multiplication from the field. A subgroup of size 1024 exists because \\(\\mathbb{F}^\\times\\) is a cyclic group of size \\(3\\cdot 2^{30}\\), so it contains a subgroup of size \\(2^i\\) for any \\(0 \\leq i \\leq 30\\)).

\n

Find a Group of Size 1024

If we find an element \\(g \\in \\mathbb{F}\\) whose (multiplicative) order is 1024, then \\(g\\) will generate such a group. Create a list called G with all the elements of \\(G\\), such that \\(G[i] := g^i\\).

\n

Evaluating on a Larger Domain

then, interpolating G over a we get a polynomial f. The trace, viewed as evaluations of a polynomial \\(f\\) on \\(G\\), can now be extended by evaluating \\(f\\) over a larger domain, thereby creating a Reed-Solomon error correction code.

\n

Cosets

To that end, we must decide on a larger domain on which \\(f\\) will be evaluated. We will work with a domain that is 8 times larger than \\(G\\).
A natural choice for such a domain is to take some group \\(H\\) of size 8192 (which exists because 8192 divides \\(|\\mathbb{F}^\\times|\\)), and shift it by the generator of \\(\\mathbb{F}^\\times\\), thereby obtaining a coset of \\(H\\).

\n

Create a list called H of the elements of \\(H\\), and multiply each of them by the generator of \\(\\mathbb{F}^\\times\\) to obtain a list called eval_domain. In other words, eval_domain = \\(\\{w\\cdot h^i | 0 \\leq i <8192 \\}\\) for \\(h\\) the generator of \\(H\\) and \\(w\\) the generator of \\(\\mathbb{F}^\\times\\).

\n

Evaluate on a Coset

1
2
f = interpolate_poly(G[:-1], a)
f_eval = [f(d) for d in eval_domain]
\n\n

Commitments

We will use Sha256-based Merkle Trees as our commitment scheme.

\n
1
2
from merkle import MerkleTree
f_merkle = MerkleTree(f_eval)
\n\n

Channel

Theoretically, a STARK proof system is a protocol for interaction between two parties - a prover and a verifier. In practice, we convert this interactive protocol into a non-interactive proof using the Fiat-Shamir Heuristic. In this tutorial you will use the Channel class, which implements this transformation. This channel replaces the verifier in the sense that the prover (which you are writing) will send data, and receive random numbers or random FieldElement instances.

\n

constraints

In this part, we are going to create a set of constraints over the trace a.

\n

Step 1 - FibonacciSq Constraints

For a to be a correct trace of a FibonacciSq sequence that proves our claim:

\n
    \n
  1. The first element has to be 1, namely \\(a[0] = 1\\).
    2. \n
  2. The last element has to be 2338775057, namely \\(a[1022] = 2338775057\\).
    3. \n
  3. The FibonacciSq rule must apply, that is - for every \\(i<1021\\), \\(a[i+2]=a[i+1]^2+a[i]^2\\).
    4. \n
\n

Step 2 - Polynomial Constraints

Recall that f is a polynomial over the trace domain, that evaluates exactly to a over \\(G \\setminus {g^{1023}}\\) where \\(G={g^i : 0\\leq i\\leq 1023}\\) is the “small” group generated by \\(g\\).

\n

We now rewrite the above three constraints in a form of polynomial constraints over f:

\n
    \n
  1. \\(a[0] = 1\\) is translated to the polynomial \\(f(x) - 1\\), which evalutes to 0 for \\(x = g^0\\) (note that \\(g^0\\) is \\(1\\)).
    2. \n
  2. \\(a[1022] = 2338775057\\) is translated to the polynomial \\(f(x) - 2338775057\\), which evalutes to 0 for \\(x = g^{1022}\\).
    3. \n
  3. \\(a[i+2]=a[i+1]^2+a[i]^2\\) for every \\(i<1021\\) is translated to the polynomial \\(f(g^2 \\cdot x) - (f(g \\cdot x))^2 - (f(x))^2\\), which evaluates to 0 for \\(x \\in G \\backslash {g^{1021}, g^{1022}, g^{1023}}\\).

    4. \n
\n

Step 3 - Rational Functions (That are in Fact Polynomials)

Each of the constraints above is represented by a polynomial \\(u(x)\\) that supposedly evaluates to \\(0\\) on certain elements of the group \\(G\\). That is, for some \\(x_0, \\ldots, x_k \\in G\\), we claim that

\n

\\[u(x_0) = \\ldots = u(x_k) = 0\\]

\n

(note that for the first two constaints, \\(k=0\\) because they only refer to one point and for the third \\(k=1021\\)).

\n

This is equivalent to saying that \\(u(x)\\) is divisible, as a polynomial, by all of \\({(x-x_i)}_{i=0}^k\\), or, equivalently, by

\n

\\[\\prod_{i=0}^k (x-x_i)\\]

\n

Therefore, each of the three constraints above can be written as a rational function of the form:

\n

\\[\\frac{u(x)}{\\prod_{i=0}^k (x-x_i)}\\]

\n

for the corresponding \\(u(x)\\) and \\({x_i}_{i=0}^k\\). In this step we will construct these three rational functions and show that they are indeed polynomials.

\n

The First Constraint:

In the first constraint, \\(f(x) - 1\\) and \\({x_i} = {1}\\).

\n

We will now construct the polynomial \\(p_0(x)=\\frac{f(x) - 1}{x - 1}\\), making sure that \\(f(x) - 1\\) is indeed divisible by \\((x-1)\\).

\n

The Second Constraint

Construct the polynomial p1 representing the second constraint, \\(p_1(x)= \\frac{f(x) - 2338775057}{x - g^{1022}}\\), similarly:

\n

The Third Constraint - Succinctness

The last constraint’s rational function is slightly more complicated:

\n

\\[p_2(x) = \\frac{f(g^2 \\cdot x) - (f(g \\cdot x))^2 - (f(x))^2}{\\prod\\limits_{i=0}^{1020} (x-g^i)}\\]

\n

whose denominator can be rewritten, so that the entire expression is easier to compute:

\n

$$\\frac{f(g^2 \\cdot x) - (f(g \\cdot x))^2 - (f(x))^2}{\\frac{x^{1024} - 1}{(x-g^{1021})(x-g^{1022})(x-g^{1023})}}$$

\n

This follows from the equality

\n

$$\\prod\\limits_{i=0}^{1023} (x-g^i) = x^{1024} - 1$$

\n

Step 4 - Composition Polynomial

Recall that we’re translating a problem of checking the validity of three polynomial constraints to checking that each of the rational functions \\(p_0, p_1, p_2\\) are polynomials.

\n

Our protocol uses an algorithm called FRI to do so, which will be discussed in the next part.

In order for the proof to be succinct (short), we prefer to work with just one rational function instead of three. For that, we take a random linear combination of \\(p_0, p_1, p_2\\) called the compostion polynomial (CP for short):

\n

$$CP(x) = \\alpha_0 \\cdot p_0(x) + \\alpha_1 \\cdot p_1(x) + \\alpha_2 \\cdot p_2(x)$$

\n

where \\(\\alpha_0, \\alpha_1, \\alpha_2 \\) are random field elements obtained from the verifier, or in our case - from the channel.

\n

Proving that (the rational function) \\(CP\\) is a polynomial guarantess, with high probability, that each of \\(p_0, p_1, p_2\\) are themselves polynomials.

\n

Commit on the Composition Polynomial

Lastly, we evaluate $cp$ over the evaluation domain (eval_domain), build a Merkle tree on top of that and send its root over the channel. This is similar to commiting on the LDE trace, as we did at the end of part 1.

\n

\"trace

\n

FRI Commitments

FRI Folding

Our goal in this part is to construct the FRI layers and commit on them.

To obtain each layer we need:

\n
    \n
  1. To generate a domain for the layer (from the previous layer’s domain).
    2. \n
  2. To generate a polynomial for the layer (from the previous layer’s polynomial and domain).
    3. \n
  3. To evaluate said polynomial on said domain - this is the next FRI layer.
    4. \n
\n

Domain Generation

The first FRI domain is simply the eval_domain that you already generated in Part 1, namely a coset of a group of order 8192. Each subsequent FRI domain is obtained by taking the first half of the previous FRI domain (dropping the second half), and squaring each of its elements.

\n

Formally - we got eval_domain by taking:

$$w, w\\cdot h, w\\cdot h^2, …, w\\cdot h^{8191}$$

\n

The next layer will therefore be:

$$w^2, (w\\cdot h)^2, (w\\cdot h^2)^2, …, (w\\cdot h^{4095})^2$$

\n

Note that taking the squares of the second half of each elements in eval_domain yields exactly
the same result as taking the squares of the first half. This is true for the next layers as well.

\n

Similarly, the domain of the third layer will be:

$$w^4, (w\\cdot h)^4, (w\\cdot h^2)^4, …, (w\\cdot h^{2047})^4$$

\n

And so on.

\n

FRI Folding Operator

The first FRI polynomial is simply the composition polynomial, i.e., cp.

Each subsequent FRI polynomial is obtained by:

\n
    \n
  1. Getting a random field element \\(\\beta\\) (by calling Channel.receive_random_field_element).
    2. \n
  2. Multiplying the odd coefficients of the previous polynomial by \\(\\beta\\).
    3. \n
  3. Summing together consecutive pairs (even-odd) of coefficients.
    4. \n
\n

Formally, let’s say that the k-th polynomial is of degree \\(< m\\) (for some \\(m\\) which is a power of 2):

\n

$$p_{k}(x) := \\sum _{i=0} ^{m-1} c_i x^i$$

\n

Then the (k+1)-th polynomial, whose degree is \\(< \\frac m 2 \\) will be:

\n

\\[ p_{k+1}(x) := \\sum_{i=0} ^{ m / 2 - 1 } (c_{2i} + \\beta \\cdot c_{2i + 1}) x^i \\]

\n

\"fri\"

\n

\"fri

\n

Generating FRI Commitments

We have now developed the tools to write the FriCommit method, that contains the main FRI commitment loop.

\n

It takes the following 5 arguments:

\n
    \n
  1. The composition polynomial, that is also the first FRI polynomial, that is - cp.
    2. \n
  2. The coset of order 8192 that is also the first FRI domain, that is - eval_domain.
    3. \n
  3. The evaluation of the former over the latter, which is also the first FRI layer , that is - cp_eval.
    4. \n
  4. The first Merkle tree (we will have one for each FRI layer) constructed from these evaluations, that is - cp_merkle.
    5. \n
  5. A channel object, that is channel.
    6. \n
\n

The method accordingly returns 4 lists:

\n
    \n
  1. The FRI polynomials.
    2. \n
  2. The FRI domains.
    3. \n
  3. The FRI layers.
    4. \n
  4. The FRI Merkle trees.
    5. \n
\n

The method contains a loop, in each iteration of which we extend these four lists, using the last element in each.
The iteration should stop once the last FRI polynomial is of degree 0, that is - when the last FRI polynomial is just a constant. It should then send over the channel this constant (i.e. - the polynomial’s free term).
The Channel class only supports sending strings, so make sure you convert anything you wish to send over the channel to a string before sending.

\n

\"commitment\"

\n

Query Phase

Get q random elements, provide a valdiation data for each

\n

\"decommitment\"

\n

references

\n"},{"title":"understanding plonk","date":"2023-11-01T09:19:04.000Z","_content":"\n\n\n\n# introduction\n[PLONK](https://eprint.iacr.org/2019/953), standing for the unwieldy quasi-backronym \"Permutations over Lagrange-bases for Oecumenical Noninteractive arguments of Knowledge\". PLONK still requires a \"universal and updateable\" trusted setup, meaning each program share the same setup. Secondly, there is a way for multiple parties to participate in the trusted setup such that it is secure as long as any one of them is honest.\n\n\n# preliminary\n## observation 1\na key fact: for non-zero \\\\( f \\in \\mathbb{F_p}^{\\le d}[X]\\\\)
\nfor \\\\( r \\leftarrow \\mathbb{F_p}: Pr[f(r) = 0] \\le d/p\\\\)\nunderstanding: polynomial \\\\(f\\\\) contains at most \\\\(d\\\\) roots of zeros.
\nsuppose \\\\( p \\approx 2^{256}\\\\) and \\\\( d \\approx 2^{40}\\\\) then \\\\(d/p\\\\) is negligible.
\nTherefore, for random \\\\( r \\leftarrow \\mathbb{F_p}\\\\) if \\\\(f(r) = 0\\\\) then \\\\(f\\\\) is identically zero w.h.p (with high probability)\n\n=> a simple zero test for a committed polynomial\n**Note** SZDL lemma: this also holds for multivariate polynomials (where d is total degree of f) \n\n## observation 2\nlet \\\\(f,g \\in \\mathbb{F_p}^{\\le d}[X]\\\\).\nfor \\\\( r \\leftarrow \\mathbb{F_p}\\\\), if \\\\(f(r) = g(r)\\\\), then \\\\(f = g\\\\) w.h.p
\n\\\\(f(r)-g(r)=0\\\\) => \\\\(f-g=0\\\\) w.h.p
\n\n=> a simple equality test for two committed polynomials\n\n# useful proof gadgets\n## 1. zero test\nlet \\\\( \\omega \\in \\mathbb{F_p} \\\\) be a primitive k-th root of unity \\\\(( \\omega ^{k} = 1)\\\\)\nset \\\\( H:= \\\\{1,\\omega,\\omega^2,\\omega^3,...,\\omega^{k-1}\\\\} \\subseteq \\mathbb{F_p} \\\\)\nlet \\\\( f \\in \\mathbb{F_p}^{\\le d}[X]\\\\) and \\\\( b, c \\in \\mathbb{F_p}\\\\) \\\\((d \\ge k)\\\\)\n\ntask: prove that \\\\(f\\\\) is identically zero on \\\\(H\\\\)\n![zero test](images/zkp/plonk/zero_test.png)\n\n**info** the box of \\\\(f\\\\) means the commitment of polynomial \\\\(f\\\\), i.e \\\\(com_f\\\\)\n\n## 2. product check\nproduct check on \\\\(\\Omega: \\quad \\prod_{a\\in \\Omega} f(a) = 1\\\\)\nSet \\\\( t \\in \\mathbb{F_p}^{\\le k}[X]\\\\) to be the degree-k polynomial:\n\\\\[ t(1) = f(\\omega^0) = f(1), \\quad t(\\omega^s) = \\prod_{i=0}^{s}f(\\omega^{i}) \\quad for \\quad s = 1,..., k-1\\\\]\nThen \n\\\\(t(\\omega) = f(1) \\cdot f(\\omega), \\quad t(\\omega^2) = f(1) \\cdot f(\\omega) \\cdot f(\\omega^2), ... \\\\)\n\\\\(t( \\omega^{k-1}) = \\prod_{a \\in \\Omega}f(a) = 1\\\\)\nand \\\\( t(\\omega \\cdot x) = t(x) \\cdot f(\\omega \\cdot x) \\\\) for all \\\\(x \\in \\Omega \\\\) (including at \\\\(x = \\omega^{k-1}\\\\))\n![prod_check_lemma](/images/zkp/plonk/prod_check_lemma.png)\n![prod_check_prove_and_verify](/images/zkp/plonk/prod_check_prove_verify.png)\n\nSame works for rational functions: \\\\( \\prod_{a \\in \\Omega}{f/g}(a) =1 \\\\)\nThe proof is similar\n\n## 3. permutation check\nlet \\\\(f,g\\\\) be polynomials in \\\\(\\mathbb{F_p}^{\\le d}[X]\\\\). Verifier has \\\\(com_f, com_g\\\\).\nProver wants to prove that \\\\((f(1),f(\\omega^1),f(\\omega^2),...,f(\\omega^{k-1})) \\in \\mathbb{F_p}^{\\le k}[X]\\\\) is a permutaion of \\\\((g(1),g(\\omega^1),g(\\omega^2),...,g(\\omega^{k-1})) \\in \\mathbb{F_p}^{\\le k}[X]\\\\)\n\n![permutation check](/images/zkp/plonk/permutation_check.png)\n\n## 4. prescribed permutation check\n![](/images/zkp/plonk/prescribed_perm_check_problem.png)\n![](/images/zkp/plonk/prescribed_perm_check_problem_quadratic.png)\n![](/images/zkp/plonk/prescribed_perm_check_problem_reduce.png)\n![](/images/zkp/plonk/prescribed_perm_check_problem_prove_verify.png)\n![](/images/zkp/plonk/prescribed_perm_check_problem_complete.png)\n\n# PLONK: a poly-IOP for a general circuit C(x,w)\n## step 1: compile circuit to a computation trace (gate fan-in = 2)\n![circuit to trace](images/zkp/plonk/plonk_circuit_to_trace.png)\n\nand encode the trace as polynomial\nlet \\\\(|C|\\\\) be the total number of gates, \\\\(|I| := |I_x| + |I_w|\\\\) be total number of inputs, where \\\\(|I_x|\\\\) is the number of public inputs, \\\\(I_w\\\\) is the number of private inputs.\nLet \\\\(d=3*|C|+|I|\\\\) and \\\\( \\Omega=\\\\{1,\\omega,\\omega^2,\\omega^3,...,\\omega^{d-1}\\\\} \\\\)\n\nprover interpolates \\\\( T \\in \\mathbb{F_p}^{\\le d}[X]\\\\) such that\n- **T encodes all inputs**: \\\\( T(\\omega^{-j}) \\\\)= input #j, for j = 1,...,|I|\n- **T encodes all wires**: \n\\\\(LeftInput=f(\\omega^{3l})\\\\), \\\\( RightInput=f(\\omega^{3l+1})\\\\), \\\\(Output=f(\\omega^{3l+2})\\\\), for \\\\(l = 0,1,..., |C| -1\\\\)\nFor the example,\n**inputs**\n\\\\(x_1= 5 = T(\\omega^9)\\\\), \\\\(x_2= 6 = T(\\omega^{10})\\\\), and \\\\(w_1 = 1=T(\\omega^{11})\\\\)\n**wires**\n\\\\(5=T(\\omega^0)\\\\), \\\\(6=T(\\omega^{1})\\\\), and \\\\(11=T(\\omega^{2})\\\\)\n\\\\(6=T(\\omega^3)\\\\), \\\\(1=T(\\omega^{4})\\\\), and \\\\(7=T(\\omega^{5})\\\\)\n\\\\(11=T(\\omega^6)\\\\), \\\\(7=T(\\omega^{7})\\\\), and \\\\(77=T(\\omega^{8})\\\\)\n\n\n## step 2: proving validity of T\nProver needs to prove 4 things\n1. **\\\\(T(x)\\\\) encodes the correct public inputs**\nBoth prover and verifier interpolate a polynomial \\\\( v(x) \\in \\mathbb{F_p}^{\\le |I_x|}[X]\\\\)\nthat encodes the \\\\(x\\\\)-inputs to the circuit:\n\\\\(v(\\omega^{-j}) =\\\\) input #j, for \\\\(j = 1, ..., |I_x|\\\\)\nIn our example, \\\\(v(\\omega^{-1} = 5), v(\\omega^{-2} = 6)\\\\)\nLet \\\\( \\Omega_{inp}=\\\\{\\omega^{-1},\\omega^{-2},...,\\omega^{-|I_x|}\\\\} \\\\)\nProver proves by using a **ZeroTest** on \\\\(\\Omega_inp\\\\) to prove that\n\\\\[T(y) - v(y) =0 \\quad \\forall y \\in \\Omega_{inp}\\\\]\n2. **every gate is evaluated correctly**\n**Idea** encode gate types using a selector polynomial \\\\(S(X)\\\\)\ndefine \\\\(S(X) \\in \\mathbb{F_p}^{\\le d}[X]\\\\) such that \\\\( \\forall l = 0, ..., |C| -1\\\\):\n- \\\\(S(\\omega^{3l}) =1\\\\) if gate #l is an addition gate\n- \\\\(S(\\omega^{3l}) =0\\\\) if gate #l is a multiplication gate\n\nThen, \\\\( \\forall y \\in \\Omega_{gates} : = \\\\{1,\\omega^{3},\\omega^{6},...,\\omega^{3(|C|-1)}\\\\} \\\\)\n\\\\(S(y) \\cdot [T(y) + T(\\omega y)] + (1-S(y))\\cdot T(y) \\cdot T(\\omega y) = T(\\omega^2 y)\\\\)\n![gate_evaluation_zero_test](/images/zkp/plonk/gate_evaluation_zero_test.png)\n\n3. **the wiring is implemented correctly (coppy constraint)**\n![](/images/zkp/plonk/copy_constraint_example.png)\n\n \\\\(T(\\omega^9,\\omega^0)=\\sigma(\\omega^0,\\omega^9)\\\\)\n \\\\(T(\\omega^{10},\\omega^1,\\omega^3)=\\sigma(\\omega^1,\\omega^3,\\omega^{10})\\\\)\n \\\\(T(\\omega^2,\\omega^6)=\\sigma(\\omega^6,\\omega^2)\\\\)\n \\\\(T(\\omega^{11},\\omega^4)=\\sigma(\\omega^4,\\omega^{11})\\\\)\n \\\\(T(\\omega^{5},\\omega^7)=\\sigma(\\omega^7,\\omega^{5})\\\\)\n**note**: 9 is actually -1, 10 is -2, 11 is -3\nDefine a polynomial \\\\(W: \\Omega -> \\Omega\\\\) that implemnets a rotation\n\\\\( W(\\omega^{10}, \\omega^1, \\omega^3) =(\\omega^1, \\omega^3, \\omega^{10}) \\\\), \\\\(W(\\omega^{9}, \\omega^0)=(\\omega^0, \\omega^{9})\\\\), ...\n\n**Lemma**: \\\\(\\forall y \\in \\Omega: T(y) = T(W(y))\\\\) => wire constraints are satisfied\nThis could be proved using a prescribed permutation check\n\n4. **the output of last gate is 0**\nthis is to prove \\\\(T(\\omega^8) -77 = 0\\\\)\n\n# custom gate\n![](/images/zkp/plonk/custom_gate.png)\n\\\\(u, v, w, t, r\\\\) are polynomials represent input variables (row number is the gate number). in the `Add`, `Mul` only circuits, there are only two inputs, namely `LeftInput` and `RightInput`. Hoever, here there are multiple inputs for custom gate. \n\nIn the above example, it is a constraint for \\\\( v_4 + w_3 \\cdot t_3 - t_4 = 0 \\\\)\n\n\n# plonkup\nplonkup is to ensure some values are in a pre-defined list. for example\n\n| x1 | x2 | x3 | Output |\n| --- | --- | --- | --- |\n| \\\\(a_{1,1}\\\\) | \\\\(a_{1,2}\\\\) | \\\\(a_{1,3}\\\\) | \\\\(a_{1,4}\\\\) |\n| \\\\(a_{2,1}\\\\) | \\\\(a_{2,2}\\\\) | \\\\(a_{2,3}\\\\) | \\\\(a_{2,4}\\\\) |\n| ... | ... | ... | ... |\n| \\\\(a_{n,1}\\\\) | \\\\(a_{n,2}\\\\) | \\\\(a_{n,3}\\\\) | \\\\(a_{n,4}\\\\) |\n\n\n\n\\\\(n\\\\) is gate number. the task is to prove a vector \n## references\n- https://hackmd.io/@learn-zkp/note-plonk-family\n- [ZKP MOOC Lecture 5: The Plonk SNARK](https://www.youtube.com/watch?v=A0oZVEXav24)\n- [CS251.stanford lecture](https://cs251.stanford.edu/lectures/lecture15.pdf)\n\n\n","source":"_posts/cryptography/zkp/understanding-plonk.md","raw":"---\ntitle: understanding plonk\ndate: 2023-11-01 17:19:04\ntags: [cryptography,zkp]\n---\n\n\n\n\n# introduction\n[PLONK](https://eprint.iacr.org/2019/953), standing for the unwieldy quasi-backronym \"Permutations over Lagrange-bases for Oecumenical Noninteractive arguments of Knowledge\". PLONK still requires a \"universal and updateable\" trusted setup, meaning each program share the same setup. Secondly, there is a way for multiple parties to participate in the trusted setup such that it is secure as long as any one of them is honest.\n\n\n# preliminary\n## observation 1\na key fact: for non-zero \\\\( f \\in \\mathbb{F_p}^{\\le d}[X]\\\\)
\nfor \\\\( r \\leftarrow \\mathbb{F_p}: Pr[f(r) = 0] \\le d/p\\\\)\nunderstanding: polynomial \\\\(f\\\\) contains at most \\\\(d\\\\) roots of zeros.
\nsuppose \\\\( p \\approx 2^{256}\\\\) and \\\\( d \\approx 2^{40}\\\\) then \\\\(d/p\\\\) is negligible.
\nTherefore, for random \\\\( r \\leftarrow \\mathbb{F_p}\\\\) if \\\\(f(r) = 0\\\\) then \\\\(f\\\\) is identically zero w.h.p (with high probability)\n\n=> a simple zero test for a committed polynomial\n**Note** SZDL lemma: this also holds for multivariate polynomials (where d is total degree of f) \n\n## observation 2\nlet \\\\(f,g \\in \\mathbb{F_p}^{\\le d}[X]\\\\).\nfor \\\\( r \\leftarrow \\mathbb{F_p}\\\\), if \\\\(f(r) = g(r)\\\\), then \\\\(f = g\\\\) w.h.p
\n\\\\(f(r)-g(r)=0\\\\) => \\\\(f-g=0\\\\) w.h.p
\n\n=> a simple equality test for two committed polynomials\n\n# useful proof gadgets\n## 1. zero test\nlet \\\\( \\omega \\in \\mathbb{F_p} \\\\) be a primitive k-th root of unity \\\\(( \\omega ^{k} = 1)\\\\)\nset \\\\( H:= \\\\{1,\\omega,\\omega^2,\\omega^3,...,\\omega^{k-1}\\\\} \\subseteq \\mathbb{F_p} \\\\)\nlet \\\\( f \\in \\mathbb{F_p}^{\\le d}[X]\\\\) and \\\\( b, c \\in \\mathbb{F_p}\\\\) \\\\((d \\ge k)\\\\)\n\ntask: prove that \\\\(f\\\\) is identically zero on \\\\(H\\\\)\n![zero test](images/zkp/plonk/zero_test.png)\n\n**info** the box of \\\\(f\\\\) means the commitment of polynomial \\\\(f\\\\), i.e \\\\(com_f\\\\)\n\n## 2. product check\nproduct check on \\\\(\\Omega: \\quad \\prod_{a\\in \\Omega} f(a) = 1\\\\)\nSet \\\\( t \\in \\mathbb{F_p}^{\\le k}[X]\\\\) to be the degree-k polynomial:\n\\\\[ t(1) = f(\\omega^0) = f(1), \\quad t(\\omega^s) = \\prod_{i=0}^{s}f(\\omega^{i}) \\quad for \\quad s = 1,..., k-1\\\\]\nThen \n\\\\(t(\\omega) = f(1) \\cdot f(\\omega), \\quad t(\\omega^2) = f(1) \\cdot f(\\omega) \\cdot f(\\omega^2), ... \\\\)\n\\\\(t( \\omega^{k-1}) = \\prod_{a \\in \\Omega}f(a) = 1\\\\)\nand \\\\( t(\\omega \\cdot x) = t(x) \\cdot f(\\omega \\cdot x) \\\\) for all \\\\(x \\in \\Omega \\\\) (including at \\\\(x = \\omega^{k-1}\\\\))\n![prod_check_lemma](/images/zkp/plonk/prod_check_lemma.png)\n![prod_check_prove_and_verify](/images/zkp/plonk/prod_check_prove_verify.png)\n\nSame works for rational functions: \\\\( \\prod_{a \\in \\Omega}{f/g}(a) =1 \\\\)\nThe proof is similar\n\n## 3. permutation check\nlet \\\\(f,g\\\\) be polynomials in \\\\(\\mathbb{F_p}^{\\le d}[X]\\\\). Verifier has \\\\(com_f, com_g\\\\).\nProver wants to prove that \\\\((f(1),f(\\omega^1),f(\\omega^2),...,f(\\omega^{k-1})) \\in \\mathbb{F_p}^{\\le k}[X]\\\\) is a permutaion of \\\\((g(1),g(\\omega^1),g(\\omega^2),...,g(\\omega^{k-1})) \\in \\mathbb{F_p}^{\\le k}[X]\\\\)\n\n![permutation check](/images/zkp/plonk/permutation_check.png)\n\n## 4. prescribed permutation check\n![](/images/zkp/plonk/prescribed_perm_check_problem.png)\n![](/images/zkp/plonk/prescribed_perm_check_problem_quadratic.png)\n![](/images/zkp/plonk/prescribed_perm_check_problem_reduce.png)\n![](/images/zkp/plonk/prescribed_perm_check_problem_prove_verify.png)\n![](/images/zkp/plonk/prescribed_perm_check_problem_complete.png)\n\n# PLONK: a poly-IOP for a general circuit C(x,w)\n## step 1: compile circuit to a computation trace (gate fan-in = 2)\n![circuit to trace](images/zkp/plonk/plonk_circuit_to_trace.png)\n\nand encode the trace as polynomial\nlet \\\\(|C|\\\\) be the total number of gates, \\\\(|I| := |I_x| + |I_w|\\\\) be total number of inputs, where \\\\(|I_x|\\\\) is the number of public inputs, \\\\(I_w\\\\) is the number of private inputs.\nLet \\\\(d=3*|C|+|I|\\\\) and \\\\( \\Omega=\\\\{1,\\omega,\\omega^2,\\omega^3,...,\\omega^{d-1}\\\\} \\\\)\n\nprover interpolates \\\\( T \\in \\mathbb{F_p}^{\\le d}[X]\\\\) such that\n- **T encodes all inputs**: \\\\( T(\\omega^{-j}) \\\\)= input #j, for j = 1,...,|I|\n- **T encodes all wires**: \n\\\\(LeftInput=f(\\omega^{3l})\\\\), \\\\( RightInput=f(\\omega^{3l+1})\\\\), \\\\(Output=f(\\omega^{3l+2})\\\\), for \\\\(l = 0,1,..., |C| -1\\\\)\nFor the example,\n**inputs**\n\\\\(x_1= 5 = T(\\omega^9)\\\\), \\\\(x_2= 6 = T(\\omega^{10})\\\\), and \\\\(w_1 = 1=T(\\omega^{11})\\\\)\n**wires**\n\\\\(5=T(\\omega^0)\\\\), \\\\(6=T(\\omega^{1})\\\\), and \\\\(11=T(\\omega^{2})\\\\)\n\\\\(6=T(\\omega^3)\\\\), \\\\(1=T(\\omega^{4})\\\\), and \\\\(7=T(\\omega^{5})\\\\)\n\\\\(11=T(\\omega^6)\\\\), \\\\(7=T(\\omega^{7})\\\\), and \\\\(77=T(\\omega^{8})\\\\)\n\n\n## step 2: proving validity of T\nProver needs to prove 4 things\n1. **\\\\(T(x)\\\\) encodes the correct public inputs**\nBoth prover and verifier interpolate a polynomial \\\\( v(x) \\in \\mathbb{F_p}^{\\le |I_x|}[X]\\\\)\nthat encodes the \\\\(x\\\\)-inputs to the circuit:\n\\\\(v(\\omega^{-j}) =\\\\) input #j, for \\\\(j = 1, ..., |I_x|\\\\)\nIn our example, \\\\(v(\\omega^{-1} = 5), v(\\omega^{-2} = 6)\\\\)\nLet \\\\( \\Omega_{inp}=\\\\{\\omega^{-1},\\omega^{-2},...,\\omega^{-|I_x|}\\\\} \\\\)\nProver proves by using a **ZeroTest** on \\\\(\\Omega_inp\\\\) to prove that\n\\\\[T(y) - v(y) =0 \\quad \\forall y \\in \\Omega_{inp}\\\\]\n2. **every gate is evaluated correctly**\n**Idea** encode gate types using a selector polynomial \\\\(S(X)\\\\)\ndefine \\\\(S(X) \\in \\mathbb{F_p}^{\\le d}[X]\\\\) such that \\\\( \\forall l = 0, ..., |C| -1\\\\):\n- \\\\(S(\\omega^{3l}) =1\\\\) if gate #l is an addition gate\n- \\\\(S(\\omega^{3l}) =0\\\\) if gate #l is a multiplication gate\n\nThen, \\\\( \\forall y \\in \\Omega_{gates} : = \\\\{1,\\omega^{3},\\omega^{6},...,\\omega^{3(|C|-1)}\\\\} \\\\)\n\\\\(S(y) \\cdot [T(y) + T(\\omega y)] + (1-S(y))\\cdot T(y) \\cdot T(\\omega y) = T(\\omega^2 y)\\\\)\n![gate_evaluation_zero_test](/images/zkp/plonk/gate_evaluation_zero_test.png)\n\n3. **the wiring is implemented correctly (coppy constraint)**\n![](/images/zkp/plonk/copy_constraint_example.png)\n\n \\\\(T(\\omega^9,\\omega^0)=\\sigma(\\omega^0,\\omega^9)\\\\)\n \\\\(T(\\omega^{10},\\omega^1,\\omega^3)=\\sigma(\\omega^1,\\omega^3,\\omega^{10})\\\\)\n \\\\(T(\\omega^2,\\omega^6)=\\sigma(\\omega^6,\\omega^2)\\\\)\n \\\\(T(\\omega^{11},\\omega^4)=\\sigma(\\omega^4,\\omega^{11})\\\\)\n \\\\(T(\\omega^{5},\\omega^7)=\\sigma(\\omega^7,\\omega^{5})\\\\)\n**note**: 9 is actually -1, 10 is -2, 11 is -3\nDefine a polynomial \\\\(W: \\Omega -> \\Omega\\\\) that implemnets a rotation\n\\\\( W(\\omega^{10}, \\omega^1, \\omega^3) =(\\omega^1, \\omega^3, \\omega^{10}) \\\\), \\\\(W(\\omega^{9}, \\omega^0)=(\\omega^0, \\omega^{9})\\\\), ...\n\n**Lemma**: \\\\(\\forall y \\in \\Omega: T(y) = T(W(y))\\\\) => wire constraints are satisfied\nThis could be proved using a prescribed permutation check\n\n4. **the output of last gate is 0**\nthis is to prove \\\\(T(\\omega^8) -77 = 0\\\\)\n\n# custom gate\n![](/images/zkp/plonk/custom_gate.png)\n\\\\(u, v, w, t, r\\\\) are polynomials represent input variables (row number is the gate number). in the `Add`, `Mul` only circuits, there are only two inputs, namely `LeftInput` and `RightInput`. Hoever, here there are multiple inputs for custom gate. \n\nIn the above example, it is a constraint for \\\\( v_4 + w_3 \\cdot t_3 - t_4 = 0 \\\\)\n\n\n# plonkup\nplonkup is to ensure some values are in a pre-defined list. for example\n\n| x1 | x2 | x3 | Output |\n| --- | --- | --- | --- |\n| \\\\(a_{1,1}\\\\) | \\\\(a_{1,2}\\\\) | \\\\(a_{1,3}\\\\) | \\\\(a_{1,4}\\\\) |\n| \\\\(a_{2,1}\\\\) | \\\\(a_{2,2}\\\\) | \\\\(a_{2,3}\\\\) | \\\\(a_{2,4}\\\\) |\n| ... | ... | ... | ... |\n| \\\\(a_{n,1}\\\\) | \\\\(a_{n,2}\\\\) | \\\\(a_{n,3}\\\\) | \\\\(a_{n,4}\\\\) |\n\n\n\n\\\\(n\\\\) is gate number. the task is to prove a vector \n## references\n- https://hackmd.io/@learn-zkp/note-plonk-family\n- [ZKP MOOC Lecture 5: The Plonk SNARK](https://www.youtube.com/watch?v=A0oZVEXav24)\n- [CS251.stanford lecture](https://cs251.stanford.edu/lectures/lecture15.pdf)\n\n\n","slug":"cryptography/zkp/understanding-plonk","published":1,"updated":"2023-11-29T03:24:45.714Z","_id":"clphpvdok000baj7u3i3te2rg","comments":1,"layout":"post","photos":[],"link":"","content":"\n\n\n

introduction

PLONK, standing for the unwieldy quasi-backronym “Permutations over Lagrange-bases for Oecumenical Noninteractive arguments of Knowledge”. PLONK still requires a “universal and updateable” trusted setup, meaning each program share the same setup. Secondly, there is a way for multiple parties to participate in the trusted setup such that it is secure as long as any one of them is honest.

\n

preliminary

observation 1

a key fact: for non-zero \\( f \\in \\mathbb{F_p}^{\\le d}[X]\\)

for \\( r \\leftarrow \\mathbb{F_p}: Pr[f(r) = 0] \\le d/p\\)
understanding: polynomial \\(f\\) contains at most \\(d\\) roots of zeros.

suppose \\( p \\approx 2^{256}\\) and \\( d \\approx 2^{40}\\) then \\(d/p\\) is negligible.

Therefore, for random \\( r \\leftarrow \\mathbb{F_p}\\) if \\(f(r) = 0\\) then \\(f\\) is identically zero w.h.p (with high probability)

\n

=> a simple zero test for a committed polynomial
Note SZDL lemma: this also holds for multivariate polynomials (where d is total degree of f)

\n

observation 2

let \\(f,g \\in \\mathbb{F_p}^{\\le d}[X]\\).
for \\( r \\leftarrow \\mathbb{F_p}\\), if \\(f(r) = g(r)\\), then \\(f = g\\) w.h.p

\\(f(r)-g(r)=0\\) => \\(f-g=0\\) w.h.p

\n

=> a simple equality test for two committed polynomials

\n

useful proof gadgets

1. zero test

let \\( \\omega \\in \\mathbb{F_p} \\) be a primitive k-th root of unity \\(( \\omega ^{k} = 1)\\)
set \\( H:= \\{1,\\omega,\\omega^2,\\omega^3,…,\\omega^{k-1}\\} \\subseteq \\mathbb{F_p} \\)
let \\( f \\in \\mathbb{F_p}^{\\le d}[X]\\) and \\( b, c \\in \\mathbb{F_p}\\) \\((d \\ge k)\\)

\n

task: prove that \\(f\\) is identically zero on \\(H\\)
\"zero

\n

info the box of \\(f\\) means the commitment of polynomial \\(f\\), i.e \\(com_f\\)

\n

2. product check

product check on \\(\\Omega: \\quad \\prod_{a\\in \\Omega} f(a) = 1\\)
Set \\( t \\in \\mathbb{F_p}^{\\le k}[X]\\) to be the degree-k polynomial:
\\[ t(1) = f(\\omega^0) = f(1), \\quad t(\\omega^s) = \\prod_{i=0}^{s}f(\\omega^{i}) \\quad for \\quad s = 1,…, k-1\\]
Then
\\(t(\\omega) = f(1) \\cdot f(\\omega), \\quad t(\\omega^2) = f(1) \\cdot f(\\omega) \\cdot f(\\omega^2), … \\)
\\(t( \\omega^{k-1}) = \\prod_{a \\in \\Omega}f(a) = 1\\)
and \\( t(\\omega \\cdot x) = t(x) \\cdot f(\\omega \\cdot x) \\) for all \\(x \\in \\Omega \\) (including at \\(x = \\omega^{k-1}\\))
\"prod_check_lemma\"
\"prod_check_prove_and_verify\"

\n

Same works for rational functions: \\( \\prod_{a \\in \\Omega}{f/g}(a) =1 \\)
The proof is similar

\n

3. permutation check

let \\(f,g\\) be polynomials in \\(\\mathbb{F_p}^{\\le d}[X]\\). Verifier has \\(com_f, com_g\\).
Prover wants to prove that \\((f(1),f(\\omega^1),f(\\omega^2),…,f(\\omega^{k-1})) \\in \\mathbb{F_p}^{\\le k}[X]\\) is a permutaion of \\((g(1),g(\\omega^1),g(\\omega^2),…,g(\\omega^{k-1})) \\in \\mathbb{F_p}^{\\le k}[X]\\)

\n

\"permutation

\n

4. prescribed permutation check





\n

PLONK: a poly-IOP for a general circuit C(x,w)

step 1: compile circuit to a computation trace (gate fan-in = 2)

\"circuit

\n

and encode the trace as polynomial
let \\(|C|\\) be the total number of gates, \\(|I| := |I_x| + |I_w|\\) be total number of inputs, where \\(|I_x|\\) is the number of public inputs, \\(I_w\\) is the number of private inputs.
Let \\(d=3*|C|+|I|\\) and \\( \\Omega=\\{1,\\omega,\\omega^2,\\omega^3,…,\\omega^{d-1}\\} \\)

\n

prover interpolates \\( T \\in \\mathbb{F_p}^{\\le d}[X]\\) such that

\n
    \n
  • T encodes all inputs: \\( T(\\omega^{-j}) \\)= input #j, for j = 1,…,|I|
    • \n
  • T encodes all wires:
    \\(LeftInput=f(\\omega^{3l})\\), \\( RightInput=f(\\omega^{3l+1})\\), \\(Output=f(\\omega^{3l+2})\\), for \\(l = 0,1,…, |C| -1\\)
    For the example,
    inputs
    \\(x_1= 5 = T(\\omega^9)\\), \\(x_2= 6 = T(\\omega^{10})\\), and \\(w_1 = 1=T(\\omega^{11})\\)
    wires
    \\(5=T(\\omega^0)\\), \\(6=T(\\omega^{1})\\), and \\(11=T(\\omega^{2})\\)
    \\(6=T(\\omega^3)\\), \\(1=T(\\omega^{4})\\), and \\(7=T(\\omega^{5})\\)
    \\(11=T(\\omega^6)\\), \\(7=T(\\omega^{7})\\), and \\(77=T(\\omega^{8})\\)
    • \n
\n

step 2: proving validity of T

Prover needs to prove 4 things

\n
    \n
  1. \\(T(x)\\) encodes the correct public inputs
    Both prover and verifier interpolate a polynomial \\( v(x) \\in \\mathbb{F_p}^{\\le |I_x|}[X]\\)
    that encodes the \\(x\\)-inputs to the circuit:
    \\(v(\\omega^{-j}) =\\) input #j, for \\(j = 1, …, |I_x|\\)
    In our example, \\(v(\\omega^{-1} = 5), v(\\omega^{-2} = 6)\\)
    Let \\( \\Omega_{inp}=\\{\\omega^{-1},\\omega^{-2},…,\\omega^{-|I_x|}\\} \\)
    Prover proves by using a ZeroTest on \\(\\Omega_inp\\) to prove that
    \\[T(y) - v(y) =0 \\quad \\forall y \\in \\Omega_{inp}\\]
    2. \n
  2. every gate is evaluated correctly
    Idea encode gate types using a selector polynomial \\(S(X)\\)
    define \\(S(X) \\in \\mathbb{F_p}^{\\le d}[X]\\) such that \\( \\forall l = 0, …, |C| -1\\):
    3. \n
\n
    \n
  • \\(S(\\omega^{3l}) =1\\) if gate #l is an addition gate
    • \n
  • \\(S(\\omega^{3l}) =0\\) if gate #l is a multiplication gate
    • \n
\n

Then, \\( \\forall y \\in \\Omega_{gates} : = \\{1,\\omega^{3},\\omega^{6},…,\\omega^{3(|C|-1)}\\} \\)
\\(S(y) \\cdot [T(y) + T(\\omega y)] + (1-S(y))\\cdot T(y) \\cdot T(\\omega y) = T(\\omega^2 y)\\)
\"gate_evaluation_zero_test\"

\n
    \n
  1. the wiring is implemented correctly (coppy constraint)
    2. \n
\n

\\(T(\\omega^9,\\omega^0)=\\sigma(\\omega^0,\\omega^9)\\)
\\(T(\\omega^{10},\\omega^1,\\omega^3)=\\sigma(\\omega^1,\\omega^3,\\omega^{10})\\)
\\(T(\\omega^2,\\omega^6)=\\sigma(\\omega^6,\\omega^2)\\)
\\(T(\\omega^{11},\\omega^4)=\\sigma(\\omega^4,\\omega^{11})\\)
\\(T(\\omega^{5},\\omega^7)=\\sigma(\\omega^7,\\omega^{5})\\)
note: 9 is actually -1, 10 is -2, 11 is -3
Define a polynomial \\(W: \\Omega -> \\Omega\\) that implemnets a rotation
\\( W(\\omega^{10}, \\omega^1, \\omega^3) =(\\omega^1, \\omega^3, \\omega^{10}) \\), \\(W(\\omega^{9}, \\omega^0)=(\\omega^0, \\omega^{9})\\), …

\n

Lemma: \\(\\forall y \\in \\Omega: T(y) = T(W(y))\\) => wire constraints are satisfied
This could be proved using a prescribed permutation check

\n
    \n
  1. the output of last gate is 0
    this is to prove \\(T(\\omega^8) -77 = 0\\)
    2. \n
\n

custom gate


\\(u, v, w, t, r\\) are polynomials represent input variables (row number is the gate number). in the Add, Mul only circuits, there are only two inputs, namely LeftInput and RightInput. Hoever, here there are multiple inputs for custom gate.

\n

In the above example, it is a constraint for \\( v_4 + w_3 \\cdot t_3 - t_4 = 0 \\)

\n

plonkup

plonkup is to ensure some values are in a pre-defined list. for example

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
x1x2x3Output
\\(a_{1,1}\\)\\(a_{1,2}\\)\\(a_{1,3}\\)\\(a_{1,4}\\)
\\(a_{2,1}\\)\\(a_{2,2}\\)\\(a_{2,3}\\)\\(a_{2,4}\\)
…………
\\(a_{n,1}\\)\\(a_{n,2}\\)\\(a_{n,3}\\)\\(a_{n,4}\\)
\n

\\(n\\) is gate number. the task is to prove a vector

\n

references

\n","site":{"data":{}},"excerpt":"","more":"\n\n\n

introduction

PLONK, standing for the unwieldy quasi-backronym “Permutations over Lagrange-bases for Oecumenical Noninteractive arguments of Knowledge”. PLONK still requires a “universal and updateable” trusted setup, meaning each program share the same setup. Secondly, there is a way for multiple parties to participate in the trusted setup such that it is secure as long as any one of them is honest.

\n

preliminary

observation 1

a key fact: for non-zero \\( f \\in \\mathbb{F_p}^{\\le d}[X]\\)

for \\( r \\leftarrow \\mathbb{F_p}: Pr[f(r) = 0] \\le d/p\\)
understanding: polynomial \\(f\\) contains at most \\(d\\) roots of zeros.

suppose \\( p \\approx 2^{256}\\) and \\( d \\approx 2^{40}\\) then \\(d/p\\) is negligible.

Therefore, for random \\( r \\leftarrow \\mathbb{F_p}\\) if \\(f(r) = 0\\) then \\(f\\) is identically zero w.h.p (with high probability)

\n

=> a simple zero test for a committed polynomial
Note SZDL lemma: this also holds for multivariate polynomials (where d is total degree of f)

\n

observation 2

let \\(f,g \\in \\mathbb{F_p}^{\\le d}[X]\\).
for \\( r \\leftarrow \\mathbb{F_p}\\), if \\(f(r) = g(r)\\), then \\(f = g\\) w.h.p

\\(f(r)-g(r)=0\\) => \\(f-g=0\\) w.h.p

\n

=> a simple equality test for two committed polynomials

\n

useful proof gadgets

1. zero test

let \\( \\omega \\in \\mathbb{F_p} \\) be a primitive k-th root of unity \\(( \\omega ^{k} = 1)\\)
set \\( H:= \\{1,\\omega,\\omega^2,\\omega^3,…,\\omega^{k-1}\\} \\subseteq \\mathbb{F_p} \\)
let \\( f \\in \\mathbb{F_p}^{\\le d}[X]\\) and \\( b, c \\in \\mathbb{F_p}\\) \\((d \\ge k)\\)

\n

task: prove that \\(f\\) is identically zero on \\(H\\)
\"zero

\n

info the box of \\(f\\) means the commitment of polynomial \\(f\\), i.e \\(com_f\\)

\n

2. product check

product check on \\(\\Omega: \\quad \\prod_{a\\in \\Omega} f(a) = 1\\)
Set \\( t \\in \\mathbb{F_p}^{\\le k}[X]\\) to be the degree-k polynomial:
\\[ t(1) = f(\\omega^0) = f(1), \\quad t(\\omega^s) = \\prod_{i=0}^{s}f(\\omega^{i}) \\quad for \\quad s = 1,…, k-1\\]
Then
\\(t(\\omega) = f(1) \\cdot f(\\omega), \\quad t(\\omega^2) = f(1) \\cdot f(\\omega) \\cdot f(\\omega^2), … \\)
\\(t( \\omega^{k-1}) = \\prod_{a \\in \\Omega}f(a) = 1\\)
and \\( t(\\omega \\cdot x) = t(x) \\cdot f(\\omega \\cdot x) \\) for all \\(x \\in \\Omega \\) (including at \\(x = \\omega^{k-1}\\))
\"prod_check_lemma\"
\"prod_check_prove_and_verify\"

\n

Same works for rational functions: \\( \\prod_{a \\in \\Omega}{f/g}(a) =1 \\)
The proof is similar

\n

3. permutation check

let \\(f,g\\) be polynomials in \\(\\mathbb{F_p}^{\\le d}[X]\\). Verifier has \\(com_f, com_g\\).
Prover wants to prove that \\((f(1),f(\\omega^1),f(\\omega^2),…,f(\\omega^{k-1})) \\in \\mathbb{F_p}^{\\le k}[X]\\) is a permutaion of \\((g(1),g(\\omega^1),g(\\omega^2),…,g(\\omega^{k-1})) \\in \\mathbb{F_p}^{\\le k}[X]\\)

\n

\"permutation

\n

4. prescribed permutation check





\n

PLONK: a poly-IOP for a general circuit C(x,w)

step 1: compile circuit to a computation trace (gate fan-in = 2)

\"circuit

\n

and encode the trace as polynomial
let \\(|C|\\) be the total number of gates, \\(|I| := |I_x| + |I_w|\\) be total number of inputs, where \\(|I_x|\\) is the number of public inputs, \\(I_w\\) is the number of private inputs.
Let \\(d=3*|C|+|I|\\) and \\( \\Omega=\\{1,\\omega,\\omega^2,\\omega^3,…,\\omega^{d-1}\\} \\)

\n

prover interpolates \\( T \\in \\mathbb{F_p}^{\\le d}[X]\\) such that

\n
    \n
  • T encodes all inputs: \\( T(\\omega^{-j}) \\)= input #j, for j = 1,…,|I|
    • \n
  • T encodes all wires:
    \\(LeftInput=f(\\omega^{3l})\\), \\( RightInput=f(\\omega^{3l+1})\\), \\(Output=f(\\omega^{3l+2})\\), for \\(l = 0,1,…, |C| -1\\)
    For the example,
    inputs
    \\(x_1= 5 = T(\\omega^9)\\), \\(x_2= 6 = T(\\omega^{10})\\), and \\(w_1 = 1=T(\\omega^{11})\\)
    wires
    \\(5=T(\\omega^0)\\), \\(6=T(\\omega^{1})\\), and \\(11=T(\\omega^{2})\\)
    \\(6=T(\\omega^3)\\), \\(1=T(\\omega^{4})\\), and \\(7=T(\\omega^{5})\\)
    \\(11=T(\\omega^6)\\), \\(7=T(\\omega^{7})\\), and \\(77=T(\\omega^{8})\\)
    • \n
\n

step 2: proving validity of T

Prover needs to prove 4 things

\n
    \n
  1. \\(T(x)\\) encodes the correct public inputs
    Both prover and verifier interpolate a polynomial \\( v(x) \\in \\mathbb{F_p}^{\\le |I_x|}[X]\\)
    that encodes the \\(x\\)-inputs to the circuit:
    \\(v(\\omega^{-j}) =\\) input #j, for \\(j = 1, …, |I_x|\\)
    In our example, \\(v(\\omega^{-1} = 5), v(\\omega^{-2} = 6)\\)
    Let \\( \\Omega_{inp}=\\{\\omega^{-1},\\omega^{-2},…,\\omega^{-|I_x|}\\} \\)
    Prover proves by using a ZeroTest on \\(\\Omega_inp\\) to prove that
    \\[T(y) - v(y) =0 \\quad \\forall y \\in \\Omega_{inp}\\]
    2. \n
  2. every gate is evaluated correctly
    Idea encode gate types using a selector polynomial \\(S(X)\\)
    define \\(S(X) \\in \\mathbb{F_p}^{\\le d}[X]\\) such that \\( \\forall l = 0, …, |C| -1\\):
    3. \n
\n
    \n
  • \\(S(\\omega^{3l}) =1\\) if gate #l is an addition gate
    • \n
  • \\(S(\\omega^{3l}) =0\\) if gate #l is a multiplication gate
    • \n
\n

Then, \\( \\forall y \\in \\Omega_{gates} : = \\{1,\\omega^{3},\\omega^{6},…,\\omega^{3(|C|-1)}\\} \\)
\\(S(y) \\cdot [T(y) + T(\\omega y)] + (1-S(y))\\cdot T(y) \\cdot T(\\omega y) = T(\\omega^2 y)\\)
\"gate_evaluation_zero_test\"

\n
    \n
  1. the wiring is implemented correctly (coppy constraint)
    2. \n
\n

\\(T(\\omega^9,\\omega^0)=\\sigma(\\omega^0,\\omega^9)\\)
\\(T(\\omega^{10},\\omega^1,\\omega^3)=\\sigma(\\omega^1,\\omega^3,\\omega^{10})\\)
\\(T(\\omega^2,\\omega^6)=\\sigma(\\omega^6,\\omega^2)\\)
\\(T(\\omega^{11},\\omega^4)=\\sigma(\\omega^4,\\omega^{11})\\)
\\(T(\\omega^{5},\\omega^7)=\\sigma(\\omega^7,\\omega^{5})\\)
note: 9 is actually -1, 10 is -2, 11 is -3
Define a polynomial \\(W: \\Omega -> \\Omega\\) that implemnets a rotation
\\( W(\\omega^{10}, \\omega^1, \\omega^3) =(\\omega^1, \\omega^3, \\omega^{10}) \\), \\(W(\\omega^{9}, \\omega^0)=(\\omega^0, \\omega^{9})\\), …

\n

Lemma: \\(\\forall y \\in \\Omega: T(y) = T(W(y))\\) => wire constraints are satisfied
This could be proved using a prescribed permutation check

\n
    \n
  1. the output of last gate is 0
    this is to prove \\(T(\\omega^8) -77 = 0\\)
    2. \n
\n

custom gate


\\(u, v, w, t, r\\) are polynomials represent input variables (row number is the gate number). in the Add, Mul only circuits, there are only two inputs, namely LeftInput and RightInput. Hoever, here there are multiple inputs for custom gate.

\n

In the above example, it is a constraint for \\( v_4 + w_3 \\cdot t_3 - t_4 = 0 \\)

\n

plonkup

plonkup is to ensure some values are in a pre-defined list. for example

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
x1x2x3Output
\\(a_{1,1}\\)\\(a_{1,2}\\)\\(a_{1,3}\\)\\(a_{1,4}\\)
\\(a_{2,1}\\)\\(a_{2,2}\\)\\(a_{2,3}\\)\\(a_{2,4}\\)
…………
\\(a_{n,1}\\)\\(a_{n,2}\\)\\(a_{n,3}\\)\\(a_{n,4}\\)
\n

\\(n\\) is gate number. the task is to prove a vector

\n

references

\n"},{"title":"zk bench","date":"2023-11-24T06:29:26.000Z","_content":"\n\n\n\n## references\n- [csdn blog on zkbench](https://blog.csdn.net/mutourend/article/details/134050672?spm=1001.2014.3001.5502)","source":"_posts/cryptography/zkp/zk-bench.md","raw":"---\ntitle: zk bench\ndate: 2023-11-24 14:29:26\ntags: [cryptography,zkp]\n---\n\n\n\n\n## references\n- [csdn blog on zkbench](https://blog.csdn.net/mutourend/article/details/134050672?spm=1001.2014.3001.5502)","slug":"cryptography/zkp/zk-bench","published":1,"updated":"2023-11-24T14:05:26.446Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clphpvdok000caj7ubjq53ldn","content":"\n\n\n\n

references

\n","site":{"data":{}},"excerpt":"","more":"\n\n\n\n

references

\n"},{"title":"goldilocks field","date":"2023-11-18T03:06:53.000Z","_content":"\n\n\n\n\n# properties\n\\\\( p = \\phi^2 - \\phi + 1\\\\)\n\\\\( \\epsilon = \\phi -1\\\\)\n\n## Goldilocks Addition\nlet \\\\( a = a_0 + a_1 \\cdot \\phi, b = b_0 + b_1 \\cdot \\phi\\\\)\n\n`(c, carry) = a + b`;\n- if carry = 0 && c > p, result should be `c -p`\n- if carry = 0 && c < p, result should be `c`\n- if carry = 1 \nlet \\\\( c = c_0 + c_1 \\cdot \\phi + c_2 \\cdot \\phi^2\\\\), and \\\\(c_2 = 1 \\\\). let \\\\( u = c_0 + c_1 \\cdot \\phi \\\\)\ntherefore \\\\(a + b \\mod p\\\\) should be \\\\( a + b - p = c_0 + c_1 \\cdot \\phi + \\phi^2 - (\\phi^2 - \\phi + 1) =c_0 + c_1 \\cdot \\phi + \\phi - 1 = u + \\phi - 1\\\\) \n - if \\\\( u < p\\\\), computationally, \\\\( u - p\\\\) will borrow from \\\\( \\phi^2\\\\), which is equivalent to\n \\\\( \\phi^2 + u - p = u + \\phi - 1 \\\\), which is \\\\( a - b \\mod p\\\\). therefore, the result is just \\\\( u-p \\\\)\n - if \\\\( u> p\\\\), this will unlikely occur, if u >p, given by \\\\( \\phi^2 > p \\\\), so \\\\( u + \\phi^2 > p + \\phi^2 > 2p\\\\) that means a or b is not in canonical form. **does not consider this case for now**\n\n## Goldilocks Subtraction\n\\\\(a - b \\mod MOD\\\\)\nif a < b, a - b is negative, and the binary representation is 2's complement of the negative value. therefore, it's requied to add by MOD and treat the final binary representation as positive value.\n\n## Goldilocks Multiplication\nlet \\\\(a = a_0 + a_1 \\cdot \\phi\\\\), and \\\\(b=b_0 + b_1 \\cdot \\phi\\\\)\nto compute \\\\(c = a \\cdot b\\\\)\n- first, turn it into \\\\( a_0 \\cdot b_0 +(a_1\\cdot b_0 +a_0\\cdot b_1)\\cdot \\phi + a_1 \\cdot b_1 \\phi^2 \\\\)\nit may carry to \\\\( \\phi^2 \\\\). so the results is a `u128` (`[u32;4]`)\n- second, reduce the `[u32;4]` to `u64`, details to be described in the following section.\n## plonky2 Goldilocks Fieldreduce128\n```rust\nfn reduce128(x: u128) -> GoldilocksField {\n let (x_lo, x_hi) = split(x); // This is a no-op\n let x_hi_hi = x_hi >> 32;\n let x_hi_lo = x_hi & EPSILON;\n\n let (mut t0, borrow) = x_lo.overflowing_sub(x_hi_hi);\n if borrow {\n branch_hint(); // A borrow is exceedingly rare. It is faster to branch.\n t0 -= EPSILON; // Cannot underflow.\n }\n let t1 = x_hi_lo * EPSILON;\n let t2 = unsafe { add_no_canonicalize_trashing_input(t0, t1) };\n GoldilocksField(t2)\n}\n```\n\nlet \\\\( n = n_0 + n_1 \\cdot \\phi + n_2 \\cdot \\phi^{2} + n_3 \\cdot \\phi^{3}\\\\)\nsince, \\\\(\\phi^{3} = -1\\\\), \\\\(n\\\\) could be reduced to \\\\( n = n_0 + n_1 \\cdot \\phi + n_2 \\cdot \\phi^{2} - n_3 \\\\),\nwhich is line 6 in the above code snipet. when borrow occurs, it means it borrows \\\\( \\phi^2\\\\) ( subtraction is of u64, so borrow 1 menas add \\\\( \\phi^2\\\\), `1<<64`). therefore, needs to reduce \\\\( \\phi^2\\\\). since \\\\( \\phi^2 = \\phi - 1\\\\), which is reduce by \\\\( \\phi - 1\\\\) (this is `EPSILON`). \n**Note** needs to consider line 23 could underflow\n\nnext step is to reduce \\\\( n_2 \\cdot \\phi^2 \\\\), which is just \\\\( n_2 \\cdot (\\phi -1) \\\\), which is \\\\( n_2 \\cdot EPSILON \\\\), line 11, `x_hi_lo * EPSILON`\n\n\n## Sppark reduce(uint32_t temp[4])\n```cpp\n inline void reduce(uint32_t temp[4])\n {\n uint32_t carry;\n\n asm(\"sub.cc.u32 %0, %0, %3; subc.cc.u32 %1, %1, %4; subc.u32 %2, 0, 0;\"\n : \"+r\"(temp[0]), \"+r\"(temp[1]), \"=r\"(carry)\n : \"r\"(temp[2]), \"r\"(temp[3]));\n asm(\"add.cc.u32 %0, %0, %2; addc.u32 %1, %1, %3;\"\n : \"+r\"(temp[1]), \"+r\"(carry)\n : \"r\"(temp[2]), \"r\"(temp[3]));\n\n asm(\"mad.lo.cc.u32 %0, %3, %4, %0; madc.hi.cc.u32 %1, %3, %4, %1; addc.u32 %2, 0, 0;\"\n : \"+r\"(temp[0]), \"+r\"(temp[1]), \"=r\"(temp[2])\n : \"r\"(carry), \"r\"(gl64_device::W));\n asm(\"mad.lo.cc.u32 %0, %2, %3, %0; madc.hi.u32 %1, %2, %3, %1;\"\n : \"+r\"(temp[0]), \"+r\"(temp[1])\n : \"r\"(temp[2]), \"r\"(gl64_device::W));\n\n\n asm(\"mad.lo.cc.u32 %0, %2, %3, %0; madc.hi.u32 %1, %2, %3, %1;\"\n : \"+r\"(temp[0]), \"+r\"(temp[1])\n : \"r\"(temp[2]), \"r\"(gl64_device::W));\n\n asm(\"mov.b64 %0, {%1, %2};\" : \"=l\"(val) : \"r\"(temp[0]), \"r\"(temp[1]));\n }\n\n```\n**note**: `W` is epsilon\nlet \\\\( n = n_0 + n_1 \\cdot \\phi + n_2 \\cdot \\phi^{2} + n_3 \\cdot \\phi^{3}\\\\) \n\\\\( n = n_0 + n_2\\cdot\\phi^2 + \\phi(n_1+n_3\\cdot \\phi^2)\\\\).\nsince \\\\( \\phi^2 = \\phi - 1\\\\)\n\\\\( n = n_0 + n_2\\cdot(\\phi-1) + \\phi(n_1+n_3\\cdot(\\phi-1))\\\\).\n\\\\( n = n_0 -n_2 +n_2\\phi + \\phi(n_1-n_3 + n_3\\cdot\\phi)\\\\).\n\\\\( n = n_0 -n_2 + \\phi(n_1-n_3 +n_2) + n_3\\cdot\\phi^2\\\\).\nline 5 computes `n0 - n2` and set to temp[0]; & computes `n1-n3` and sets to temp[1]\nline 8 computes `n1- n3 + n2`; add n_3 with carry and put it to carry.\n\nlet \\\\(c\\\\) be the carry, at \\\\(\\phi^2\\\\), it is actually \\\\( c \\cdot \\phi^2\\\\). it could be further reduced to \\\\(c \\cdot (\\phi - 1) = c \\cdot W\\\\)\nline 12 is to reduce the carry part to temp[0], temp[1], and temp[2].\nif temp[2] still exist. line 15 will reduce it to temp[0], temp[1] similar as above.\nat this step. only temp[0] and temp[1] will contains value and return as the result\n\n\n## appendix\n- underflow\n```\nuint64_t tmp;\nuint32_t borrow;\nasm(\"{ .reg.pred %top;\");\nasm(\"sub.cc.u64 %0, %0, %2; subc.u32 %1, 0, 0;\"\n : \"+l\"(val), \"=r\"(borrow)\n : \"l\"(b.val));\n```\nall bits of borrow will be set to 1 if underflow occurs\n\n# references\n- [The Goldilocks Prime by Remco Bloemem](https://xn--2-umb.com/22/goldilocks/)\n- [parameters of goldilocks field](https://cronokirby.com/notes/2022/09/the-goldilocks-field/)\n- [csdn blog](https://blog.csdn.net/mutourend/article/details/126407028)\n- [cuda implementation of GoldilocksNTT by yrrid](https://github.com/yrrid/GoldilocksNTT/tree/main)","source":"_posts/arithmatic/goldilocks-field.md","raw":"---\ntitle: goldilocks field\ndate: 2023-11-18 11:06:53\ntags: [arithmatic]\n---\n\n\n\n\n\n# properties\n\\\\( p = \\phi^2 - \\phi + 1\\\\)\n\\\\( \\epsilon = \\phi -1\\\\)\n\n## Goldilocks Addition\nlet \\\\( a = a_0 + a_1 \\cdot \\phi, b = b_0 + b_1 \\cdot \\phi\\\\)\n\n`(c, carry) = a + b`;\n- if carry = 0 && c > p, result should be `c -p`\n- if carry = 0 && c < p, result should be `c`\n- if carry = 1 \nlet \\\\( c = c_0 + c_1 \\cdot \\phi + c_2 \\cdot \\phi^2\\\\), and \\\\(c_2 = 1 \\\\). let \\\\( u = c_0 + c_1 \\cdot \\phi \\\\)\ntherefore \\\\(a + b \\mod p\\\\) should be \\\\( a + b - p = c_0 + c_1 \\cdot \\phi + \\phi^2 - (\\phi^2 - \\phi + 1) =c_0 + c_1 \\cdot \\phi + \\phi - 1 = u + \\phi - 1\\\\) \n - if \\\\( u < p\\\\), computationally, \\\\( u - p\\\\) will borrow from \\\\( \\phi^2\\\\), which is equivalent to\n \\\\( \\phi^2 + u - p = u + \\phi - 1 \\\\), which is \\\\( a - b \\mod p\\\\). therefore, the result is just \\\\( u-p \\\\)\n - if \\\\( u> p\\\\), this will unlikely occur, if u >p, given by \\\\( \\phi^2 > p \\\\), so \\\\( u + \\phi^2 > p + \\phi^2 > 2p\\\\) that means a or b is not in canonical form. **does not consider this case for now**\n\n## Goldilocks Subtraction\n\\\\(a - b \\mod MOD\\\\)\nif a < b, a - b is negative, and the binary representation is 2's complement of the negative value. therefore, it's requied to add by MOD and treat the final binary representation as positive value.\n\n## Goldilocks Multiplication\nlet \\\\(a = a_0 + a_1 \\cdot \\phi\\\\), and \\\\(b=b_0 + b_1 \\cdot \\phi\\\\)\nto compute \\\\(c = a \\cdot b\\\\)\n- first, turn it into \\\\( a_0 \\cdot b_0 +(a_1\\cdot b_0 +a_0\\cdot b_1)\\cdot \\phi + a_1 \\cdot b_1 \\phi^2 \\\\)\nit may carry to \\\\( \\phi^2 \\\\). so the results is a `u128` (`[u32;4]`)\n- second, reduce the `[u32;4]` to `u64`, details to be described in the following section.\n## plonky2 Goldilocks Fieldreduce128\n```rust\nfn reduce128(x: u128) -> GoldilocksField {\n let (x_lo, x_hi) = split(x); // This is a no-op\n let x_hi_hi = x_hi >> 32;\n let x_hi_lo = x_hi & EPSILON;\n\n let (mut t0, borrow) = x_lo.overflowing_sub(x_hi_hi);\n if borrow {\n branch_hint(); // A borrow is exceedingly rare. It is faster to branch.\n t0 -= EPSILON; // Cannot underflow.\n }\n let t1 = x_hi_lo * EPSILON;\n let t2 = unsafe { add_no_canonicalize_trashing_input(t0, t1) };\n GoldilocksField(t2)\n}\n```\n\nlet \\\\( n = n_0 + n_1 \\cdot \\phi + n_2 \\cdot \\phi^{2} + n_3 \\cdot \\phi^{3}\\\\)\nsince, \\\\(\\phi^{3} = -1\\\\), \\\\(n\\\\) could be reduced to \\\\( n = n_0 + n_1 \\cdot \\phi + n_2 \\cdot \\phi^{2} - n_3 \\\\),\nwhich is line 6 in the above code snipet. when borrow occurs, it means it borrows \\\\( \\phi^2\\\\) ( subtraction is of u64, so borrow 1 menas add \\\\( \\phi^2\\\\), `1<<64`). therefore, needs to reduce \\\\( \\phi^2\\\\). since \\\\( \\phi^2 = \\phi - 1\\\\), which is reduce by \\\\( \\phi - 1\\\\) (this is `EPSILON`). \n**Note** needs to consider line 23 could underflow\n\nnext step is to reduce \\\\( n_2 \\cdot \\phi^2 \\\\), which is just \\\\( n_2 \\cdot (\\phi -1) \\\\), which is \\\\( n_2 \\cdot EPSILON \\\\), line 11, `x_hi_lo * EPSILON`\n\n\n## Sppark reduce(uint32_t temp[4])\n```cpp\n inline void reduce(uint32_t temp[4])\n {\n uint32_t carry;\n\n asm(\"sub.cc.u32 %0, %0, %3; subc.cc.u32 %1, %1, %4; subc.u32 %2, 0, 0;\"\n : \"+r\"(temp[0]), \"+r\"(temp[1]), \"=r\"(carry)\n : \"r\"(temp[2]), \"r\"(temp[3]));\n asm(\"add.cc.u32 %0, %0, %2; addc.u32 %1, %1, %3;\"\n : \"+r\"(temp[1]), \"+r\"(carry)\n : \"r\"(temp[2]), \"r\"(temp[3]));\n\n asm(\"mad.lo.cc.u32 %0, %3, %4, %0; madc.hi.cc.u32 %1, %3, %4, %1; addc.u32 %2, 0, 0;\"\n : \"+r\"(temp[0]), \"+r\"(temp[1]), \"=r\"(temp[2])\n : \"r\"(carry), \"r\"(gl64_device::W));\n asm(\"mad.lo.cc.u32 %0, %2, %3, %0; madc.hi.u32 %1, %2, %3, %1;\"\n : \"+r\"(temp[0]), \"+r\"(temp[1])\n : \"r\"(temp[2]), \"r\"(gl64_device::W));\n\n\n asm(\"mad.lo.cc.u32 %0, %2, %3, %0; madc.hi.u32 %1, %2, %3, %1;\"\n : \"+r\"(temp[0]), \"+r\"(temp[1])\n : \"r\"(temp[2]), \"r\"(gl64_device::W));\n\n asm(\"mov.b64 %0, {%1, %2};\" : \"=l\"(val) : \"r\"(temp[0]), \"r\"(temp[1]));\n }\n\n```\n**note**: `W` is epsilon\nlet \\\\( n = n_0 + n_1 \\cdot \\phi + n_2 \\cdot \\phi^{2} + n_3 \\cdot \\phi^{3}\\\\) \n\\\\( n = n_0 + n_2\\cdot\\phi^2 + \\phi(n_1+n_3\\cdot \\phi^2)\\\\).\nsince \\\\( \\phi^2 = \\phi - 1\\\\)\n\\\\( n = n_0 + n_2\\cdot(\\phi-1) + \\phi(n_1+n_3\\cdot(\\phi-1))\\\\).\n\\\\( n = n_0 -n_2 +n_2\\phi + \\phi(n_1-n_3 + n_3\\cdot\\phi)\\\\).\n\\\\( n = n_0 -n_2 + \\phi(n_1-n_3 +n_2) + n_3\\cdot\\phi^2\\\\).\nline 5 computes `n0 - n2` and set to temp[0]; & computes `n1-n3` and sets to temp[1]\nline 8 computes `n1- n3 + n2`; add n_3 with carry and put it to carry.\n\nlet \\\\(c\\\\) be the carry, at \\\\(\\phi^2\\\\), it is actually \\\\( c \\cdot \\phi^2\\\\). it could be further reduced to \\\\(c \\cdot (\\phi - 1) = c \\cdot W\\\\)\nline 12 is to reduce the carry part to temp[0], temp[1], and temp[2].\nif temp[2] still exist. line 15 will reduce it to temp[0], temp[1] similar as above.\nat this step. only temp[0] and temp[1] will contains value and return as the result\n\n\n## appendix\n- underflow\n```\nuint64_t tmp;\nuint32_t borrow;\nasm(\"{ .reg.pred %top;\");\nasm(\"sub.cc.u64 %0, %0, %2; subc.u32 %1, 0, 0;\"\n : \"+l\"(val), \"=r\"(borrow)\n : \"l\"(b.val));\n```\nall bits of borrow will be set to 1 if underflow occurs\n\n# references\n- [The Goldilocks Prime by Remco Bloemem](https://xn--2-umb.com/22/goldilocks/)\n- [parameters of goldilocks field](https://cronokirby.com/notes/2022/09/the-goldilocks-field/)\n- [csdn blog](https://blog.csdn.net/mutourend/article/details/126407028)\n- [cuda implementation of GoldilocksNTT by yrrid](https://github.com/yrrid/GoldilocksNTT/tree/main)","slug":"arithmatic/goldilocks-field","published":1,"updated":"2023-11-28T09:44:37.469Z","_id":"clpi1wlcr0000067ufpzn63zv","comments":1,"layout":"post","photos":[],"link":"","content":"\n\n\n

properties

\\( p = \\phi^2 - \\phi + 1\\)
\\( \\epsilon = \\phi -1\\)

\n

Goldilocks Addition

let \\( a = a_0 + a_1 \\cdot \\phi, b = b_0 + b_1 \\cdot \\phi\\)

\n

(c, carry) = a + b;

\n
    \n
  • if carry = 0 && c > p, result should be c -p
    • \n
  • if carry = 0 && c < p, result should be c
    • \n
  • if carry = 1
    let \\( c = c_0 + c_1 \\cdot \\phi + c_2 \\cdot \\phi^2\\), and \\(c_2 = 1 \\). let \\( u = c_0 + c_1 \\cdot \\phi \\)
    therefore \\(a + b \\mod p\\) should be \\( a + b - p = c_0 + c_1 \\cdot \\phi + \\phi^2 - (\\phi^2 - \\phi + 1) =c_0 + c_1 \\cdot \\phi + \\phi - 1 = u + \\phi - 1\\)
      \n
    • if \\( u < p\\), computationally, \\( u - p\\) will borrow from \\( \\phi^2\\), which is equivalent to
      \\( \\phi^2 + u - p = u + \\phi - 1 \\), which is \\( a - b \\mod p\\). therefore, the result is just \\( u-p \\)
      • \n
    • if \\( u> p\\), this will unlikely occur, if u >p, given by \\( \\phi^2 > p \\), so \\( u + \\phi^2 > p + \\phi^2 > 2p\\) that means a or b is not in canonical form. does not consider this case for now
      • \n
    \n
    • \n
\n

Goldilocks Subtraction

\\(a - b \\mod MOD\\)
if a < b, a - b is negative, and the binary representation is 2’s complement of the negative value. therefore, it’s requied to add by MOD and treat the final binary representation as positive value.

\n

Goldilocks Multiplication

let \\(a = a_0 + a_1 \\cdot \\phi\\), and \\(b=b_0 + b_1 \\cdot \\phi\\)
to compute \\(c = a \\cdot b\\)

\n
    \n
  • first, turn it into \\( a_0 \\cdot b_0 +(a_1\\cdot b_0 +a_0\\cdot b_1)\\cdot \\phi + a_1 \\cdot b_1 \\phi^2 \\)
    it may carry to \\( \\phi^2 \\). so the results is a u128 ([u32;4])
    • \n
  • second, reduce the [u32;4] to u64, details to be described in the following section.
    • \n
\n

plonky2 Goldilocks Fieldreduce128

1
2
3
4
5
6
7
8
9
10
11
12
13
14
fn reduce128(x: u128) -> GoldilocksField {
    let (x_lo, x_hi) = split(x); // This is a no-op
    let x_hi_hi = x_hi >> 32;
    let x_hi_lo = x_hi & EPSILON;

    let (mut t0, borrow) = x_lo.overflowing_sub(x_hi_hi);
    if borrow {
        branch_hint(); // A borrow is exceedingly rare. It is faster to branch.
        t0 -= EPSILON; // Cannot underflow.
    }
    let t1 = x_hi_lo * EPSILON;
    let t2 = unsafe { add_no_canonicalize_trashing_input(t0, t1) };
    GoldilocksField(t2)
}
\n\n

let \\( n = n_0 + n_1 \\cdot \\phi + n_2 \\cdot \\phi^{2} + n_3 \\cdot \\phi^{3}\\)
since, \\(\\phi^{3} = -1\\), \\(n\\) could be reduced to \\( n = n_0 + n_1 \\cdot \\phi + n_2 \\cdot \\phi^{2} - n_3 \\),
which is line 6 in the above code snipet. when borrow occurs, it means it borrows \\( \\phi^2\\) ( subtraction is of u64, so borrow 1 menas add \\( \\phi^2\\), 1<<64). therefore, needs to reduce \\( \\phi^2\\). since \\( \\phi^2 = \\phi - 1\\), which is reduce by \\( \\phi - 1\\) (this is EPSILON).
Note needs to consider line 23 could underflow

\n

next step is to reduce \\( n_2 \\cdot \\phi^2 \\), which is just \\( n_2 \\cdot (\\phi -1) \\), which is \\( n_2 \\cdot EPSILON \\), line 11, x_hi_lo * EPSILON

\n

Sppark reduce(uint32_t temp[4])

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
inline void reduce(uint32_t temp[4])
{
    uint32_t carry;

    asm("sub.cc.u32 %0, %0, %3; subc.cc.u32 %1, %1, %4; subc.u32 %2, 0, 0;"
        : "+r"(temp[0]), "+r"(temp[1]), "=r"(carry)
        : "r"(temp[2]), "r"(temp[3]));
    asm("add.cc.u32 %0, %0, %2; addc.u32 %1, %1, %3;"
        : "+r"(temp[1]), "+r"(carry)
        : "r"(temp[2]), "r"(temp[3]));

    asm("mad.lo.cc.u32 %0, %3, %4, %0; madc.hi.cc.u32 %1, %3, %4, %1; addc.u32 %2, 0, 0;"
        : "+r"(temp[0]), "+r"(temp[1]), "=r"(temp[2])
        : "r"(carry), "r"(gl64_device::W));
    asm("mad.lo.cc.u32 %0, %2, %3, %0; madc.hi.u32 %1, %2, %3, %1;"
        : "+r"(temp[0]), "+r"(temp[1])
        : "r"(temp[2]), "r"(gl64_device::W));


    asm("mad.lo.cc.u32 %0, %2, %3, %0; madc.hi.u32 %1, %2, %3, %1;"
        : "+r"(temp[0]), "+r"(temp[1])
        : "r"(temp[2]), "r"(gl64_device::W));

    asm("mov.b64 %0, {%1, %2};" : "=l"(val) : "r"(temp[0]), "r"(temp[1]));
}

\n

note: W is epsilon
let \\( n = n_0 + n_1 \\cdot \\phi + n_2 \\cdot \\phi^{2} + n_3 \\cdot \\phi^{3}\\)
\\( n = n_0 + n_2\\cdot\\phi^2 + \\phi(n_1+n_3\\cdot \\phi^2)\\).
since \\( \\phi^2 = \\phi - 1\\)
\\( n = n_0 + n_2\\cdot(\\phi-1) + \\phi(n_1+n_3\\cdot(\\phi-1))\\).
\\( n = n_0 -n_2 +n_2\\phi + \\phi(n_1-n_3 + n_3\\cdot\\phi)\\).
\\( n = n_0 -n_2 + \\phi(n_1-n_3 +n_2) + n_3\\cdot\\phi^2\\).
line 5 computes n0 - n2 and set to temp[0]; & computes n1-n3 and sets to temp[1]
line 8 computes n1- n3 + n2; add n_3 with carry and put it to carry.

\n

let \\(c\\) be the carry, at \\(\\phi^2\\), it is actually \\( c \\cdot \\phi^2\\). it could be further reduced to \\(c \\cdot (\\phi - 1) = c \\cdot W\\)
line 12 is to reduce the carry part to temp[0], temp[1], and temp[2].
if temp[2] still exist. line 15 will reduce it to temp[0], temp[1] similar as above.
at this step. only temp[0] and temp[1] will contains value and return as the result

\n

appendix

    \n
  • underflow
    1
    2
    3
    4
    5
    6
    uint64_t tmp;
    uint32_t borrow;
    asm("{ .reg.pred %top;");
    asm("sub.cc.u64 %0, %0, %2; subc.u32 %1, 0, 0;"
        : "+l"(val), "=r"(borrow)
        : "l"(b.val));
    \nall bits of borrow will be set to 1 if underflow occurs
    • \n
\n

references

\n","site":{"data":{}},"excerpt":"","more":"\n\n\n

properties

\\( p = \\phi^2 - \\phi + 1\\)
\\( \\epsilon = \\phi -1\\)

\n

Goldilocks Addition

let \\( a = a_0 + a_1 \\cdot \\phi, b = b_0 + b_1 \\cdot \\phi\\)

\n

(c, carry) = a + b;

\n
    \n
  • if carry = 0 && c > p, result should be c -p
    • \n
  • if carry = 0 && c < p, result should be c
    • \n
  • if carry = 1
    let \\( c = c_0 + c_1 \\cdot \\phi + c_2 \\cdot \\phi^2\\), and \\(c_2 = 1 \\). let \\( u = c_0 + c_1 \\cdot \\phi \\)
    therefore \\(a + b \\mod p\\) should be \\( a + b - p = c_0 + c_1 \\cdot \\phi + \\phi^2 - (\\phi^2 - \\phi + 1) =c_0 + c_1 \\cdot \\phi + \\phi - 1 = u + \\phi - 1\\)
      \n
    • if \\( u < p\\), computationally, \\( u - p\\) will borrow from \\( \\phi^2\\), which is equivalent to
      \\( \\phi^2 + u - p = u + \\phi - 1 \\), which is \\( a - b \\mod p\\). therefore, the result is just \\( u-p \\)
      • \n
    • if \\( u> p\\), this will unlikely occur, if u >p, given by \\( \\phi^2 > p \\), so \\( u + \\phi^2 > p + \\phi^2 > 2p\\) that means a or b is not in canonical form. does not consider this case for now
      • \n
    \n
    • \n
\n

Goldilocks Subtraction

\\(a - b \\mod MOD\\)
if a < b, a - b is negative, and the binary representation is 2’s complement of the negative value. therefore, it’s requied to add by MOD and treat the final binary representation as positive value.

\n

Goldilocks Multiplication

let \\(a = a_0 + a_1 \\cdot \\phi\\), and \\(b=b_0 + b_1 \\cdot \\phi\\)
to compute \\(c = a \\cdot b\\)

\n
    \n
  • first, turn it into \\( a_0 \\cdot b_0 +(a_1\\cdot b_0 +a_0\\cdot b_1)\\cdot \\phi + a_1 \\cdot b_1 \\phi^2 \\)
    it may carry to \\( \\phi^2 \\). so the results is a u128 ([u32;4])
    • \n
  • second, reduce the [u32;4] to u64, details to be described in the following section.
    • \n
\n

plonky2 Goldilocks Fieldreduce128

1
2
3
4
5
6
7
8
9
10
11
12
13
14
fn reduce128(x: u128) -> GoldilocksField {
    let (x_lo, x_hi) = split(x); // This is a no-op
    let x_hi_hi = x_hi >> 32;
    let x_hi_lo = x_hi & EPSILON;

    let (mut t0, borrow) = x_lo.overflowing_sub(x_hi_hi);
    if borrow {
        branch_hint(); // A borrow is exceedingly rare. It is faster to branch.
        t0 -= EPSILON; // Cannot underflow.
    }
    let t1 = x_hi_lo * EPSILON;
    let t2 = unsafe { add_no_canonicalize_trashing_input(t0, t1) };
    GoldilocksField(t2)
}
\n\n

let \\( n = n_0 + n_1 \\cdot \\phi + n_2 \\cdot \\phi^{2} + n_3 \\cdot \\phi^{3}\\)
since, \\(\\phi^{3} = -1\\), \\(n\\) could be reduced to \\( n = n_0 + n_1 \\cdot \\phi + n_2 \\cdot \\phi^{2} - n_3 \\),
which is line 6 in the above code snipet. when borrow occurs, it means it borrows \\( \\phi^2\\) ( subtraction is of u64, so borrow 1 menas add \\( \\phi^2\\), 1<<64). therefore, needs to reduce \\( \\phi^2\\). since \\( \\phi^2 = \\phi - 1\\), which is reduce by \\( \\phi - 1\\) (this is EPSILON).
Note needs to consider line 23 could underflow

\n

next step is to reduce \\( n_2 \\cdot \\phi^2 \\), which is just \\( n_2 \\cdot (\\phi -1) \\), which is \\( n_2 \\cdot EPSILON \\), line 11, x_hi_lo * EPSILON

\n

Sppark reduce(uint32_t temp[4])

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
inline void reduce(uint32_t temp[4])
{
    uint32_t carry;

    asm("sub.cc.u32 %0, %0, %3; subc.cc.u32 %1, %1, %4; subc.u32 %2, 0, 0;"
        : "+r"(temp[0]), "+r"(temp[1]), "=r"(carry)
        : "r"(temp[2]), "r"(temp[3]));
    asm("add.cc.u32 %0, %0, %2; addc.u32 %1, %1, %3;"
        : "+r"(temp[1]), "+r"(carry)
        : "r"(temp[2]), "r"(temp[3]));

    asm("mad.lo.cc.u32 %0, %3, %4, %0; madc.hi.cc.u32 %1, %3, %4, %1; addc.u32 %2, 0, 0;"
        : "+r"(temp[0]), "+r"(temp[1]), "=r"(temp[2])
        : "r"(carry), "r"(gl64_device::W));
    asm("mad.lo.cc.u32 %0, %2, %3, %0; madc.hi.u32 %1, %2, %3, %1;"
        : "+r"(temp[0]), "+r"(temp[1])
        : "r"(temp[2]), "r"(gl64_device::W));


    asm("mad.lo.cc.u32 %0, %2, %3, %0; madc.hi.u32 %1, %2, %3, %1;"
        : "+r"(temp[0]), "+r"(temp[1])
        : "r"(temp[2]), "r"(gl64_device::W));

    asm("mov.b64 %0, {%1, %2};" : "=l"(val) : "r"(temp[0]), "r"(temp[1]));
}

\n

note: W is epsilon
let \\( n = n_0 + n_1 \\cdot \\phi + n_2 \\cdot \\phi^{2} + n_3 \\cdot \\phi^{3}\\)
\\( n = n_0 + n_2\\cdot\\phi^2 + \\phi(n_1+n_3\\cdot \\phi^2)\\).
since \\( \\phi^2 = \\phi - 1\\)
\\( n = n_0 + n_2\\cdot(\\phi-1) + \\phi(n_1+n_3\\cdot(\\phi-1))\\).
\\( n = n_0 -n_2 +n_2\\phi + \\phi(n_1-n_3 + n_3\\cdot\\phi)\\).
\\( n = n_0 -n_2 + \\phi(n_1-n_3 +n_2) + n_3\\cdot\\phi^2\\).
line 5 computes n0 - n2 and set to temp[0]; & computes n1-n3 and sets to temp[1]
line 8 computes n1- n3 + n2; add n_3 with carry and put it to carry.

\n

let \\(c\\) be the carry, at \\(\\phi^2\\), it is actually \\( c \\cdot \\phi^2\\). it could be further reduced to \\(c \\cdot (\\phi - 1) = c \\cdot W\\)
line 12 is to reduce the carry part to temp[0], temp[1], and temp[2].
if temp[2] still exist. line 15 will reduce it to temp[0], temp[1] similar as above.
at this step. only temp[0] and temp[1] will contains value and return as the result

\n

appendix

    \n
  • underflow
    1
    2
    3
    4
    5
    6
    uint64_t tmp;
    uint32_t borrow;
    asm("{ .reg.pred %top;");
    asm("sub.cc.u64 %0, %0, %2; subc.u32 %1, 0, 0;"
        : "+l"(val), "=r"(borrow)
        : "l"(b.val));
    \nall bits of borrow will be set to 1 if underflow occurs
    • \n
\n

references

\n"},{"title":"bls12_381","date":"2023-12-01T11:09:11.000Z","_content":"\n## introduction\nBLS12-381 is a pairing-friendly elliptic curve. Curve BLS12-381 was designed by [Sean Bowe](https://twitter.com/ebfull) in early 2017 as the foundation for an upgrade to the Zcash protocol. It is both pairing-friendly (making it efficient for digital signatures) and effective for constructing zkSnarks. BLS12-381 is part of a family of curves described by Barreto, Lynn, and Scott (BLS). The 12 is the embedding degree of the curve: neither too low, nor too high. The 381 is the number of bits needed to represent coordinates on the curve: the field modulus, \\\\(q\\\\). 381 is a fairly handy number as we can use 48 bytes per field element, with 3 bits left over for useful flags or arithmetic optimisations. This size of this number is guided both by security requirements and implementation efficiency.\n\n## curve equation and parameters\nThe basic equation of the BLS12-381 curve is \\\\( y^2 = x^3 + 4\\\\)\nThe key parameters for a BLS curve are set using a single parameter \\\\(\\mathbf{x}\\\\) (different from the in the curve equation!). BLS12-381 is derived from the \\\\( k \\equiv 0 \\mod 6\\\\) case of Construction 6.6 in the [taxonomy](https://eprint.iacr.org/2006/372.pdf).\n\nSpecific design goals for BLS12-381 are:\n- \\\\(\\mathbf{x}\\\\) has “low hamming weight”, meaning that it has very few bits set to 1. This is particularly important for the efficiency of the algorithm that calculates pairings (the Miller loop).\n- The field modulus \\\\(q\\\\) mentioned above is prime and has 383 bits or fewer, which makes 64-bit or 32-bit arithmetic on it more efficient.\n- The order \\\\(r\\\\) of the subgroups we use is prime and has 255 bits or fewer, which is good for the same reason as above.\n- To support zkSnark schemes, we want to have a large power of two root of unity in the field. This means we want **\\\\(2^n\\\\) to be a factor of \\\\(r-1\\\\)**, for some biggish \\\\(n\\\\). (Making \\\\(\\mathbf{x}\\\\) a multiple of \\\\(2^{n/2}\\\\) will achieve this.) This property is key to being able to use fast Fourier transforms for interesting things like polynomial multiplication.\n\nThe value \\\\(\\mathbf{x}\\\\) = -0xd201000000010000 (hexadecimal, note that it is negative) gives the largest and the lowest Hamming weight meeting these criteria. With this value we have,\n| parameters |equation|value(hex)|comments|\n|---|:---:|:---:|---:|\n|Field modulus \\\\(q\\\\)|\\\\(\\frac{1}{3}(\\mathbf{x}-1)^2(\\mathbf{x}^4-\\mathbf{x}^2+1) + \\mathbf{x}\\\\)|0x1a0111ea397fe69a4b1ba7b6434bacd764774b84f3
8512bf6730d2a0f6b0f6241eabfffeb153ffffb9feffffffffaaab|381 bits, prime|\n|Subgroup size \\\\(r\\\\)|\\\\((\\mathbf{x}^4-\\mathbf{x}^2+1)\\\\)|0x73eda753299d7d483339d80809a1d80553
bda402fffe5bfeffffffff00000001|255 bits, prime|\n\n## Field extensions\nField extensions are fundamental to elliptic curve pairings. The “12” is BLS12-381 is not only the embedding degree, it is also (relatedly) the degree of field extension that we will need to use.\nFor example, The complex numbers are a quadratic extension of the real numbers (\\\\(x^2 = 1\\\\)). Complex numbers can’t be extended any further because there are [no irreducible polynomials over the complex numbers](https://en.wikipedia.org/wiki/Fundamental_theorem_of_algebra). But for finite fields, if we can find an irreducible \\\\(k\\\\)-degree polynomial in our field \\\\(F_q\\\\), and we often can, then we are able to extend the field to \\\\(F_{q^k}\\\\), , and represent the elements of the extended field as degree \\\\(k-1\\\\) polynomials, \\\\(a_0 + a_1 x + ... + a_{k-1}x^{k-1}\\\\). we can represent this compactly as (\\\\(a_0, a_1, ..., a_{k-1}\\\\))\nIn practice, large extension fields like \\\\(F_{q^{12}}\\\\)are implemented as towers of smaller extensions.\n\n## the curves\nBLS12-381 is really dealing with two curves. Both curves share more-or-less the same curve equation, but are defined over different fields.\nThe simpler one is over the finite field \\\\(F_q\\\\), which is just the integers mod \\\\(q\\\\). So the curve has points only where the equation \\\\(y^2 = x^3 +4\\\\) has solutions with \\\\(x\\\\) and \\\\(y\\\\) both integers less than \\\\(q\\\\). We shall call this curve \\\\(E(F_q)\\\\).\nThe other curve is defined over an extension of \\\\(F_q\\\\)to \\\\(F_{q^2}\\\\) (think complex numbers). In this case, the curve equation is slightly modified to be \\\\(y^2 = x^3+4(1+i)\\\\), and we call the curve \\\\(E'(F_{q^2})\\\\)\n\n## the subgroups\nA pairing is a bilinear map. This means that it takes as input two points, each from a group of the same order, \\\\(r\\\\). for rather technical reasons, these two groups need to be distinct. Let’s call them \n\\\\(G_1\\\\) and \\\\(G_2\\\\).\nUnfortunately, our simple curve \\\\(E(F_q)\\\\) **has only a single large subgroup of order \\\\(r\\\\)**, so we can’t define a pairing based solely on \\\\(E(F_q)\\\\). However, if we keep extending the field over which \n\\\\(E\\\\) is defined, it can be proved that we eventually find a curve that has more than one subgroup of order \\\\(r\\\\). that is, for some \\\\(k\\\\), **\\\\(E(F_{q^k})\\\\) contains other subgroups of order \\\\(r\\\\) that we can use**. One of these subgroups contains only points having a **trace of zero**[1], and we choose that subgroup to be \\\\(G_2\\\\)\n\n\nThis number \\\\(k\\\\), the amount that we need to extend the base field by to find the new group, is called the embedding degree of the curve, which in our case is the “12” in BLS12-381. For completeness, note that each of \\\\(G_1\\\\) and \\\\(G_2\\\\) shares with its containing curve the “point at infinity”. This is the identity element of the elliptic curve arithmetic group, often denoted \\\\(\\mathcal O\\\\)\n. For any point \\\\(P\\\\), \\\\( P + \\mathcal O = \\mathcal O + P = P\\\\). \n\n## Twists\nBut there’s another challenge. As discussed earlier, doing arithmetic in \\\\(F_{q^{12}}\\\\) is horribly complicated and inefficient. And curve operations need a lot of arithmetic. A [twist](http://indigo.ie/~mscott/twists.pdf) is something like a coordinate transformation. Rather wonderfully, this can be used to transform our \\\\(E(F_{q^{12}})\\\\) curve into a curve defined over a lower degree field that still has an order \\\\(r\\\\) subgroup. Moreover, this subgroup has a simple mapping to and from our \\\\(G_2\\\\) group\n\nBLS12-381 uses a “sextic twist”. This means that it reduces the degree of the extension field by a factor of six. So \\\\(G_2\\\\) on the twisted curve can be defined over \\\\(F_{q^2}\\\\) instead of \\\\(F_{q^{12}}\\\\)\nI haven’t seen this written down anywhere—but attempting to decode section 3 of [this](https://eprint.iacr.org/2005/133.pdf)—if we find a \\\\(u\\\\) such that \\\\(u^6 = (1+i)^{-1}\\\\), then we can define our twisting transformation as \\\\( (x,y) -> (x/u^2, y/u^3)\\\\). This transforms our original curve \\\\(E: y^2 = x^3 + 4\\\\) into the curve \\\\(E': y^2 + 4/u^6 = x^3 + 4(1+i)\\\\). **\\\\(E\\\\) and \\\\(E'\\\\) look different, but are actually the same object presented with respect to coefficinets in different base fields**.\n\nSo these are the two groups we will be using:\n- \\\\(G_1 \\subset E(F_q)\\\\), where \\\\(E: y^2 = x^3 + 4\\\\)\n- \\\\(G_2 \\subset E(F_{q^2})\\\\), where \\\\(E: y^2 = x^3 + 4(1+i)\\\\)\nNote that coordinates of points in the \\\\(G_1\\\\) group are pairs of integers, and coordinates of points in the \\\\(G_2\\\\) group are pairs of complex integers.\n\n## Parings\ns far as BLS12-381 is concerned, a pairing simply takes a point \\\\(P \\in G_1 \\subset E(F_q)\\\\), and a point \\\\(Q \\in G_2 \\subset E'(F_{q^2})\\\\) and outputs a point from a group \\\\(G_T \\subset F_{q^{12}}\\\\). That is, for a paring \\\\(e\\\\), \\\\(e: G_1 \\times G_2 \\rightarrow G_T\\\\)\n\nproperties of pairing\n\\\\( e(P, Q+R) = e(P,Q) \\cdot e(P,R) \\\\), \n**note** as a way of memory, can think it as \\\\(P^{Q+R} = P^Q \\cdot P^R\\\\)\n\\\\( e(P+S, R) = e(P,R) \\cdot e(S,R)\\\\)\nFrom this, we can deduce that all of the following identities hold:\n\\\\(e([a]P, [b]Q) = e(P, [b]Q)^a = e(P,Q)^{ab} = e(P, [a]Q)^b = e([b]P, [a]Q)\\\\)\n\n\n## Embedding degree\nThe embedding degree, \\\\(k\\\\), is calculated as the smallest positive integer such that \\\\(r\\\\) divides \\\\(q^k -1\\\\). So, in the case of BLS12-381, \\\\(r\\\\) is a factor of \\\\(q^{12} -1\\\\), but not of any lower power.\nThe choice of an embedding degree is a balance between security and efficiency (as ever). On the security front, the embedding degree is also known as the security multiplier: a higher embedding degree makes the discrete logarithm problem harder to solve in \n. However, a high embedding degree means we have to do field operations in high-degree extensions, which is clunky and inefficient. \n\n\n## cofactor\nA subgroup’s cofactor is the ratio of the size of the whole group to the size of the subgroup. \n| Group |Cofactor|Equation|value(hex)|\n|---|:---:|:---:|---:|\n|\\\\(G_1\\\\)|\\\\(h_1\\\\)|\\\\(\\frac{\\mathbf{x-1}^2}{3}\\\\)|0x396c8c005555e
1568c00aaab0000aaab|\n|\\\\(G_2\\\\)|\\\\(h_2\\\\)|\\\\(\\frac{\\mathbf{x}^8 -4\\mathbf{x}^7+5\\mathbf{x}^6-4\\mathbf{x}^4+6\\mathbf{x}^3-4\\mathbf{x}^2-4\\mathbf{x}+13}{9}\\\\)|0x5d543a95414e7f1091d50792876a202cd91de
4547085abaa68a205b2e5a7ddfa628f1cb4d9e82ef2
1537e293a6691ae1616ec6e786f0c70cf1c38e31c7238e5|\n\n**note** multiplying by the cofactor turns out to be a straightforward way to map any arbitrary point on the elliptic curve into the respective subgroup \\\\(G_1\\\\) or \\\\(G_2\\\\)\n\n## Generators\n\\\\(G_1\\\\) and \\\\(G_2\\\\) are cyclic groups of prime order, so any point (except the identity/point at infinity) is a generator. Thus, picking generators is just a matter of convention.\nGenerator points \\\\(G_1\\\\) and \\\\(G_2\\\\) are specified in decimal [here](https://github.com/zcash/librustzcash/blob/6e0364cd42a2b3d2b958a54771ef51a8db79dd29/pairing/src/bls12_381/README.md#generators)\n\nThese were chosen as follows:\n\n> The generators of \\\\(G_1\\\\) and \\\\(G_2\\\\) are computed by finding the lexicographically smallest valid x-coordinate, and its lexicographically smallest > y-coordinate and scaling it by the cofactor such that the result is not the point at infinity.\n\n\n## Foot notes\n[1] The “trace zero subgroup” qualifies as an obscure incantation. Basically, the trace of a point is \\\\(\\sum_{i=0}^{k-1}(x^{q^i}, y^{q^i})\\\\), where \\\\(k=12\\\\) in our case. Understanding this involves stuff like the Frobenius endomorphism.\n\n\n## references\n- https://hackmd.io/@benjaminion/bls12-381\n- [initial post of bls12-381](https://electriccoin.co/blog/new-snark-curve/)\n- [implementation: blst](https://github.com/supranational/blst)","source":"_posts/cryptography/elliptic_curve/bls12-381.md","raw":"---\ntitle: bls12_381\ndate: 2023-12-01 19:09:11\ntags: [cryptography, ec]\n---\n\n## introduction\nBLS12-381 is a pairing-friendly elliptic curve. Curve BLS12-381 was designed by [Sean Bowe](https://twitter.com/ebfull) in early 2017 as the foundation for an upgrade to the Zcash protocol. It is both pairing-friendly (making it efficient for digital signatures) and effective for constructing zkSnarks. BLS12-381 is part of a family of curves described by Barreto, Lynn, and Scott (BLS). The 12 is the embedding degree of the curve: neither too low, nor too high. The 381 is the number of bits needed to represent coordinates on the curve: the field modulus, \\\\(q\\\\). 381 is a fairly handy number as we can use 48 bytes per field element, with 3 bits left over for useful flags or arithmetic optimisations. This size of this number is guided both by security requirements and implementation efficiency.\n\n## curve equation and parameters\nThe basic equation of the BLS12-381 curve is \\\\( y^2 = x^3 + 4\\\\)\nThe key parameters for a BLS curve are set using a single parameter \\\\(\\mathbf{x}\\\\) (different from the in the curve equation!). BLS12-381 is derived from the \\\\( k \\equiv 0 \\mod 6\\\\) case of Construction 6.6 in the [taxonomy](https://eprint.iacr.org/2006/372.pdf).\n\nSpecific design goals for BLS12-381 are:\n- \\\\(\\mathbf{x}\\\\) has “low hamming weight”, meaning that it has very few bits set to 1. This is particularly important for the efficiency of the algorithm that calculates pairings (the Miller loop).\n- The field modulus \\\\(q\\\\) mentioned above is prime and has 383 bits or fewer, which makes 64-bit or 32-bit arithmetic on it more efficient.\n- The order \\\\(r\\\\) of the subgroups we use is prime and has 255 bits or fewer, which is good for the same reason as above.\n- To support zkSnark schemes, we want to have a large power of two root of unity in the field. This means we want **\\\\(2^n\\\\) to be a factor of \\\\(r-1\\\\)**, for some biggish \\\\(n\\\\). (Making \\\\(\\mathbf{x}\\\\) a multiple of \\\\(2^{n/2}\\\\) will achieve this.) This property is key to being able to use fast Fourier transforms for interesting things like polynomial multiplication.\n\nThe value \\\\(\\mathbf{x}\\\\) = -0xd201000000010000 (hexadecimal, note that it is negative) gives the largest and the lowest Hamming weight meeting these criteria. With this value we have,\n| parameters |equation|value(hex)|comments|\n|---|:---:|:---:|---:|\n|Field modulus \\\\(q\\\\)|\\\\(\\frac{1}{3}(\\mathbf{x}-1)^2(\\mathbf{x}^4-\\mathbf{x}^2+1) + \\mathbf{x}\\\\)|0x1a0111ea397fe69a4b1ba7b6434bacd764774b84f3
8512bf6730d2a0f6b0f6241eabfffeb153ffffb9feffffffffaaab|381 bits, prime|\n|Subgroup size \\\\(r\\\\)|\\\\((\\mathbf{x}^4-\\mathbf{x}^2+1)\\\\)|0x73eda753299d7d483339d80809a1d80553
bda402fffe5bfeffffffff00000001|255 bits, prime|\n\n## Field extensions\nField extensions are fundamental to elliptic curve pairings. The “12” is BLS12-381 is not only the embedding degree, it is also (relatedly) the degree of field extension that we will need to use.\nFor example, The complex numbers are a quadratic extension of the real numbers (\\\\(x^2 = 1\\\\)). Complex numbers can’t be extended any further because there are [no irreducible polynomials over the complex numbers](https://en.wikipedia.org/wiki/Fundamental_theorem_of_algebra). But for finite fields, if we can find an irreducible \\\\(k\\\\)-degree polynomial in our field \\\\(F_q\\\\), and we often can, then we are able to extend the field to \\\\(F_{q^k}\\\\), , and represent the elements of the extended field as degree \\\\(k-1\\\\) polynomials, \\\\(a_0 + a_1 x + ... + a_{k-1}x^{k-1}\\\\). we can represent this compactly as (\\\\(a_0, a_1, ..., a_{k-1}\\\\))\nIn practice, large extension fields like \\\\(F_{q^{12}}\\\\)are implemented as towers of smaller extensions.\n\n## the curves\nBLS12-381 is really dealing with two curves. Both curves share more-or-less the same curve equation, but are defined over different fields.\nThe simpler one is over the finite field \\\\(F_q\\\\), which is just the integers mod \\\\(q\\\\). So the curve has points only where the equation \\\\(y^2 = x^3 +4\\\\) has solutions with \\\\(x\\\\) and \\\\(y\\\\) both integers less than \\\\(q\\\\). We shall call this curve \\\\(E(F_q)\\\\).\nThe other curve is defined over an extension of \\\\(F_q\\\\)to \\\\(F_{q^2}\\\\) (think complex numbers). In this case, the curve equation is slightly modified to be \\\\(y^2 = x^3+4(1+i)\\\\), and we call the curve \\\\(E'(F_{q^2})\\\\)\n\n## the subgroups\nA pairing is a bilinear map. This means that it takes as input two points, each from a group of the same order, \\\\(r\\\\). for rather technical reasons, these two groups need to be distinct. Let’s call them \n\\\\(G_1\\\\) and \\\\(G_2\\\\).\nUnfortunately, our simple curve \\\\(E(F_q)\\\\) **has only a single large subgroup of order \\\\(r\\\\)**, so we can’t define a pairing based solely on \\\\(E(F_q)\\\\). However, if we keep extending the field over which \n\\\\(E\\\\) is defined, it can be proved that we eventually find a curve that has more than one subgroup of order \\\\(r\\\\). that is, for some \\\\(k\\\\), **\\\\(E(F_{q^k})\\\\) contains other subgroups of order \\\\(r\\\\) that we can use**. One of these subgroups contains only points having a **trace of zero**[1], and we choose that subgroup to be \\\\(G_2\\\\)\n\n\nThis number \\\\(k\\\\), the amount that we need to extend the base field by to find the new group, is called the embedding degree of the curve, which in our case is the “12” in BLS12-381. For completeness, note that each of \\\\(G_1\\\\) and \\\\(G_2\\\\) shares with its containing curve the “point at infinity”. This is the identity element of the elliptic curve arithmetic group, often denoted \\\\(\\mathcal O\\\\)\n. For any point \\\\(P\\\\), \\\\( P + \\mathcal O = \\mathcal O + P = P\\\\). \n\n## Twists\nBut there’s another challenge. As discussed earlier, doing arithmetic in \\\\(F_{q^{12}}\\\\) is horribly complicated and inefficient. And curve operations need a lot of arithmetic. A [twist](http://indigo.ie/~mscott/twists.pdf) is something like a coordinate transformation. Rather wonderfully, this can be used to transform our \\\\(E(F_{q^{12}})\\\\) curve into a curve defined over a lower degree field that still has an order \\\\(r\\\\) subgroup. Moreover, this subgroup has a simple mapping to and from our \\\\(G_2\\\\) group\n\nBLS12-381 uses a “sextic twist”. This means that it reduces the degree of the extension field by a factor of six. So \\\\(G_2\\\\) on the twisted curve can be defined over \\\\(F_{q^2}\\\\) instead of \\\\(F_{q^{12}}\\\\)\nI haven’t seen this written down anywhere—but attempting to decode section 3 of [this](https://eprint.iacr.org/2005/133.pdf)—if we find a \\\\(u\\\\) such that \\\\(u^6 = (1+i)^{-1}\\\\), then we can define our twisting transformation as \\\\( (x,y) -> (x/u^2, y/u^3)\\\\). This transforms our original curve \\\\(E: y^2 = x^3 + 4\\\\) into the curve \\\\(E': y^2 + 4/u^6 = x^3 + 4(1+i)\\\\). **\\\\(E\\\\) and \\\\(E'\\\\) look different, but are actually the same object presented with respect to coefficinets in different base fields**.\n\nSo these are the two groups we will be using:\n- \\\\(G_1 \\subset E(F_q)\\\\), where \\\\(E: y^2 = x^3 + 4\\\\)\n- \\\\(G_2 \\subset E(F_{q^2})\\\\), where \\\\(E: y^2 = x^3 + 4(1+i)\\\\)\nNote that coordinates of points in the \\\\(G_1\\\\) group are pairs of integers, and coordinates of points in the \\\\(G_2\\\\) group are pairs of complex integers.\n\n## Parings\ns far as BLS12-381 is concerned, a pairing simply takes a point \\\\(P \\in G_1 \\subset E(F_q)\\\\), and a point \\\\(Q \\in G_2 \\subset E'(F_{q^2})\\\\) and outputs a point from a group \\\\(G_T \\subset F_{q^{12}}\\\\). That is, for a paring \\\\(e\\\\), \\\\(e: G_1 \\times G_2 \\rightarrow G_T\\\\)\n\nproperties of pairing\n\\\\( e(P, Q+R) = e(P,Q) \\cdot e(P,R) \\\\), \n**note** as a way of memory, can think it as \\\\(P^{Q+R} = P^Q \\cdot P^R\\\\)\n\\\\( e(P+S, R) = e(P,R) \\cdot e(S,R)\\\\)\nFrom this, we can deduce that all of the following identities hold:\n\\\\(e([a]P, [b]Q) = e(P, [b]Q)^a = e(P,Q)^{ab} = e(P, [a]Q)^b = e([b]P, [a]Q)\\\\)\n\n\n## Embedding degree\nThe embedding degree, \\\\(k\\\\), is calculated as the smallest positive integer such that \\\\(r\\\\) divides \\\\(q^k -1\\\\). So, in the case of BLS12-381, \\\\(r\\\\) is a factor of \\\\(q^{12} -1\\\\), but not of any lower power.\nThe choice of an embedding degree is a balance between security and efficiency (as ever). On the security front, the embedding degree is also known as the security multiplier: a higher embedding degree makes the discrete logarithm problem harder to solve in \n. However, a high embedding degree means we have to do field operations in high-degree extensions, which is clunky and inefficient. \n\n\n## cofactor\nA subgroup’s cofactor is the ratio of the size of the whole group to the size of the subgroup. \n| Group |Cofactor|Equation|value(hex)|\n|---|:---:|:---:|---:|\n|\\\\(G_1\\\\)|\\\\(h_1\\\\)|\\\\(\\frac{\\mathbf{x-1}^2}{3}\\\\)|0x396c8c005555e
1568c00aaab0000aaab|\n|\\\\(G_2\\\\)|\\\\(h_2\\\\)|\\\\(\\frac{\\mathbf{x}^8 -4\\mathbf{x}^7+5\\mathbf{x}^6-4\\mathbf{x}^4+6\\mathbf{x}^3-4\\mathbf{x}^2-4\\mathbf{x}+13}{9}\\\\)|0x5d543a95414e7f1091d50792876a202cd91de
4547085abaa68a205b2e5a7ddfa628f1cb4d9e82ef2
1537e293a6691ae1616ec6e786f0c70cf1c38e31c7238e5|\n\n**note** multiplying by the cofactor turns out to be a straightforward way to map any arbitrary point on the elliptic curve into the respective subgroup \\\\(G_1\\\\) or \\\\(G_2\\\\)\n\n## Generators\n\\\\(G_1\\\\) and \\\\(G_2\\\\) are cyclic groups of prime order, so any point (except the identity/point at infinity) is a generator. Thus, picking generators is just a matter of convention.\nGenerator points \\\\(G_1\\\\) and \\\\(G_2\\\\) are specified in decimal [here](https://github.com/zcash/librustzcash/blob/6e0364cd42a2b3d2b958a54771ef51a8db79dd29/pairing/src/bls12_381/README.md#generators)\n\nThese were chosen as follows:\n\n> The generators of \\\\(G_1\\\\) and \\\\(G_2\\\\) are computed by finding the lexicographically smallest valid x-coordinate, and its lexicographically smallest > y-coordinate and scaling it by the cofactor such that the result is not the point at infinity.\n\n\n## Foot notes\n[1] The “trace zero subgroup” qualifies as an obscure incantation. Basically, the trace of a point is \\\\(\\sum_{i=0}^{k-1}(x^{q^i}, y^{q^i})\\\\), where \\\\(k=12\\\\) in our case. Understanding this involves stuff like the Frobenius endomorphism.\n\n\n## references\n- https://hackmd.io/@benjaminion/bls12-381\n- [initial post of bls12-381](https://electriccoin.co/blog/new-snark-curve/)\n- [implementation: blst](https://github.com/supranational/blst)","slug":"cryptography/elliptic_curve/bls12-381","published":1,"updated":"2023-12-03T03:53:57.439Z","_id":"clpm1qei10000hpsj4saxbd96","comments":1,"layout":"post","photos":[],"link":"","content":"

introduction

BLS12-381 is a pairing-friendly elliptic curve. Curve BLS12-381 was designed by Sean Bowe in early 2017 as the foundation for an upgrade to the Zcash protocol. It is both pairing-friendly (making it efficient for digital signatures) and effective for constructing zkSnarks. BLS12-381 is part of a family of curves described by Barreto, Lynn, and Scott (BLS). The 12 is the embedding degree of the curve: neither too low, nor too high. The 381 is the number of bits needed to represent coordinates on the curve: the field modulus, \\(q\\). 381 is a fairly handy number as we can use 48 bytes per field element, with 3 bits left over for useful flags or arithmetic optimisations. This size of this number is guided both by security requirements and implementation efficiency.

\n

curve equation and parameters

The basic equation of the BLS12-381 curve is \\( y^2 = x^3 + 4\\)
The key parameters for a BLS curve are set using a single parameter \\(\\mathbf{x}\\) (different from the in the curve equation!). BLS12-381 is derived from the \\( k \\equiv 0 \\mod 6\\) case of Construction 6.6 in the taxonomy.

\n

Specific design goals for BLS12-381 are:

\n
    \n
  • \\(\\mathbf{x}\\) has “low hamming weight”, meaning that it has very few bits set to 1. This is particularly important for the efficiency of the algorithm that calculates pairings (the Miller loop).
    • \n
  • The field modulus \\(q\\) mentioned above is prime and has 383 bits or fewer, which makes 64-bit or 32-bit arithmetic on it more efficient.
    • \n
  • The order \\(r\\) of the subgroups we use is prime and has 255 bits or fewer, which is good for the same reason as above.
    • \n
  • To support zkSnark schemes, we want to have a large power of two root of unity in the field. This means we want \\(2^n\\) to be a factor of \\(r-1\\), for some biggish \\(n\\). (Making \\(\\mathbf{x}\\) a multiple of \\(2^{n/2}\\) will achieve this.) This property is key to being able to use fast Fourier transforms for interesting things like polynomial multiplication.
    • \n
\n

The value \\(\\mathbf{x}\\) = -0xd201000000010000 (hexadecimal, note that it is negative) gives the largest and the lowest Hamming weight meeting these criteria. With this value we have,

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
parametersequationvalue(hex)comments
Field modulus \\(q\\)\\(\\frac{1}{3}(\\mathbf{x}-1)^2(\\mathbf{x}^4-\\mathbf{x}^2+1) + \\mathbf{x}\\)0x1a0111ea397fe69a4b1ba7b6434bacd764774b84f3
8512bf6730d2a0f6b0f6241eabfffeb153ffffb9feffffffffaaab		381 bits, prime
Subgroup size \\(r\\)\\((\\mathbf{x}^4-\\mathbf{x}^2+1)\\)0x73eda753299d7d483339d80809a1d80553
bda402fffe5bfeffffffff00000001		255 bits, prime
\n

Field extensions

Field extensions are fundamental to elliptic curve pairings. The “12” is BLS12-381 is not only the embedding degree, it is also (relatedly) the degree of field extension that we will need to use.
For example, The complex numbers are a quadratic extension of the real numbers (\\(x^2 = 1\\)). Complex numbers can’t be extended any further because there are no irreducible polynomials over the complex numbers. But for finite fields, if we can find an irreducible \\(k\\)-degree polynomial in our field \\(F_q\\), and we often can, then we are able to extend the field to \\(F_{q^k}\\), , and represent the elements of the extended field as degree \\(k-1\\) polynomials, \\(a_0 + a_1 x + … + a_{k-1}x^{k-1}\\). we can represent this compactly as (\\(a_0, a_1, …, a_{k-1}\\))
In practice, large extension fields like \\(F_{q^{12}}\\)are implemented as towers of smaller extensions.

\n

the curves

BLS12-381 is really dealing with two curves. Both curves share more-or-less the same curve equation, but are defined over different fields.
The simpler one is over the finite field \\(F_q\\), which is just the integers mod \\(q\\). So the curve has points only where the equation \\(y^2 = x^3 +4\\) has solutions with \\(x\\) and \\(y\\) both integers less than \\(q\\). We shall call this curve \\(E(F_q)\\).
The other curve is defined over an extension of \\(F_q\\)to \\(F_{q^2}\\) (think complex numbers). In this case, the curve equation is slightly modified to be \\(y^2 = x^3+4(1+i)\\), and we call the curve \\(E’(F_{q^2})\\)

\n

the subgroups

A pairing is a bilinear map. This means that it takes as input two points, each from a group of the same order, \\(r\\). for rather technical reasons, these two groups need to be distinct. Let’s call them
\\(G_1\\) and \\(G_2\\).
Unfortunately, our simple curve \\(E(F_q)\\) has only a single large subgroup of order \\(r\\), so we can’t define a pairing based solely on \\(E(F_q)\\). However, if we keep extending the field over which
\\(E\\) is defined, it can be proved that we eventually find a curve that has more than one subgroup of order \\(r\\). that is, for some \\(k\\), \\(E(F_{q^k})\\) contains other subgroups of order \\(r\\) that we can use. One of these subgroups contains only points having a trace of zero[1], and we choose that subgroup to be \\(G_2\\)

\n

This number \\(k\\), the amount that we need to extend the base field by to find the new group, is called the embedding degree of the curve, which in our case is the “12” in BLS12-381. For completeness, note that each of \\(G_1\\) and \\(G_2\\) shares with its containing curve the “point at infinity”. This is the identity element of the elliptic curve arithmetic group, often denoted \\(\\mathcal O\\)
. For any point \\(P\\), \\( P + \\mathcal O = \\mathcal O + P = P\\).

\n

Twists

But there’s another challenge. As discussed earlier, doing arithmetic in \\(F_{q^{12}}\\) is horribly complicated and inefficient. And curve operations need a lot of arithmetic. A twist is something like a coordinate transformation. Rather wonderfully, this can be used to transform our \\(E(F_{q^{12}})\\) curve into a curve defined over a lower degree field that still has an order \\(r\\) subgroup. Moreover, this subgroup has a simple mapping to and from our \\(G_2\\) group

\n

BLS12-381 uses a “sextic twist”. This means that it reduces the degree of the extension field by a factor of six. So \\(G_2\\) on the twisted curve can be defined over \\(F_{q^2}\\) instead of \\(F_{q^{12}}\\)
I haven’t seen this written down anywhere—but attempting to decode section 3 of this—if we find a \\(u\\) such that \\(u^6 = (1+i)^{-1}\\), then we can define our twisting transformation as \\( (x,y) -> (x/u^2, y/u^3)\\). This transforms our original curve \\(E: y^2 = x^3 + 4\\) into the curve \\(E’: y^2 + 4/u^6 = x^3 + 4(1+i)\\). \\(E\\) and \\(E’\\) look different, but are actually the same object presented with respect to coefficinets in different base fields.

\n

So these are the two groups we will be using:

\n
    \n
  • \\(G_1 \\subset E(F_q)\\), where \\(E: y^2 = x^3 + 4\\)
    • \n
  • \\(G_2 \\subset E(F_{q^2})\\), where \\(E: y^2 = x^3 + 4(1+i)\\)
    Note that coordinates of points in the \\(G_1\\) group are pairs of integers, and coordinates of points in the \\(G_2\\) group are pairs of complex integers.
    • \n
\n

Parings

s far as BLS12-381 is concerned, a pairing simply takes a point \\(P \\in G_1 \\subset E(F_q)\\), and a point \\(Q \\in G_2 \\subset E’(F_{q^2})\\) and outputs a point from a group \\(G_T \\subset F_{q^{12}}\\). That is, for a paring \\(e\\), \\(e: G_1 \\times G_2 \\rightarrow G_T\\)

\n

properties of pairing
\\( e(P, Q+R) = e(P,Q) \\cdot e(P,R) \\),
note as a way of memory, can think it as \\(P^{Q+R} = P^Q \\cdot P^R\\)
\\( e(P+S, R) = e(P,R) \\cdot e(S,R)\\)
From this, we can deduce that all of the following identities hold:
\\(e([a]P, [b]Q) = e(P, [b]Q)^a = e(P,Q)^{ab} = e(P, [a]Q)^b = e([b]P, [a]Q)\\)

\n

Embedding degree

The embedding degree, \\(k\\), is calculated as the smallest positive integer such that \\(r\\) divides \\(q^k -1\\). So, in the case of BLS12-381, \\(r\\) is a factor of \\(q^{12} -1\\), but not of any lower power.
The choice of an embedding degree is a balance between security and efficiency (as ever). On the security front, the embedding degree is also known as the security multiplier: a higher embedding degree makes the discrete logarithm problem harder to solve in
. However, a high embedding degree means we have to do field operations in high-degree extensions, which is clunky and inefficient.

\n

cofactor

A subgroup’s cofactor is the ratio of the size of the whole group to the size of the subgroup.

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
GroupCofactorEquationvalue(hex)
\\(G_1\\)\\(h_1\\)\\(\\frac{\\mathbf{x-1}^2}{3}\\)0x396c8c005555e
1568c00aaab0000aaab
\\(G_2\\)\\(h_2\\)\\(\\frac{\\mathbf{x}^8 -4\\mathbf{x}^7+5\\mathbf{x}^6-4\\mathbf{x}^4+6\\mathbf{x}^3-4\\mathbf{x}^2-4\\mathbf{x}+13}{9}\\)0x5d543a95414e7f1091d50792876a202cd91de
4547085abaa68a205b2e5a7ddfa628f1cb4d9e82ef2
1537e293a6691ae1616ec6e786f0c70cf1c38e31c7238e5
\n

note multiplying by the cofactor turns out to be a straightforward way to map any arbitrary point on the elliptic curve into the respective subgroup \\(G_1\\) or \\(G_2\\)

\n

Generators

\\(G_1\\) and \\(G_2\\) are cyclic groups of prime order, so any point (except the identity/point at infinity) is a generator. Thus, picking generators is just a matter of convention.
Generator points \\(G_1\\) and \\(G_2\\) are specified in decimal here

\n

These were chosen as follows:

\n
\n

The generators of \\(G_1\\) and \\(G_2\\) are computed by finding the lexicographically smallest valid x-coordinate, and its lexicographically smallest > y-coordinate and scaling it by the cofactor such that the result is not the point at infinity.

\n
\n

Foot notes

[1] The “trace zero subgroup” qualifies as an obscure incantation. Basically, the trace of a point is \\(\\sum_{i=0}^{k-1}(x^{q^i}, y^{q^i})\\), where \\(k=12\\) in our case. Understanding this involves stuff like the Frobenius endomorphism.

\n

references

\n","site":{"data":{}},"excerpt":"","more":"

introduction

BLS12-381 is a pairing-friendly elliptic curve. Curve BLS12-381 was designed by Sean Bowe in early 2017 as the foundation for an upgrade to the Zcash protocol. It is both pairing-friendly (making it efficient for digital signatures) and effective for constructing zkSnarks. BLS12-381 is part of a family of curves described by Barreto, Lynn, and Scott (BLS). The 12 is the embedding degree of the curve: neither too low, nor too high. The 381 is the number of bits needed to represent coordinates on the curve: the field modulus, \\(q\\). 381 is a fairly handy number as we can use 48 bytes per field element, with 3 bits left over for useful flags or arithmetic optimisations. This size of this number is guided both by security requirements and implementation efficiency.

\n

curve equation and parameters

The basic equation of the BLS12-381 curve is \\( y^2 = x^3 + 4\\)
The key parameters for a BLS curve are set using a single parameter \\(\\mathbf{x}\\) (different from the in the curve equation!). BLS12-381 is derived from the \\( k \\equiv 0 \\mod 6\\) case of Construction 6.6 in the taxonomy.

\n

Specific design goals for BLS12-381 are:

\n
    \n
  • \\(\\mathbf{x}\\) has “low hamming weight”, meaning that it has very few bits set to 1. This is particularly important for the efficiency of the algorithm that calculates pairings (the Miller loop).
    • \n
  • The field modulus \\(q\\) mentioned above is prime and has 383 bits or fewer, which makes 64-bit or 32-bit arithmetic on it more efficient.
    • \n
  • The order \\(r\\) of the subgroups we use is prime and has 255 bits or fewer, which is good for the same reason as above.
    • \n
  • To support zkSnark schemes, we want to have a large power of two root of unity in the field. This means we want \\(2^n\\) to be a factor of \\(r-1\\), for some biggish \\(n\\). (Making \\(\\mathbf{x}\\) a multiple of \\(2^{n/2}\\) will achieve this.) This property is key to being able to use fast Fourier transforms for interesting things like polynomial multiplication.
    • \n
\n

The value \\(\\mathbf{x}\\) = -0xd201000000010000 (hexadecimal, note that it is negative) gives the largest and the lowest Hamming weight meeting these criteria. With this value we have,

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
parametersequationvalue(hex)comments
Field modulus \\(q\\)\\(\\frac{1}{3}(\\mathbf{x}-1)^2(\\mathbf{x}^4-\\mathbf{x}^2+1) + \\mathbf{x}\\)0x1a0111ea397fe69a4b1ba7b6434bacd764774b84f3
8512bf6730d2a0f6b0f6241eabfffeb153ffffb9feffffffffaaab		381 bits, prime
Subgroup size \\(r\\)\\((\\mathbf{x}^4-\\mathbf{x}^2+1)\\)0x73eda753299d7d483339d80809a1d80553
bda402fffe5bfeffffffff00000001		255 bits, prime
\n

Field extensions

Field extensions are fundamental to elliptic curve pairings. The “12” is BLS12-381 is not only the embedding degree, it is also (relatedly) the degree of field extension that we will need to use.
For example, The complex numbers are a quadratic extension of the real numbers (\\(x^2 = 1\\)). Complex numbers can’t be extended any further because there are no irreducible polynomials over the complex numbers. But for finite fields, if we can find an irreducible \\(k\\)-degree polynomial in our field \\(F_q\\), and we often can, then we are able to extend the field to \\(F_{q^k}\\), , and represent the elements of the extended field as degree \\(k-1\\) polynomials, \\(a_0 + a_1 x + … + a_{k-1}x^{k-1}\\). we can represent this compactly as (\\(a_0, a_1, …, a_{k-1}\\))
In practice, large extension fields like \\(F_{q^{12}}\\)are implemented as towers of smaller extensions.

\n

the curves

BLS12-381 is really dealing with two curves. Both curves share more-or-less the same curve equation, but are defined over different fields.
The simpler one is over the finite field \\(F_q\\), which is just the integers mod \\(q\\). So the curve has points only where the equation \\(y^2 = x^3 +4\\) has solutions with \\(x\\) and \\(y\\) both integers less than \\(q\\). We shall call this curve \\(E(F_q)\\).
The other curve is defined over an extension of \\(F_q\\)to \\(F_{q^2}\\) (think complex numbers). In this case, the curve equation is slightly modified to be \\(y^2 = x^3+4(1+i)\\), and we call the curve \\(E’(F_{q^2})\\)

\n

the subgroups

A pairing is a bilinear map. This means that it takes as input two points, each from a group of the same order, \\(r\\). for rather technical reasons, these two groups need to be distinct. Let’s call them
\\(G_1\\) and \\(G_2\\).
Unfortunately, our simple curve \\(E(F_q)\\) has only a single large subgroup of order \\(r\\), so we can’t define a pairing based solely on \\(E(F_q)\\). However, if we keep extending the field over which
\\(E\\) is defined, it can be proved that we eventually find a curve that has more than one subgroup of order \\(r\\). that is, for some \\(k\\), \\(E(F_{q^k})\\) contains other subgroups of order \\(r\\) that we can use. One of these subgroups contains only points having a trace of zero[1], and we choose that subgroup to be \\(G_2\\)

\n

This number \\(k\\), the amount that we need to extend the base field by to find the new group, is called the embedding degree of the curve, which in our case is the “12” in BLS12-381. For completeness, note that each of \\(G_1\\) and \\(G_2\\) shares with its containing curve the “point at infinity”. This is the identity element of the elliptic curve arithmetic group, often denoted \\(\\mathcal O\\)
. For any point \\(P\\), \\( P + \\mathcal O = \\mathcal O + P = P\\).

\n

Twists

But there’s another challenge. As discussed earlier, doing arithmetic in \\(F_{q^{12}}\\) is horribly complicated and inefficient. And curve operations need a lot of arithmetic. A twist is something like a coordinate transformation. Rather wonderfully, this can be used to transform our \\(E(F_{q^{12}})\\) curve into a curve defined over a lower degree field that still has an order \\(r\\) subgroup. Moreover, this subgroup has a simple mapping to and from our \\(G_2\\) group

\n

BLS12-381 uses a “sextic twist”. This means that it reduces the degree of the extension field by a factor of six. So \\(G_2\\) on the twisted curve can be defined over \\(F_{q^2}\\) instead of \\(F_{q^{12}}\\)
I haven’t seen this written down anywhere—but attempting to decode section 3 of this—if we find a \\(u\\) such that \\(u^6 = (1+i)^{-1}\\), then we can define our twisting transformation as \\( (x,y) -> (x/u^2, y/u^3)\\). This transforms our original curve \\(E: y^2 = x^3 + 4\\) into the curve \\(E’: y^2 + 4/u^6 = x^3 + 4(1+i)\\). \\(E\\) and \\(E’\\) look different, but are actually the same object presented with respect to coefficinets in different base fields.

\n

So these are the two groups we will be using:

\n
    \n
  • \\(G_1 \\subset E(F_q)\\), where \\(E: y^2 = x^3 + 4\\)
    • \n
  • \\(G_2 \\subset E(F_{q^2})\\), where \\(E: y^2 = x^3 + 4(1+i)\\)
    Note that coordinates of points in the \\(G_1\\) group are pairs of integers, and coordinates of points in the \\(G_2\\) group are pairs of complex integers.
    • \n
\n

Parings

s far as BLS12-381 is concerned, a pairing simply takes a point \\(P \\in G_1 \\subset E(F_q)\\), and a point \\(Q \\in G_2 \\subset E’(F_{q^2})\\) and outputs a point from a group \\(G_T \\subset F_{q^{12}}\\). That is, for a paring \\(e\\), \\(e: G_1 \\times G_2 \\rightarrow G_T\\)

\n

properties of pairing
\\( e(P, Q+R) = e(P,Q) \\cdot e(P,R) \\),
note as a way of memory, can think it as \\(P^{Q+R} = P^Q \\cdot P^R\\)
\\( e(P+S, R) = e(P,R) \\cdot e(S,R)\\)
From this, we can deduce that all of the following identities hold:
\\(e([a]P, [b]Q) = e(P, [b]Q)^a = e(P,Q)^{ab} = e(P, [a]Q)^b = e([b]P, [a]Q)\\)

\n

Embedding degree

The embedding degree, \\(k\\), is calculated as the smallest positive integer such that \\(r\\) divides \\(q^k -1\\). So, in the case of BLS12-381, \\(r\\) is a factor of \\(q^{12} -1\\), but not of any lower power.
The choice of an embedding degree is a balance between security and efficiency (as ever). On the security front, the embedding degree is also known as the security multiplier: a higher embedding degree makes the discrete logarithm problem harder to solve in
. However, a high embedding degree means we have to do field operations in high-degree extensions, which is clunky and inefficient.

\n

cofactor

A subgroup’s cofactor is the ratio of the size of the whole group to the size of the subgroup.

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
GroupCofactorEquationvalue(hex)
\\(G_1\\)\\(h_1\\)\\(\\frac{\\mathbf{x-1}^2}{3}\\)0x396c8c005555e
1568c00aaab0000aaab
\\(G_2\\)\\(h_2\\)\\(\\frac{\\mathbf{x}^8 -4\\mathbf{x}^7+5\\mathbf{x}^6-4\\mathbf{x}^4+6\\mathbf{x}^3-4\\mathbf{x}^2-4\\mathbf{x}+13}{9}\\)0x5d543a95414e7f1091d50792876a202cd91de
4547085abaa68a205b2e5a7ddfa628f1cb4d9e82ef2
1537e293a6691ae1616ec6e786f0c70cf1c38e31c7238e5
\n

note multiplying by the cofactor turns out to be a straightforward way to map any arbitrary point on the elliptic curve into the respective subgroup \\(G_1\\) or \\(G_2\\)

\n

Generators

\\(G_1\\) and \\(G_2\\) are cyclic groups of prime order, so any point (except the identity/point at infinity) is a generator. Thus, picking generators is just a matter of convention.
Generator points \\(G_1\\) and \\(G_2\\) are specified in decimal here

\n

These were chosen as follows:

\n
\n

The generators of \\(G_1\\) and \\(G_2\\) are computed by finding the lexicographically smallest valid x-coordinate, and its lexicographically smallest > y-coordinate and scaling it by the cofactor such that the result is not the point at infinity.

\n
\n

Foot notes

[1] The “trace zero subgroup” qualifies as an obscure incantation. Basically, the trace of a point is \\(\\sum_{i=0}^{k-1}(x^{q^i}, y^{q^i})\\), where \\(k=12\\) in our case. Understanding this involves stuff like the Frobenius endomorphism.

\n

references

\n"},{"title":"kzg polynomial commitment","date":"2023-12-03T03:19:44.000Z","_content":"\n\n\n\n## introduction\nKZG polynomial commitments was introduced by Kate, Zaverucha and Goldberg. It is called a commitment, because having sent the commitment value (an elliptic curve point) to someone (the verifier), the prover cannot change the polynomial they are working with.\n\n## Comparison to Merkle trees\nA Merkle tree is what cryptographers call a vector commitment: Using a Merkle tree of depth \\\\(d\\\\), you can compute a commitment to a vector. A polynomial commitment can be achived by making a merkle tree commitment of all polynomials coefficients. however, it is not efficient.\n\n## parings\nLet \\\\(\\mathbb{G_1}\\\\) and \\\\(\\mathbb{G_2}\\\\) be two elliptic curves wit a paring \\\\(e: \\mathbb{G_1} \\times \\mathbb{G_2} \\rightarrow \\mathbb{G_T}\\\\). Let \\\\(p\\\\) be the order of \\\\(\\mathbb{G_1}\\\\) and \\\\(\\mathbb{G_2}\\\\), and \\\\(G\\\\) and \\\\(H\\\\) be generators of \\\\(\\mathbb{G_1}\\\\) and \\\\(\\mathbb{G_2}\\\\). We will use a very useful shorhand notation\n\\\\( [x]_1 = xG \\in \\mathbb{G_1}\\\\), and \\\\([x]_2 = xH \\in \\mathbb{G_2} \\\\), for any \\\\(x \\in \\mathbb{F_p}\\\\)\n\n## Trusted setup\nLet’s assume we have a trusted setup, so that for some secret \\\\(s\\\\), the elements \\\\([s^i]_1\\\\) and \\\\([s^i]_2\\\\) are available to both prover and verifier for \\\\(i=0, ..., n-1\\\\). \n\nLet \\\\(p(X) = \\sum_{i=0}^{n}p_i X^i \\\\) be a polynomial, the prover can compute\n\n\n\\\\( \\left[ p(s) \\right]_1 = p_0[s^0]_1 + p_1[s^1]_1 + ... + p_n[s^n]_1 \\\\)\n\n\n## Kate commitment\nIn the Kate commitment scheme, the eleemnt \\\\(C = [p(s)]_1\\\\) is the commitment to the polynomial \\\\(p(X)\\\\). Could the prover (without knowing \\\\(s\\\\)) find another polynomial \\\\(q(X) \\neq p(X)\\\\) that has the same commitment. Let's assume that this were the case. Then it would mean \\\\([p(s) - q(s)]_1 = 0\\\\), implying \\\\(p(s) - q(s) = 0\\\\).\n\nNow, \\\\(r(X) = p(X) - q(X)\\\\) is itself a polynomial. we know that it's not constant because \\\\(p(X) \\neq q(X) \\\\). It is a well-known fact that any non-constant polynomial of degree \\\\(n\\\\) can have at most \\\\(n\\\\) zeroes. \nSince the prover doesn’t know \\\\(s\\\\), the only way they could achieve that \\\\(p(s) - q(s) = 0\\\\)\n is by making \\\\( p(X) - q(X)=0\\\\) in as many places as possible. But since they can do that in at most \\\\(n\\\\)\n places, as we’ve just proved, they are very unlikely to succeed: since \\\\(n\\\\) is much smaller than the order of the curve, \\\\(p\\\\). the probability that \\\\(s\\\\) will be one of the points they chose to make \\\\(p(X) = q(X)\\\\) will be vanishingly tiny.\n\n## Multiplying polynomials\nWhile elliptic curves themselves don’t allow polynomial multiplication, luckily we can do it with pairings: We have that\n\\\\[ e([a]_1, [b]_2) = e(G,H)^{(ab)} = [ab]_T\\\\]\nwhile we unfortunately can’t just multiply two field elements inside an elliptic curve and get their product as an elliptic curve element, we can multiply two field elements if we commited to them in different curves (one in \\\\(G_1\\\\) and one in \\\\(G_2\\\\), and the output is a \n\\\\(G_T\\\\) element.\n\nNoe let's say we want to prove that \\\\(p(z)=y\\\\). we will use the polynomial \\\\(p(X) - y\\\\). this polynomial is clearly zero at \\\\(z\\\\). Let \\\\(q(X)\\\\) be the polynomial \\\\(p(X) - y\\\\) divided by the linear factor \\\\(X-z\\\\), i.e\n\\\\[ q(X) = \\frac{p(X) - y}{X-z} \\\\]\nwhich is equivalent to saying that \\\\(q(X)(X-z) = p(X) - y\\\\)\n\n## Kate proofs\nnow a kate proof for the evaluation \\\\(p(z) = y\\\\) is defined as \\\\(\\pi = [q(s)]_1\\\\). remember the commitment to the polynomial \\\\(p(X)\\\\) is defined as \\\\(C=[p(s)]_1\\\\)\nthe verifier checks this proof using the following equation\n\\\\( e(\\pi, [s-z]_2) = e(C-[y]_1, H) \\\\)\n\nNote that the verifier can compute \\\\([s-z]_2\\\\), because it is just a combination of the element \\\\([s]_2\\\\)\n from the trusted setup and \\\\(z\\\\) is the point at which the polynomial is evaluated. \n\n## multiproofs\nSo far we have shown how to prove an evaluation of a polynomial at a single point. Note that this is already pretty amazing: You can show that some polynomial that could be any degree – say \\\\(2^{28}\\\\) – at some point takes a certain value, by only sending a single group element (that could be 48 bytes, for example in BLS12_381). The toy example of using Merkle trees as polynomial commitments would have needed to send \\\\(2^{28}\\\\) elements – all the coefficients of the polynomial.\n\nLet’s say we have a list of \\\\(k\\\\) points \\\\((z_0, y_0),(z_1, y_1), ..., (z_{k-1}, y_{k-1})\\\\). Using lagrange interpolation, we can find polynomial \\\\(I(X)\\\\) goes through all of these points.\n\nNow let's assume that we know \\\\(p(X)\\\\) goes through all these points. then the polynomial \\\\(p(X) - I(X)\\\\) will clearly be zero at each \\\\(z_0, z_1, ..., z_{k-1}\\\\). the zero polynomial is\n\\\\(Z(X) = (X-z_0)\\cdot (X-z_1)...(X-z_{k-1})\\\\)\nNow, we can compute the quotient\n\\\\(q(X) = \\frac{p(X)-I(X)}{Z(X)}\\\\)\nWe can now define the Kate multiproof for the evaluations \\\\((z_0, y_0),(z_1, y_1),..., (z_{k-1}, y_{k-1})\\\\): \\\\(\\pi = [q(s)]_1\\\\). note that this is still only one group element.\n\nNow, to check this, the verifier will also have to compute the interpolation polynomial \\\\(I(X)\\\\) and the zero polynomial Z(X). Using this, they can compute \\\\([Z(s)]_2\\\\) and \\\\([I(s)]_1\\\\), and thus verify the paring equation\n\\\\[ e(\\pi, [Z(s)]_2) = e(C-[I(s)]_1, H) \\\\]\n\n## Kate as a vector commitment\nWhile the Kate commitment scheme is designed as a polynomial commitment, it actually also makes a really nice vector commitment. Remember that a vector commitment commits to a vector \\\\(a_0, ..., a_{n-1}\\\\) and lets you prove that you committed to \\\\(a_i\\\\) for some \\\\(i\\\\). We can reproduce this using the Kate commitment scheme: Let \\\\(p(X)\\\\) be the polynomial that for all \\\\(i\\\\) evaluates as \\\\(p(i)=a_i\\\\). \nNow using this polynomial, we can prove any number of elements in the vector using just a single group element! Note how much more efficient (in terms of proof size) this is compared to Merkle trees\n\n## references\n[Dankrad Feist Post](https://dankradfeist.de/ethereum/2020/06/16/kate-polynomial-commitments.html)\n","source":"_posts/cryptography/zkp/kzg-commitment.md","raw":"---\ntitle: kzg polynomial commitment\ndate: 2023-12-03 11:19:44\ntags: [cryptography,zkp]\n---\n\n\n\n\n## introduction\nKZG polynomial commitments was introduced by Kate, Zaverucha and Goldberg. It is called a commitment, because having sent the commitment value (an elliptic curve point) to someone (the verifier), the prover cannot change the polynomial they are working with.\n\n## Comparison to Merkle trees\nA Merkle tree is what cryptographers call a vector commitment: Using a Merkle tree of depth \\\\(d\\\\), you can compute a commitment to a vector. A polynomial commitment can be achived by making a merkle tree commitment of all polynomials coefficients. however, it is not efficient.\n\n## parings\nLet \\\\(\\mathbb{G_1}\\\\) and \\\\(\\mathbb{G_2}\\\\) be two elliptic curves wit a paring \\\\(e: \\mathbb{G_1} \\times \\mathbb{G_2} \\rightarrow \\mathbb{G_T}\\\\). Let \\\\(p\\\\) be the order of \\\\(\\mathbb{G_1}\\\\) and \\\\(\\mathbb{G_2}\\\\), and \\\\(G\\\\) and \\\\(H\\\\) be generators of \\\\(\\mathbb{G_1}\\\\) and \\\\(\\mathbb{G_2}\\\\). We will use a very useful shorhand notation\n\\\\( [x]_1 = xG \\in \\mathbb{G_1}\\\\), and \\\\([x]_2 = xH \\in \\mathbb{G_2} \\\\), for any \\\\(x \\in \\mathbb{F_p}\\\\)\n\n## Trusted setup\nLet’s assume we have a trusted setup, so that for some secret \\\\(s\\\\), the elements \\\\([s^i]_1\\\\) and \\\\([s^i]_2\\\\) are available to both prover and verifier for \\\\(i=0, ..., n-1\\\\). \n\nLet \\\\(p(X) = \\sum_{i=0}^{n}p_i X^i \\\\) be a polynomial, the prover can compute\n\n\n\\\\( \\left[ p(s) \\right]_1 = p_0[s^0]_1 + p_1[s^1]_1 + ... + p_n[s^n]_1 \\\\)\n\n\n## Kate commitment\nIn the Kate commitment scheme, the eleemnt \\\\(C = [p(s)]_1\\\\) is the commitment to the polynomial \\\\(p(X)\\\\). Could the prover (without knowing \\\\(s\\\\)) find another polynomial \\\\(q(X) \\neq p(X)\\\\) that has the same commitment. Let's assume that this were the case. Then it would mean \\\\([p(s) - q(s)]_1 = 0\\\\), implying \\\\(p(s) - q(s) = 0\\\\).\n\nNow, \\\\(r(X) = p(X) - q(X)\\\\) is itself a polynomial. we know that it's not constant because \\\\(p(X) \\neq q(X) \\\\). It is a well-known fact that any non-constant polynomial of degree \\\\(n\\\\) can have at most \\\\(n\\\\) zeroes. \nSince the prover doesn’t know \\\\(s\\\\), the only way they could achieve that \\\\(p(s) - q(s) = 0\\\\)\n is by making \\\\( p(X) - q(X)=0\\\\) in as many places as possible. But since they can do that in at most \\\\(n\\\\)\n places, as we’ve just proved, they are very unlikely to succeed: since \\\\(n\\\\) is much smaller than the order of the curve, \\\\(p\\\\). the probability that \\\\(s\\\\) will be one of the points they chose to make \\\\(p(X) = q(X)\\\\) will be vanishingly tiny.\n\n## Multiplying polynomials\nWhile elliptic curves themselves don’t allow polynomial multiplication, luckily we can do it with pairings: We have that\n\\\\[ e([a]_1, [b]_2) = e(G,H)^{(ab)} = [ab]_T\\\\]\nwhile we unfortunately can’t just multiply two field elements inside an elliptic curve and get their product as an elliptic curve element, we can multiply two field elements if we commited to them in different curves (one in \\\\(G_1\\\\) and one in \\\\(G_2\\\\), and the output is a \n\\\\(G_T\\\\) element.\n\nNoe let's say we want to prove that \\\\(p(z)=y\\\\). we will use the polynomial \\\\(p(X) - y\\\\). this polynomial is clearly zero at \\\\(z\\\\). Let \\\\(q(X)\\\\) be the polynomial \\\\(p(X) - y\\\\) divided by the linear factor \\\\(X-z\\\\), i.e\n\\\\[ q(X) = \\frac{p(X) - y}{X-z} \\\\]\nwhich is equivalent to saying that \\\\(q(X)(X-z) = p(X) - y\\\\)\n\n## Kate proofs\nnow a kate proof for the evaluation \\\\(p(z) = y\\\\) is defined as \\\\(\\pi = [q(s)]_1\\\\). remember the commitment to the polynomial \\\\(p(X)\\\\) is defined as \\\\(C=[p(s)]_1\\\\)\nthe verifier checks this proof using the following equation\n\\\\( e(\\pi, [s-z]_2) = e(C-[y]_1, H) \\\\)\n\nNote that the verifier can compute \\\\([s-z]_2\\\\), because it is just a combination of the element \\\\([s]_2\\\\)\n from the trusted setup and \\\\(z\\\\) is the point at which the polynomial is evaluated. \n\n## multiproofs\nSo far we have shown how to prove an evaluation of a polynomial at a single point. Note that this is already pretty amazing: You can show that some polynomial that could be any degree – say \\\\(2^{28}\\\\) – at some point takes a certain value, by only sending a single group element (that could be 48 bytes, for example in BLS12_381). The toy example of using Merkle trees as polynomial commitments would have needed to send \\\\(2^{28}\\\\) elements – all the coefficients of the polynomial.\n\nLet’s say we have a list of \\\\(k\\\\) points \\\\((z_0, y_0),(z_1, y_1), ..., (z_{k-1}, y_{k-1})\\\\). Using lagrange interpolation, we can find polynomial \\\\(I(X)\\\\) goes through all of these points.\n\nNow let's assume that we know \\\\(p(X)\\\\) goes through all these points. then the polynomial \\\\(p(X) - I(X)\\\\) will clearly be zero at each \\\\(z_0, z_1, ..., z_{k-1}\\\\). the zero polynomial is\n\\\\(Z(X) = (X-z_0)\\cdot (X-z_1)...(X-z_{k-1})\\\\)\nNow, we can compute the quotient\n\\\\(q(X) = \\frac{p(X)-I(X)}{Z(X)}\\\\)\nWe can now define the Kate multiproof for the evaluations \\\\((z_0, y_0),(z_1, y_1),..., (z_{k-1}, y_{k-1})\\\\): \\\\(\\pi = [q(s)]_1\\\\). note that this is still only one group element.\n\nNow, to check this, the verifier will also have to compute the interpolation polynomial \\\\(I(X)\\\\) and the zero polynomial Z(X). Using this, they can compute \\\\([Z(s)]_2\\\\) and \\\\([I(s)]_1\\\\), and thus verify the paring equation\n\\\\[ e(\\pi, [Z(s)]_2) = e(C-[I(s)]_1, H) \\\\]\n\n## Kate as a vector commitment\nWhile the Kate commitment scheme is designed as a polynomial commitment, it actually also makes a really nice vector commitment. Remember that a vector commitment commits to a vector \\\\(a_0, ..., a_{n-1}\\\\) and lets you prove that you committed to \\\\(a_i\\\\) for some \\\\(i\\\\). We can reproduce this using the Kate commitment scheme: Let \\\\(p(X)\\\\) be the polynomial that for all \\\\(i\\\\) evaluates as \\\\(p(i)=a_i\\\\). \nNow using this polynomial, we can prove any number of elements in the vector using just a single group element! Note how much more efficient (in terms of proof size) this is compared to Merkle trees\n\n## references\n[Dankrad Feist Post](https://dankradfeist.de/ethereum/2020/06/16/kate-polynomial-commitments.html)\n","slug":"cryptography/zkp/kzg-commitment","published":1,"updated":"2023-12-11T10:07:27.075Z","_id":"clpoya0gr0000pksj12ah1gzj","comments":1,"layout":"post","photos":[],"link":"","content":"\n\n\n

introduction

KZG polynomial commitments was introduced by Kate, Zaverucha and Goldberg. It is called a commitment, because having sent the commitment value (an elliptic curve point) to someone (the verifier), the prover cannot change the polynomial they are working with.

\n

Comparison to Merkle trees

A Merkle tree is what cryptographers call a vector commitment: Using a Merkle tree of depth \\(d\\), you can compute a commitment to a vector. A polynomial commitment can be achived by making a merkle tree commitment of all polynomials coefficients. however, it is not efficient.

\n

parings

Let \\(\\mathbb{G_1}\\) and \\(\\mathbb{G_2}\\) be two elliptic curves wit a paring \\(e: \\mathbb{G_1} \\times \\mathbb{G_2} \\rightarrow \\mathbb{G_T}\\). Let \\(p\\) be the order of \\(\\mathbb{G_1}\\) and \\(\\mathbb{G_2}\\), and \\(G\\) and \\(H\\) be generators of \\(\\mathbb{G_1}\\) and \\(\\mathbb{G_2}\\). We will use a very useful shorhand notation
\\( [x]_1 = xG \\in \\mathbb{G_1}\\), and \\([x]_2 = xH \\in \\mathbb{G_2} \\), for any \\(x \\in \\mathbb{F_p}\\)

\n

Trusted setup

Let’s assume we have a trusted setup, so that for some secret \\(s\\), the elements \\([s^i]_1\\) and \\([s^i]_2\\) are available to both prover and verifier for \\(i=0, …, n-1\\).

\n

Let \\(p(X) = \\sum_{i=0}^{n}p_i X^i \\) be a polynomial, the prover can compute

\n

\\( \\left[ p(s) \\right]_1 = p_0[s^0]_1 + p_1[s^1]_1 + … + p_n[s^n]_1 \\)

\n

Kate commitment

In the Kate commitment scheme, the eleemnt \\(C = [p(s)]_1\\) is the commitment to the polynomial \\(p(X)\\). Could the prover (without knowing \\(s\\)) find another polynomial \\(q(X) \\neq p(X)\\) that has the same commitment. Let’s assume that this were the case. Then it would mean \\([p(s) - q(s)]_1 = 0\\), implying \\(p(s) - q(s) = 0\\).

\n

Now, \\(r(X) = p(X) - q(X)\\) is itself a polynomial. we know that it’s not constant because \\(p(X) \\neq q(X) \\). It is a well-known fact that any non-constant polynomial of degree \\(n\\) can have at most \\(n\\) zeroes.
Since the prover doesn’t know \\(s\\), the only way they could achieve that \\(p(s) - q(s) = 0\\)
is by making \\( p(X) - q(X)=0\\) in as many places as possible. But since they can do that in at most \\(n\\)
places, as we’ve just proved, they are very unlikely to succeed: since \\(n\\) is much smaller than the order of the curve, \\(p\\). the probability that \\(s\\) will be one of the points they chose to make \\(p(X) = q(X)\\) will be vanishingly tiny.

\n

Multiplying polynomials

While elliptic curves themselves don’t allow polynomial multiplication, luckily we can do it with pairings: We have that
\\[ e([a]_1, [b]_2) = e(G,H)^{(ab)} = [ab]_T\\]
while we unfortunately can’t just multiply two field elements inside an elliptic curve and get their product as an elliptic curve element, we can multiply two field elements if we commited to them in different curves (one in \\(G_1\\) and one in \\(G_2\\), and the output is a
\\(G_T\\) element.

\n

Noe let’s say we want to prove that \\(p(z)=y\\). we will use the polynomial \\(p(X) - y\\). this polynomial is clearly zero at \\(z\\). Let \\(q(X)\\) be the polynomial \\(p(X) - y\\) divided by the linear factor \\(X-z\\), i.e
\\[ q(X) = \\frac{p(X) - y}{X-z} \\]
which is equivalent to saying that \\(q(X)(X-z) = p(X) - y\\)

\n

Kate proofs

now a kate proof for the evaluation \\(p(z) = y\\) is defined as \\(\\pi = [q(s)]_1\\). remember the commitment to the polynomial \\(p(X)\\) is defined as \\(C=[p(s)]_1\\)
the verifier checks this proof using the following equation
\\( e(\\pi, [s-z]_2) = e(C-[y]_1, H) \\)

\n

Note that the verifier can compute \\([s-z]_2\\), because it is just a combination of the element \\([s]_2\\)
from the trusted setup and \\(z\\) is the point at which the polynomial is evaluated.

\n

multiproofs

So far we have shown how to prove an evaluation of a polynomial at a single point. Note that this is already pretty amazing: You can show that some polynomial that could be any degree – say \\(2^{28}\\) – at some point takes a certain value, by only sending a single group element (that could be 48 bytes, for example in BLS12_381). The toy example of using Merkle trees as polynomial commitments would have needed to send \\(2^{28}\\) elements – all the coefficients of the polynomial.

\n

Let’s say we have a list of \\(k\\) points \\((z_0, y_0),(z_1, y_1), …, (z_{k-1}, y_{k-1})\\). Using lagrange interpolation, we can find polynomial \\(I(X)\\) goes through all of these points.

\n

Now let’s assume that we know \\(p(X)\\) goes through all these points. then the polynomial \\(p(X) - I(X)\\) will clearly be zero at each \\(z_0, z_1, …, z_{k-1}\\). the zero polynomial is
\\(Z(X) = (X-z_0)\\cdot (X-z_1)…(X-z_{k-1})\\)
Now, we can compute the quotient
\\(q(X) = \\frac{p(X)-I(X)}{Z(X)}\\)
We can now define the Kate multiproof for the evaluations \\((z_0, y_0),(z_1, y_1),…, (z_{k-1}, y_{k-1})\\): \\(\\pi = [q(s)]_1\\). note that this is still only one group element.

\n

Now, to check this, the verifier will also have to compute the interpolation polynomial \\(I(X)\\) and the zero polynomial Z(X). Using this, they can compute \\([Z(s)]_2\\) and \\([I(s)]_1\\), and thus verify the paring equation
\\[ e(\\pi, [Z(s)]_2) = e(C-[I(s)]_1, H) \\]

\n

Kate as a vector commitment

While the Kate commitment scheme is designed as a polynomial commitment, it actually also makes a really nice vector commitment. Remember that a vector commitment commits to a vector \\(a_0, …, a_{n-1}\\) and lets you prove that you committed to \\(a_i\\) for some \\(i\\). We can reproduce this using the Kate commitment scheme: Let \\(p(X)\\) be the polynomial that for all \\(i\\) evaluates as \\(p(i)=a_i\\).
Now using this polynomial, we can prove any number of elements in the vector using just a single group element! Note how much more efficient (in terms of proof size) this is compared to Merkle trees

\n

references

Dankrad Feist Post

\n","site":{"data":{}},"excerpt":"","more":"\n\n\n

introduction

KZG polynomial commitments was introduced by Kate, Zaverucha and Goldberg. It is called a commitment, because having sent the commitment value (an elliptic curve point) to someone (the verifier), the prover cannot change the polynomial they are working with.

\n

Comparison to Merkle trees

A Merkle tree is what cryptographers call a vector commitment: Using a Merkle tree of depth \\(d\\), you can compute a commitment to a vector. A polynomial commitment can be achived by making a merkle tree commitment of all polynomials coefficients. however, it is not efficient.

\n

parings

Let \\(\\mathbb{G_1}\\) and \\(\\mathbb{G_2}\\) be two elliptic curves wit a paring \\(e: \\mathbb{G_1} \\times \\mathbb{G_2} \\rightarrow \\mathbb{G_T}\\). Let \\(p\\) be the order of \\(\\mathbb{G_1}\\) and \\(\\mathbb{G_2}\\), and \\(G\\) and \\(H\\) be generators of \\(\\mathbb{G_1}\\) and \\(\\mathbb{G_2}\\). We will use a very useful shorhand notation
\\( [x]_1 = xG \\in \\mathbb{G_1}\\), and \\([x]_2 = xH \\in \\mathbb{G_2} \\), for any \\(x \\in \\mathbb{F_p}\\)

\n

Trusted setup

Let’s assume we have a trusted setup, so that for some secret \\(s\\), the elements \\([s^i]_1\\) and \\([s^i]_2\\) are available to both prover and verifier for \\(i=0, …, n-1\\).

\n

Let \\(p(X) = \\sum_{i=0}^{n}p_i X^i \\) be a polynomial, the prover can compute

\n

\\( \\left[ p(s) \\right]_1 = p_0[s^0]_1 + p_1[s^1]_1 + … + p_n[s^n]_1 \\)

\n

Kate commitment

In the Kate commitment scheme, the eleemnt \\(C = [p(s)]_1\\) is the commitment to the polynomial \\(p(X)\\). Could the prover (without knowing \\(s\\)) find another polynomial \\(q(X) \\neq p(X)\\) that has the same commitment. Let’s assume that this were the case. Then it would mean \\([p(s) - q(s)]_1 = 0\\), implying \\(p(s) - q(s) = 0\\).

\n

Now, \\(r(X) = p(X) - q(X)\\) is itself a polynomial. we know that it’s not constant because \\(p(X) \\neq q(X) \\). It is a well-known fact that any non-constant polynomial of degree \\(n\\) can have at most \\(n\\) zeroes.
Since the prover doesn’t know \\(s\\), the only way they could achieve that \\(p(s) - q(s) = 0\\)
is by making \\( p(X) - q(X)=0\\) in as many places as possible. But since they can do that in at most \\(n\\)
places, as we’ve just proved, they are very unlikely to succeed: since \\(n\\) is much smaller than the order of the curve, \\(p\\). the probability that \\(s\\) will be one of the points they chose to make \\(p(X) = q(X)\\) will be vanishingly tiny.

\n

Multiplying polynomials

While elliptic curves themselves don’t allow polynomial multiplication, luckily we can do it with pairings: We have that
\\[ e([a]_1, [b]_2) = e(G,H)^{(ab)} = [ab]_T\\]
while we unfortunately can’t just multiply two field elements inside an elliptic curve and get their product as an elliptic curve element, we can multiply two field elements if we commited to them in different curves (one in \\(G_1\\) and one in \\(G_2\\), and the output is a
\\(G_T\\) element.

\n

Noe let’s say we want to prove that \\(p(z)=y\\). we will use the polynomial \\(p(X) - y\\). this polynomial is clearly zero at \\(z\\). Let \\(q(X)\\) be the polynomial \\(p(X) - y\\) divided by the linear factor \\(X-z\\), i.e
\\[ q(X) = \\frac{p(X) - y}{X-z} \\]
which is equivalent to saying that \\(q(X)(X-z) = p(X) - y\\)

\n

Kate proofs

now a kate proof for the evaluation \\(p(z) = y\\) is defined as \\(\\pi = [q(s)]_1\\). remember the commitment to the polynomial \\(p(X)\\) is defined as \\(C=[p(s)]_1\\)
the verifier checks this proof using the following equation
\\( e(\\pi, [s-z]_2) = e(C-[y]_1, H) \\)

\n

Note that the verifier can compute \\([s-z]_2\\), because it is just a combination of the element \\([s]_2\\)
from the trusted setup and \\(z\\) is the point at which the polynomial is evaluated.

\n

multiproofs

So far we have shown how to prove an evaluation of a polynomial at a single point. Note that this is already pretty amazing: You can show that some polynomial that could be any degree – say \\(2^{28}\\) – at some point takes a certain value, by only sending a single group element (that could be 48 bytes, for example in BLS12_381). The toy example of using Merkle trees as polynomial commitments would have needed to send \\(2^{28}\\) elements – all the coefficients of the polynomial.

\n

Let’s say we have a list of \\(k\\) points \\((z_0, y_0),(z_1, y_1), …, (z_{k-1}, y_{k-1})\\). Using lagrange interpolation, we can find polynomial \\(I(X)\\) goes through all of these points.

\n

Now let’s assume that we know \\(p(X)\\) goes through all these points. then the polynomial \\(p(X) - I(X)\\) will clearly be zero at each \\(z_0, z_1, …, z_{k-1}\\). the zero polynomial is
\\(Z(X) = (X-z_0)\\cdot (X-z_1)…(X-z_{k-1})\\)
Now, we can compute the quotient
\\(q(X) = \\frac{p(X)-I(X)}{Z(X)}\\)
We can now define the Kate multiproof for the evaluations \\((z_0, y_0),(z_1, y_1),…, (z_{k-1}, y_{k-1})\\): \\(\\pi = [q(s)]_1\\). note that this is still only one group element.

\n

Now, to check this, the verifier will also have to compute the interpolation polynomial \\(I(X)\\) and the zero polynomial Z(X). Using this, they can compute \\([Z(s)]_2\\) and \\([I(s)]_1\\), and thus verify the paring equation
\\[ e(\\pi, [Z(s)]_2) = e(C-[I(s)]_1, H) \\]

\n

Kate as a vector commitment

While the Kate commitment scheme is designed as a polynomial commitment, it actually also makes a really nice vector commitment. Remember that a vector commitment commits to a vector \\(a_0, …, a_{n-1}\\) and lets you prove that you committed to \\(a_i\\) for some \\(i\\). We can reproduce this using the Kate commitment scheme: Let \\(p(X)\\) be the polynomial that for all \\(i\\) evaluates as \\(p(i)=a_i\\).
Now using this polynomial, we can prove any number of elements in the vector using just a single group element! Note how much more efficient (in terms of proof size) this is compared to Merkle trees

\n

references

Dankrad Feist Post

\n"},{"title":"plonky2 code analysis","date":"2023-11-06T05:24:03.000Z","_content":"\n\n\n\n## introduction\nthis post is a source code analysis of https://github.com/0xPolygonZero/plonky2\n\n## 1. Config\n### 1.1 CircuitConfig\nthe default config is `Self::standard_recursion_config()` with value as below\n```rust\nSelf {\n num_wires: 135,\n num_routed_wires: 80, \n num_constants: 2,\n use_base_arithmetic_gate: true,\n security_bits: 100,\n num_challenges: 2,\n zero_knowledge: false,\n max_quotient_degree_factor: 8,\n fri_config: FriConfig {\n rate_bits: 3,\n cap_height: 4,\n proof_of_work_bits: 16,\n reduction_strategy: FriReductionStrategy::ConstantArityBits(4, 5),\n num_query_rounds: 28,\n },\n}\n```\n\n**notes**\n- *routable wire* It's a wire that can be connected to other gates' wires, rather than an \"advice\" wire which is for internal state. For example if we had a gate for computing x^8, we'd have two routable wires for the input and output, x and x^8. But if we wanted lower-degree constraints, we could add advice wires for the intermediate results x^2 and x^4\n\n- *num_routed_wires*, the default value is 80. For eample, for a arithmetic gate, it computes `const_0 * multiplicand_0 * multiplicand_1 + const_1 * addend`. i.e 2 wires for multiplicands and 1 wire for the addend and 1 for output. total is 4. therefore, the gate can have `num_routed_wires/4` operations, in this case, it is 20. the circuit builder will add gate when number of operations exceeds 20.\n\n### 1.2 FriConfig\n```rust\npub struct FriConfig {\n/// `rate = 2^{-rate_bits}`. default is 3\npub rate_bits: usize, \n\n/// Height of Merkle tree caps. default is 4\npub cap_height: usize,\n/// default is 16\npub proof_of_work_bits: u32,\n\npub reduction_strategy: FriReductionStrategy,\n\n/// Number of query rounds to perform. default is 28\npub num_query_rounds: usize,\n}\n```\n\n## 2. Gates\n### ArithmeticGate\n```rust\n/// A gate which can perform a weighted multiply-add, i.e. `result = c0 x y + c1 z`. If the config\n/// supports enough routed wires, it can support several such operations in one gate.\npub struct ArithmeticGate {\n /// Number of arithmetic operations performed by an arithmetic gate.\n pub num_ops: usize,\n}\n\n/// A gate which takes a single constant parameter and outputs that value.\npub struct ConstantGate {\n pub(crate) num_consts: usize,\n}\n```\n\n## 3. Circuit Builder\n```rust\npub struct CircuitBuilder, const D: usize> {\n pub config: CircuitConfig,\n\n /// A domain separator, which is included in the initial Fiat-Shamir seed. This is generally not\n /// needed, but can be used to ensure that proofs for one application are not valid for another.\n /// Defaults to the empty vector.\n domain_separator: Option>,\n\n /// The types of gates used in this circuit.\n gates: HashSet>,\n\n /// The concrete placement of each gate.\n pub(crate) gate_instances: Vec>,\n\n /// Targets to be made public.\n public_inputs: Vec,\n\n /// The next available index for a `VirtualTarget`.\n virtual_target_index: usize,\n\n copy_constraints: Vec,\n\n /// A tree of named scopes, used for debugging.\n context_log: ContextTree,\n\n /// Generators used to generate the witness.\n generators: Vec>,\n\n /// maps constant value -> ConstantTarget(a virtual target)\n constants_to_targets: HashMap,\n targets_to_constants: HashMap,\n\n /// Memoized results of `arithmetic` calls.\n pub(crate) base_arithmetic_results: HashMap, Target>,\n\n /// Memoized results of `arithmetic_extension` calls.\n pub(crate) arithmetic_results: HashMap, ExtensionTarget>,\n\n /// Map between gate type and the current gate of this type with available slots.\n current_slots: HashMap, CurrentSlot>,\n\n /// List of constant generators used to fill the constant wires.\n constant_generators: Vec>,\n\n /// Rows for each LUT: LookupWire contains: first `LookupGate`, first `LookupTableGate`, last `LookupTableGate`.\n lookup_rows: Vec,\n\n /// For each LUT index, vector of `(looking_in, looking_out)` pairs.\n lut_to_lookups: Vec,\n\n // Lookup tables in the form of `Vec<(input_value, output_value)>`.\n luts: Vec,\n\n /// Optional common data. When it is `Some(goal_data)`, the `build` function panics if the resulting\n /// common data doesn't equal `goal_data`.\n /// This is used in cyclic recursion.\n pub(crate) goal_common_data: Option>,\n\n /// Optional verifier data that is registered as public inputs.\n /// This is used in cyclic recursion to hold the circuit's own verifier key.\n pub(crate) verifier_data_public_input: Option,\n}\n```\n- **virtual_target**: This is not an actual wire in the witness, but just a target that help facilitate witness generation. In particular, a generator can assign a values to a virtual target, which can then be copied to other (virtual or concrete) targets. When we generate the final witness (a grid of wire values), these virtual targets will go away.\n\n### 3.1 methods\n```rust\n\n/**\n * @param k_is, the coset shift, g^{j}, j \\in [0,num_shifts - 1], num_shifts is equal to num_routed_wires (columns); e.q 100\n * @subgroup: the \\Omega domain. \\Omega^{i}, i \\in [0, N-1], N is number of gates (rows); e.q 8 (2^3)\n * @return(0): vector of the sigma polynomial (sigma polynomial is the copy constraint permutation); the size is equal to the number of cols\n * @return(1): the forest is a data structure representing all wires copy constraint. the collected wires forms a unique partition\n */\nfn sigma_vecs(&self, k_is: &[F], subgroup: &[F]) -> (Vec>, Forest) {}\n```\n\n### 3.2 functions\n```rust\n/// Finds a set of shifts that result in unique cosets for the multiplicative subgroup of size `2^subgroup_bits`.\n/// Let `g` be a generator of the entire multiplicative group. the result is just g^0, ..., g^(num_shifts - 1)\npub fn get_unique_coset_shifts(subgroup_size: usize, num_shifts: usize) -> Vec {}\n```\n\n## 4. permutation\n```rust\npub struct WirePartition {\n // each element of the vector is a vector of copy constraint\n partition: Vec>,\n}\n\n/// Disjoint Set Forest data-structure following .\npub struct Forest {\n /// A map of parent pointers, stored as indices. the parent value is the partition it belongs to\n /// those node with same parent value, means they are in the copy constraint \n pub(crate) parents: Vec,\n\n num_wires: usize,\n num_routed_wires: usize, // num_routed_wires is used to construct all wires\n degree: usize,\n}\n```\n\n## 5. hash\ndefined in `CircuitBuilder`\n```rust\n/// Hash function used for building Merkle trees.\ntype Hasher: Hasher;\n/// Algebraic hash function used for the challenger and hashing public inputs.\ntype InnerHasher: AlgebraicHasher;\n\n/// Trait for algebraic hash functions, built from a permutation using the sponge construction.\npub trait AlgebraicHasher: Hasher> {\n type AlgebraicPermutation: PlonkyPermutation;\n\n /// Circuit to conditionally swap two chunks of the inputs (useful in verifying Merkle proofs),\n /// then apply the permutation.\n fn permute_swapped(\n inputs: Self::AlgebraicPermutation,\n swap: BoolTarget,\n builder: &mut CircuitBuilder,\n ) -> Self::AlgebraicPermutation\n where\n F: RichField + Extendable;\n}\n```\n\n\n\n\n\n\n## Questions\n- CircuitBuilder: self.constant_generators\n- Gate: extra_constant_wires\n- Constant Gate\n```\n fn eval_unfiltered(&self, vars: EvaluationVars) -> Vec {\n (0..self.num_consts)\n .map(|i| {\n vars.local_constants[self.const_input(i)] - vars.local_wires[self.wire_output(i)]\n })\n .collect()\n }\n```","source":"_posts/cryptography/zkp/plonky2-code-analysis.md","raw":"---\ntitle: plonky2 code analysis\ndate: 2023-11-06 13:24:03\ntags: [cryptography,zkp]\n---\n\n\n\n\n## introduction\nthis post is a source code analysis of https://github.com/0xPolygonZero/plonky2\n\n## 1. Config\n### 1.1 CircuitConfig\nthe default config is `Self::standard_recursion_config()` with value as below\n```rust\nSelf {\n num_wires: 135,\n num_routed_wires: 80, \n num_constants: 2,\n use_base_arithmetic_gate: true,\n security_bits: 100,\n num_challenges: 2,\n zero_knowledge: false,\n max_quotient_degree_factor: 8,\n fri_config: FriConfig {\n rate_bits: 3,\n cap_height: 4,\n proof_of_work_bits: 16,\n reduction_strategy: FriReductionStrategy::ConstantArityBits(4, 5),\n num_query_rounds: 28,\n },\n}\n```\n\n**notes**\n- *routable wire* It's a wire that can be connected to other gates' wires, rather than an \"advice\" wire which is for internal state. For example if we had a gate for computing x^8, we'd have two routable wires for the input and output, x and x^8. But if we wanted lower-degree constraints, we could add advice wires for the intermediate results x^2 and x^4\n\n- *num_routed_wires*, the default value is 80. For eample, for a arithmetic gate, it computes `const_0 * multiplicand_0 * multiplicand_1 + const_1 * addend`. i.e 2 wires for multiplicands and 1 wire for the addend and 1 for output. total is 4. therefore, the gate can have `num_routed_wires/4` operations, in this case, it is 20. the circuit builder will add gate when number of operations exceeds 20.\n\n### 1.2 FriConfig\n```rust\npub struct FriConfig {\n/// `rate = 2^{-rate_bits}`. default is 3\npub rate_bits: usize, \n\n/// Height of Merkle tree caps. default is 4\npub cap_height: usize,\n/// default is 16\npub proof_of_work_bits: u32,\n\npub reduction_strategy: FriReductionStrategy,\n\n/// Number of query rounds to perform. default is 28\npub num_query_rounds: usize,\n}\n```\n\n## 2. Gates\n### ArithmeticGate\n```rust\n/// A gate which can perform a weighted multiply-add, i.e. `result = c0 x y + c1 z`. If the config\n/// supports enough routed wires, it can support several such operations in one gate.\npub struct ArithmeticGate {\n /// Number of arithmetic operations performed by an arithmetic gate.\n pub num_ops: usize,\n}\n\n/// A gate which takes a single constant parameter and outputs that value.\npub struct ConstantGate {\n pub(crate) num_consts: usize,\n}\n```\n\n## 3. Circuit Builder\n```rust\npub struct CircuitBuilder, const D: usize> {\n pub config: CircuitConfig,\n\n /// A domain separator, which is included in the initial Fiat-Shamir seed. This is generally not\n /// needed, but can be used to ensure that proofs for one application are not valid for another.\n /// Defaults to the empty vector.\n domain_separator: Option>,\n\n /// The types of gates used in this circuit.\n gates: HashSet>,\n\n /// The concrete placement of each gate.\n pub(crate) gate_instances: Vec>,\n\n /// Targets to be made public.\n public_inputs: Vec,\n\n /// The next available index for a `VirtualTarget`.\n virtual_target_index: usize,\n\n copy_constraints: Vec,\n\n /// A tree of named scopes, used for debugging.\n context_log: ContextTree,\n\n /// Generators used to generate the witness.\n generators: Vec>,\n\n /// maps constant value -> ConstantTarget(a virtual target)\n constants_to_targets: HashMap,\n targets_to_constants: HashMap,\n\n /// Memoized results of `arithmetic` calls.\n pub(crate) base_arithmetic_results: HashMap, Target>,\n\n /// Memoized results of `arithmetic_extension` calls.\n pub(crate) arithmetic_results: HashMap, ExtensionTarget>,\n\n /// Map between gate type and the current gate of this type with available slots.\n current_slots: HashMap, CurrentSlot>,\n\n /// List of constant generators used to fill the constant wires.\n constant_generators: Vec>,\n\n /// Rows for each LUT: LookupWire contains: first `LookupGate`, first `LookupTableGate`, last `LookupTableGate`.\n lookup_rows: Vec,\n\n /// For each LUT index, vector of `(looking_in, looking_out)` pairs.\n lut_to_lookups: Vec,\n\n // Lookup tables in the form of `Vec<(input_value, output_value)>`.\n luts: Vec,\n\n /// Optional common data. When it is `Some(goal_data)`, the `build` function panics if the resulting\n /// common data doesn't equal `goal_data`.\n /// This is used in cyclic recursion.\n pub(crate) goal_common_data: Option>,\n\n /// Optional verifier data that is registered as public inputs.\n /// This is used in cyclic recursion to hold the circuit's own verifier key.\n pub(crate) verifier_data_public_input: Option,\n}\n```\n- **virtual_target**: This is not an actual wire in the witness, but just a target that help facilitate witness generation. In particular, a generator can assign a values to a virtual target, which can then be copied to other (virtual or concrete) targets. When we generate the final witness (a grid of wire values), these virtual targets will go away.\n\n### 3.1 methods\n```rust\n\n/**\n * @param k_is, the coset shift, g^{j}, j \\in [0,num_shifts - 1], num_shifts is equal to num_routed_wires (columns); e.q 100\n * @subgroup: the \\Omega domain. \\Omega^{i}, i \\in [0, N-1], N is number of gates (rows); e.q 8 (2^3)\n * @return(0): vector of the sigma polynomial (sigma polynomial is the copy constraint permutation); the size is equal to the number of cols\n * @return(1): the forest is a data structure representing all wires copy constraint. the collected wires forms a unique partition\n */\nfn sigma_vecs(&self, k_is: &[F], subgroup: &[F]) -> (Vec>, Forest) {}\n```\n\n### 3.2 functions\n```rust\n/// Finds a set of shifts that result in unique cosets for the multiplicative subgroup of size `2^subgroup_bits`.\n/// Let `g` be a generator of the entire multiplicative group. the result is just g^0, ..., g^(num_shifts - 1)\npub fn get_unique_coset_shifts(subgroup_size: usize, num_shifts: usize) -> Vec {}\n```\n\n## 4. permutation\n```rust\npub struct WirePartition {\n // each element of the vector is a vector of copy constraint\n partition: Vec>,\n}\n\n/// Disjoint Set Forest data-structure following .\npub struct Forest {\n /// A map of parent pointers, stored as indices. the parent value is the partition it belongs to\n /// those node with same parent value, means they are in the copy constraint \n pub(crate) parents: Vec,\n\n num_wires: usize,\n num_routed_wires: usize, // num_routed_wires is used to construct all wires\n degree: usize,\n}\n```\n\n## 5. hash\ndefined in `CircuitBuilder`\n```rust\n/// Hash function used for building Merkle trees.\ntype Hasher: Hasher;\n/// Algebraic hash function used for the challenger and hashing public inputs.\ntype InnerHasher: AlgebraicHasher;\n\n/// Trait for algebraic hash functions, built from a permutation using the sponge construction.\npub trait AlgebraicHasher: Hasher> {\n type AlgebraicPermutation: PlonkyPermutation;\n\n /// Circuit to conditionally swap two chunks of the inputs (useful in verifying Merkle proofs),\n /// then apply the permutation.\n fn permute_swapped(\n inputs: Self::AlgebraicPermutation,\n swap: BoolTarget,\n builder: &mut CircuitBuilder,\n ) -> Self::AlgebraicPermutation\n where\n F: RichField + Extendable;\n}\n```\n\n\n\n\n\n\n## Questions\n- CircuitBuilder: self.constant_generators\n- Gate: extra_constant_wires\n- Constant Gate\n```\n fn eval_unfiltered(&self, vars: EvaluationVars) -> Vec {\n (0..self.num_consts)\n .map(|i| {\n vars.local_constants[self.const_input(i)] - vars.local_wires[self.wire_output(i)]\n })\n .collect()\n }\n```","slug":"cryptography/zkp/plonky2-code-analysis","published":1,"updated":"2023-12-12T07:12:04.799Z","_id":"clq0nd00w00004z7u3a6ubwbd","comments":1,"layout":"post","photos":[],"link":"","content":"\n\n\n

introduction

this post is a source code analysis of https://github.com/0xPolygonZero/plonky2

\n

1. Config

1.1 CircuitConfig

the default config is Self::standard_recursion_config() with value as below

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
Self {
  num_wires: 135,
  num_routed_wires: 80, 
  num_constants: 2,
  use_base_arithmetic_gate: true,
  security_bits: 100,
  num_challenges: 2,
  zero_knowledge: false,
  max_quotient_degree_factor: 8,
  fri_config: FriConfig {
      rate_bits: 3,
      cap_height: 4,
      proof_of_work_bits: 16,
      reduction_strategy: FriReductionStrategy::ConstantArityBits(4, 5),
      num_query_rounds: 28,
  },
}
\n\n

notes

\n
    \n

  • routable wire It’s a wire that can be connected to other gates’ wires, rather than an “advice” wire which is for internal state. For example if we had a gate for computing x^8, we’d have two routable wires for the input and output, x and x^8. But if we wanted lower-degree constraints, we could add advice wires for the intermediate results x^2 and x^4

    \n
    • \n

  • num_routed_wires, the default value is 80. For eample, for a arithmetic gate, it computes const_0 * multiplicand_0 * multiplicand_1 + const_1 * addend. i.e 2 wires for multiplicands and 1 wire for the addend and 1 for output. total is 4. therefore, the gate can have num_routed_wires/4 operations, in this case, it is 20. the circuit builder will add gate when number of operations exceeds 20.

    \n
    • \n
\n

1.2 FriConfig

1
2
3
4
5
6
7
8
9
10
11
12
13
14
pub struct FriConfig {
/// `rate = 2^{-rate_bits}`. default is 3
pub rate_bits: usize,  

/// Height of Merkle tree caps. default is 4
pub cap_height: usize,
/// default is 16
pub proof_of_work_bits: u32,

pub reduction_strategy: FriReductionStrategy,

/// Number of query rounds to perform. default is 28
pub num_query_rounds: usize,
}
\n\n

2. Gates

ArithmeticGate

1
2
3
4
5
6
7
8
9
10
11
/// A gate which can perform a weighted multiply-add, i.e. `result = c0 x y + c1 z`. If the config
/// supports enough routed wires, it can support several such operations in one gate.
pub struct ArithmeticGate {
    /// Number of arithmetic operations performed by an arithmetic gate.
    pub num_ops: usize,
}

/// A gate which takes a single constant parameter and outputs that value.
pub struct ConstantGate {
    pub(crate) num_consts: usize,
}
\n\n

3. Circuit Builder

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
pub struct CircuitBuilder<F: RichField + Extendable<D>, const D: usize> {
    pub config: CircuitConfig,

    /// A domain separator, which is included in the initial Fiat-Shamir seed. This is generally not
    /// needed, but can be used to ensure that proofs for one application are not valid for another.
    /// Defaults to the empty vector.
    domain_separator: Option<Vec<F>>,

    /// The types of gates used in this circuit.
    gates: HashSet<GateRef<F, D>>,

    /// The concrete placement of each gate.
    pub(crate) gate_instances: Vec<GateInstance<F, D>>,

    /// Targets to be made public.
    public_inputs: Vec<Target>,

    /// The next available index for a `VirtualTarget`.
    virtual_target_index: usize,

    copy_constraints: Vec<CopyConstraint>,

    /// A tree of named scopes, used for debugging.
    context_log: ContextTree,

    /// Generators used to generate the witness.
    generators: Vec<WitnessGeneratorRef<F, D>>,

    /// maps constant value -> ConstantTarget(a virtual target)
    constants_to_targets: HashMap<F, Target>,
    targets_to_constants: HashMap<Target, F>,

    /// Memoized results of `arithmetic` calls.
    pub(crate) base_arithmetic_results: HashMap<BaseArithmeticOperation<F>, Target>,

    /// Memoized results of `arithmetic_extension` calls.
    pub(crate) arithmetic_results: HashMap<ExtensionArithmeticOperation<F, D>, ExtensionTarget<D>>,

    /// Map between gate type and the current gate of this type with available slots.
    current_slots: HashMap<GateRef<F, D>, CurrentSlot<F, D>>,

    /// List of constant generators used to fill the constant wires.
    constant_generators: Vec<ConstantGenerator<F>>,

    /// Rows for each LUT: LookupWire contains: first `LookupGate`, first `LookupTableGate`, last `LookupTableGate`.
    lookup_rows: Vec<LookupWire>,

    /// For each LUT index, vector of `(looking_in, looking_out)` pairs.
    lut_to_lookups: Vec<Lookup>,

    // Lookup tables in the form of `Vec<(input_value, output_value)>`.
    luts: Vec<LookupTable>,

    /// Optional common data. When it is `Some(goal_data)`, the `build` function panics if the resulting
    /// common data doesn't equal `goal_data`.
    /// This is used in cyclic recursion.
    pub(crate) goal_common_data: Option<CommonCircuitData<F, D>>,

    /// Optional verifier data that is registered as public inputs.
    /// This is used in cyclic recursion to hold the circuit's own verifier key.
    pub(crate) verifier_data_public_input: Option<VerifierCircuitTarget>,
}
\n
    \n
  • virtual_target: This is not an actual wire in the witness, but just a target that help facilitate witness generation. In particular, a generator can assign a values to a virtual target, which can then be copied to other (virtual or concrete) targets. When we generate the final witness (a grid of wire values), these virtual targets will go away.
    • \n
\n

3.1 methods

1
2
3
4
5
6
7
8

/**
  * @param k_is, the coset shift, g^{j}, j \\in [0,num_shifts - 1], num_shifts is equal to num_routed_wires (columns); e.q 100
  * @subgroup: the \\Omega domain. \\Omega^{i}, i \\in [0, N-1], N is number of gates (rows); e.q 8 (2^3)
  * @return(0): vector of the sigma polynomial (sigma polynomial is the copy constraint permutation); the size is equal to the number of cols
  * @return(1): the forest is a data structure representing all wires copy constraint. the collected wires forms a unique partition
  */
fn sigma_vecs(&self, k_is: &[F], subgroup: &[F]) -> (Vec<PolynomialValues<F>>, Forest) {}
\n\n

3.2 functions

1
2
3
/// Finds a set of shifts that result in unique cosets for the multiplicative subgroup of size `2^subgroup_bits`.
/// Let `g` be a generator of the entire multiplicative group. the result is just g^0, ..., g^(num_shifts - 1)
pub fn get_unique_coset_shifts<F: Field>(subgroup_size: usize, num_shifts: usize) -> Vec<F> {}
\n\n

4. permutation

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
pub struct WirePartition {
    // each element of the vector is a vector of copy constraint
    partition: Vec<Vec<Wire>>,
}

/// Disjoint Set Forest data-structure following <https://en.wikipedia.org/wiki/Disjoint-set_data_structure>.
pub struct Forest {
    /// A map of parent pointers, stored as indices. the parent value is the partition it belongs to
    /// those node with same parent value, means they are in the copy constraint 
    pub(crate) parents: Vec<usize>,

    num_wires: usize,
    num_routed_wires: usize, // num_routed_wires is used to construct all wires
    degree: usize,
}
\n\n

5. hash

defined in CircuitBuilder

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
/// Hash function used for building Merkle trees.
type Hasher: Hasher<Self::F>;
/// Algebraic hash function used for the challenger and hashing public inputs.
type InnerHasher: AlgebraicHasher<Self::F>;

/// Trait for algebraic hash functions, built from a permutation using the sponge construction.
pub trait AlgebraicHasher<F: RichField>: Hasher<F, Hash = HashOut<F>> {
    type AlgebraicPermutation: PlonkyPermutation<Target>;

    /// Circuit to conditionally swap two chunks of the inputs (useful in verifying Merkle proofs),
    /// then apply the permutation.
    fn permute_swapped<const D: usize>(
        inputs: Self::AlgebraicPermutation,
        swap: BoolTarget,
        builder: &mut CircuitBuilder<F, D>,
    ) -> Self::AlgebraicPermutation
    where
        F: RichField + Extendable<D>;
}
\n\n\n\n\n\n\n

Questions

    \n
  • CircuitBuilder: self.constant_generators
    • \n
  • Gate: extra_constant_wires
    • \n
  • Constant Gate
    1
    2
    3
    4
    5
    6
    7
    fn eval_unfiltered(&self, vars: EvaluationVars<F, D>) -> Vec<F::Extension> {
        (0..self.num_consts)
            .map(|i| {
                vars.local_constants[self.const_input(i)] - vars.local_wires[self.wire_output(i)]
            })
            .collect()
    }
    • \n
\n","site":{"data":{}},"excerpt":"","more":"\n\n\n

introduction

this post is a source code analysis of https://github.com/0xPolygonZero/plonky2

\n

1. Config

1.1 CircuitConfig

the default config is Self::standard_recursion_config() with value as below

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
Self {
  num_wires: 135,
  num_routed_wires: 80, 
  num_constants: 2,
  use_base_arithmetic_gate: true,
  security_bits: 100,
  num_challenges: 2,
  zero_knowledge: false,
  max_quotient_degree_factor: 8,
  fri_config: FriConfig {
      rate_bits: 3,
      cap_height: 4,
      proof_of_work_bits: 16,
      reduction_strategy: FriReductionStrategy::ConstantArityBits(4, 5),
      num_query_rounds: 28,
  },
}
\n\n

notes

\n
    \n

  • routable wire It’s a wire that can be connected to other gates’ wires, rather than an “advice” wire which is for internal state. For example if we had a gate for computing x^8, we’d have two routable wires for the input and output, x and x^8. But if we wanted lower-degree constraints, we could add advice wires for the intermediate results x^2 and x^4

    \n
    • \n

  • num_routed_wires, the default value is 80. For eample, for a arithmetic gate, it computes const_0 * multiplicand_0 * multiplicand_1 + const_1 * addend. i.e 2 wires for multiplicands and 1 wire for the addend and 1 for output. total is 4. therefore, the gate can have num_routed_wires/4 operations, in this case, it is 20. the circuit builder will add gate when number of operations exceeds 20.

    \n
    • \n
\n

1.2 FriConfig

1
2
3
4
5
6
7
8
9
10
11
12
13
14
pub struct FriConfig {
/// `rate = 2^{-rate_bits}`. default is 3
pub rate_bits: usize,  

/// Height of Merkle tree caps. default is 4
pub cap_height: usize,
/// default is 16
pub proof_of_work_bits: u32,

pub reduction_strategy: FriReductionStrategy,

/// Number of query rounds to perform. default is 28
pub num_query_rounds: usize,
}
\n\n

2. Gates

ArithmeticGate

1
2
3
4
5
6
7
8
9
10
11
/// A gate which can perform a weighted multiply-add, i.e. `result = c0 x y + c1 z`. If the config
/// supports enough routed wires, it can support several such operations in one gate.
pub struct ArithmeticGate {
    /// Number of arithmetic operations performed by an arithmetic gate.
    pub num_ops: usize,
}

/// A gate which takes a single constant parameter and outputs that value.
pub struct ConstantGate {
    pub(crate) num_consts: usize,
}
\n\n

3. Circuit Builder

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
pub struct CircuitBuilder<F: RichField + Extendable<D>, const D: usize> {
    pub config: CircuitConfig,

    /// A domain separator, which is included in the initial Fiat-Shamir seed. This is generally not
    /// needed, but can be used to ensure that proofs for one application are not valid for another.
    /// Defaults to the empty vector.
    domain_separator: Option<Vec<F>>,

    /// The types of gates used in this circuit.
    gates: HashSet<GateRef<F, D>>,

    /// The concrete placement of each gate.
    pub(crate) gate_instances: Vec<GateInstance<F, D>>,

    /// Targets to be made public.
    public_inputs: Vec<Target>,

    /// The next available index for a `VirtualTarget`.
    virtual_target_index: usize,

    copy_constraints: Vec<CopyConstraint>,

    /// A tree of named scopes, used for debugging.
    context_log: ContextTree,

    /// Generators used to generate the witness.
    generators: Vec<WitnessGeneratorRef<F, D>>,

    /// maps constant value -> ConstantTarget(a virtual target)
    constants_to_targets: HashMap<F, Target>,
    targets_to_constants: HashMap<Target, F>,

    /// Memoized results of `arithmetic` calls.
    pub(crate) base_arithmetic_results: HashMap<BaseArithmeticOperation<F>, Target>,

    /// Memoized results of `arithmetic_extension` calls.
    pub(crate) arithmetic_results: HashMap<ExtensionArithmeticOperation<F, D>, ExtensionTarget<D>>,

    /// Map between gate type and the current gate of this type with available slots.
    current_slots: HashMap<GateRef<F, D>, CurrentSlot<F, D>>,

    /// List of constant generators used to fill the constant wires.
    constant_generators: Vec<ConstantGenerator<F>>,

    /// Rows for each LUT: LookupWire contains: first `LookupGate`, first `LookupTableGate`, last `LookupTableGate`.
    lookup_rows: Vec<LookupWire>,

    /// For each LUT index, vector of `(looking_in, looking_out)` pairs.
    lut_to_lookups: Vec<Lookup>,

    // Lookup tables in the form of `Vec<(input_value, output_value)>`.
    luts: Vec<LookupTable>,

    /// Optional common data. When it is `Some(goal_data)`, the `build` function panics if the resulting
    /// common data doesn't equal `goal_data`.
    /// This is used in cyclic recursion.
    pub(crate) goal_common_data: Option<CommonCircuitData<F, D>>,

    /// Optional verifier data that is registered as public inputs.
    /// This is used in cyclic recursion to hold the circuit's own verifier key.
    pub(crate) verifier_data_public_input: Option<VerifierCircuitTarget>,
}
\n
    \n
  • virtual_target: This is not an actual wire in the witness, but just a target that help facilitate witness generation. In particular, a generator can assign a values to a virtual target, which can then be copied to other (virtual or concrete) targets. When we generate the final witness (a grid of wire values), these virtual targets will go away.
    • \n
\n

3.1 methods

1
2
3
4
5
6
7
8

/**
  * @param k_is, the coset shift, g^{j}, j \\in [0,num_shifts - 1], num_shifts is equal to num_routed_wires (columns); e.q 100
  * @subgroup: the \\Omega domain. \\Omega^{i}, i \\in [0, N-1], N is number of gates (rows); e.q 8 (2^3)
  * @return(0): vector of the sigma polynomial (sigma polynomial is the copy constraint permutation); the size is equal to the number of cols
  * @return(1): the forest is a data structure representing all wires copy constraint. the collected wires forms a unique partition
  */
fn sigma_vecs(&self, k_is: &[F], subgroup: &[F]) -> (Vec<PolynomialValues<F>>, Forest) {}
\n\n

3.2 functions

1
2
3
/// Finds a set of shifts that result in unique cosets for the multiplicative subgroup of size `2^subgroup_bits`.
/// Let `g` be a generator of the entire multiplicative group. the result is just g^0, ..., g^(num_shifts - 1)
pub fn get_unique_coset_shifts<F: Field>(subgroup_size: usize, num_shifts: usize) -> Vec<F> {}
\n\n

4. permutation

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
pub struct WirePartition {
    // each element of the vector is a vector of copy constraint
    partition: Vec<Vec<Wire>>,
}

/// Disjoint Set Forest data-structure following <https://en.wikipedia.org/wiki/Disjoint-set_data_structure>.
pub struct Forest {
    /// A map of parent pointers, stored as indices. the parent value is the partition it belongs to
    /// those node with same parent value, means they are in the copy constraint 
    pub(crate) parents: Vec<usize>,

    num_wires: usize,
    num_routed_wires: usize, // num_routed_wires is used to construct all wires
    degree: usize,
}
\n\n

5. hash

defined in CircuitBuilder

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
/// Hash function used for building Merkle trees.
type Hasher: Hasher<Self::F>;
/// Algebraic hash function used for the challenger and hashing public inputs.
type InnerHasher: AlgebraicHasher<Self::F>;

/// Trait for algebraic hash functions, built from a permutation using the sponge construction.
pub trait AlgebraicHasher<F: RichField>: Hasher<F, Hash = HashOut<F>> {
    type AlgebraicPermutation: PlonkyPermutation<Target>;

    /// Circuit to conditionally swap two chunks of the inputs (useful in verifying Merkle proofs),
    /// then apply the permutation.
    fn permute_swapped<const D: usize>(
        inputs: Self::AlgebraicPermutation,
        swap: BoolTarget,
        builder: &mut CircuitBuilder<F, D>,
    ) -> Self::AlgebraicPermutation
    where
        F: RichField + Extendable<D>;
}
\n\n\n\n\n\n\n

Questions

    \n
  • CircuitBuilder: self.constant_generators
    • \n
  • Gate: extra_constant_wires
    • \n
  • Constant Gate
    1
    2
    3
    4
    5
    6
    7
    fn eval_unfiltered(&self, vars: EvaluationVars<F, D>) -> Vec<F::Extension> {
        (0..self.num_consts)
            .map(|i| {
                vars.local_constants[self.const_input(i)] - vars.local_wires[self.wire_output(i)]
            })
            .collect()
    }
    • \n
\n"},{"title":"polygon zkEVM","date":"2023-12-11T05:24:03.000Z","_content":"\n\n\n\n## zkProver\n### State Machines\nThe zkProver follows modularity of design to the extent that, except for a few components, it is mainly a cluster of State Machines. It has a total of thirteen (13) State Machines;\n- The Main State Machine\n- Secondary State Machines → Binary SM, Storage SM, Memory SM, Arithmetic SM, Keccak Function SM, PoseidonG SM,\n- Auxiliary State Machines → Padding-PG SM, Padding-KK SM, Bits2Field SM, Memory Align SM, Byte4 SM, ROM SM.\nDue to the modular design of zkProver, the Main State Machine can delegate as many of tasks as possible to other specialist State Machines. This heavily improves the efficiency of Main SM.\n![](/images/zkp/zkevm/polygon/state_machines.png)\n\n### Two Novel Languages For zkProver\n#### Zero-Knowledge Assembly(zkASM)\nAs an Assembly language, the Zero-Knowledge Assembly (or zkASM) language is specially designed to map instructions from the zkProver's Main State Machine to other State Machines. In case of the State Machines with firmware, zkASM is the Interpreter for the firmware.\n\n#### Polynomial Identity Language(PIL)\nThe Polynomial Identity Language (or PIL) is especially designed for the zkProver. Almost all state machines express computations in terms of polynomials. Therefore, state transitions in State Machines must satisfy computation-specific polynomial identities.\n\n### Microprocessor State Machines\nThere are two microprocessor-type state machines; the Main SM and the Storage SM. These two State Machines have the Firmware and the Hardware part. It is worth noting that each of these Microprocessor SM has its own ROM.\n\nThe Firmware part runs the zkASM language to set up the logic and rules, which are expressed in JSON format and stored in a ROM. The JSON-file is then parsed to the specific SM Executor, which executes Storage Actions in compliance with the rules and logic in the JSON-file.\n\nThe Hardware component, which uses the Polynomial Identity Language (PIL), defines constraints (or polynomial identities), expresses them in JSON format, and stores them in the accompanying JSON-file. These constraints are parsed to the specific SM Executor, because all computations must be executed in conformance to the polynomial identities.\n\n![](/images/zkp/zkevm/polygon/micro_processor.png)\n","source":"_posts/cryptography/zkp/polygon_zkevm.md","raw":"---\ntitle: polygon zkEVM\ndate: 2023-12-11 13:24:03\ntags: [cryptography,zkp]\n---\n\n\n\n\n## zkProver\n### State Machines\nThe zkProver follows modularity of design to the extent that, except for a few components, it is mainly a cluster of State Machines. It has a total of thirteen (13) State Machines;\n- The Main State Machine\n- Secondary State Machines → Binary SM, Storage SM, Memory SM, Arithmetic SM, Keccak Function SM, PoseidonG SM,\n- Auxiliary State Machines → Padding-PG SM, Padding-KK SM, Bits2Field SM, Memory Align SM, Byte4 SM, ROM SM.\nDue to the modular design of zkProver, the Main State Machine can delegate as many of tasks as possible to other specialist State Machines. This heavily improves the efficiency of Main SM.\n![](/images/zkp/zkevm/polygon/state_machines.png)\n\n### Two Novel Languages For zkProver\n#### Zero-Knowledge Assembly(zkASM)\nAs an Assembly language, the Zero-Knowledge Assembly (or zkASM) language is specially designed to map instructions from the zkProver's Main State Machine to other State Machines. In case of the State Machines with firmware, zkASM is the Interpreter for the firmware.\n\n#### Polynomial Identity Language(PIL)\nThe Polynomial Identity Language (or PIL) is especially designed for the zkProver. Almost all state machines express computations in terms of polynomials. Therefore, state transitions in State Machines must satisfy computation-specific polynomial identities.\n\n### Microprocessor State Machines\nThere are two microprocessor-type state machines; the Main SM and the Storage SM. These two State Machines have the Firmware and the Hardware part. It is worth noting that each of these Microprocessor SM has its own ROM.\n\nThe Firmware part runs the zkASM language to set up the logic and rules, which are expressed in JSON format and stored in a ROM. The JSON-file is then parsed to the specific SM Executor, which executes Storage Actions in compliance with the rules and logic in the JSON-file.\n\nThe Hardware component, which uses the Polynomial Identity Language (PIL), defines constraints (or polynomial identities), expresses them in JSON format, and stores them in the accompanying JSON-file. These constraints are parsed to the specific SM Executor, because all computations must be executed in conformance to the polynomial identities.\n\n![](/images/zkp/zkevm/polygon/micro_processor.png)\n","slug":"cryptography/zkp/polygon_zkevm","published":1,"updated":"2023-12-11T08:34:59.530Z","_id":"clq0nd00w00024z7u843t2waw","comments":1,"layout":"post","photos":[],"link":"","content":"\n\n\n

zkProver

State Machines

The zkProver follows modularity of design to the extent that, except for a few components, it is mainly a cluster of State Machines. It has a total of thirteen (13) State Machines;

\n
    \n
  • The Main State Machine
    • \n
  • Secondary State Machines → Binary SM, Storage SM, Memory SM, Arithmetic SM, Keccak Function SM, PoseidonG SM,
    • \n
  • Auxiliary State Machines → Padding-PG SM, Padding-KK SM, Bits2Field SM, Memory Align SM, Byte4 SM, ROM SM.
    Due to the modular design of zkProver, the Main State Machine can delegate as many of tasks as possible to other specialist State Machines. This heavily improves the efficiency of Main SM.
    • \n
\n

Two Novel Languages For zkProver

Zero-Knowledge Assembly(zkASM)

As an Assembly language, the Zero-Knowledge Assembly (or zkASM) language is specially designed to map instructions from the zkProver’s Main State Machine to other State Machines. In case of the State Machines with firmware, zkASM is the Interpreter for the firmware.

\n

Polynomial Identity Language(PIL)

The Polynomial Identity Language (or PIL) is especially designed for the zkProver. Almost all state machines express computations in terms of polynomials. Therefore, state transitions in State Machines must satisfy computation-specific polynomial identities.

\n

Microprocessor State Machines

There are two microprocessor-type state machines; the Main SM and the Storage SM. These two State Machines have the Firmware and the Hardware part. It is worth noting that each of these Microprocessor SM has its own ROM.

\n

The Firmware part runs the zkASM language to set up the logic and rules, which are expressed in JSON format and stored in a ROM. The JSON-file is then parsed to the specific SM Executor, which executes Storage Actions in compliance with the rules and logic in the JSON-file.

\n

The Hardware component, which uses the Polynomial Identity Language (PIL), defines constraints (or polynomial identities), expresses them in JSON format, and stores them in the accompanying JSON-file. These constraints are parsed to the specific SM Executor, because all computations must be executed in conformance to the polynomial identities.

\n

\n","site":{"data":{}},"excerpt":"","more":"\n\n\n

zkProver

State Machines

The zkProver follows modularity of design to the extent that, except for a few components, it is mainly a cluster of State Machines. It has a total of thirteen (13) State Machines;

\n
    \n
  • The Main State Machine
    • \n
  • Secondary State Machines → Binary SM, Storage SM, Memory SM, Arithmetic SM, Keccak Function SM, PoseidonG SM,
    • \n
  • Auxiliary State Machines → Padding-PG SM, Padding-KK SM, Bits2Field SM, Memory Align SM, Byte4 SM, ROM SM.
    Due to the modular design of zkProver, the Main State Machine can delegate as many of tasks as possible to other specialist State Machines. This heavily improves the efficiency of Main SM.
    • \n
\n

Two Novel Languages For zkProver

Zero-Knowledge Assembly(zkASM)

As an Assembly language, the Zero-Knowledge Assembly (or zkASM) language is specially designed to map instructions from the zkProver’s Main State Machine to other State Machines. In case of the State Machines with firmware, zkASM is the Interpreter for the firmware.

\n

Polynomial Identity Language(PIL)

The Polynomial Identity Language (or PIL) is especially designed for the zkProver. Almost all state machines express computations in terms of polynomials. Therefore, state transitions in State Machines must satisfy computation-specific polynomial identities.

\n

Microprocessor State Machines

There are two microprocessor-type state machines; the Main SM and the Storage SM. These two State Machines have the Firmware and the Hardware part. It is worth noting that each of these Microprocessor SM has its own ROM.

\n

The Firmware part runs the zkASM language to set up the logic and rules, which are expressed in JSON format and stored in a ROM. The JSON-file is then parsed to the specific SM Executor, which executes Storage Actions in compliance with the rules and logic in the JSON-file.

\n

The Hardware component, which uses the Polynomial Identity Language (PIL), defines constraints (or polynomial identities), expresses them in JSON format, and stores them in the accompanying JSON-file. These constraints are parsed to the specific SM Executor, because all computations must be executed in conformance to the polynomial identities.

\n

\n"}],"PostAsset":[],"PostCategory":[],"PostTag":[{"post_id":"clokyy8dm0005qwsja95zaawe","tag_id":"clokyy8dk0002qwsj0l5me3u1","_id":"clokyy8dm0008qwsj3znq2qrc"},{"post_id":"clokyy8dh0000qwsj6ebuelv1","tag_id":"clokyy8dk0002qwsj0l5me3u1","_id":"clokyy8dn000cqwsja7jo4in6"},{"post_id":"clokyy8dh0000qwsj6ebuelv1","tag_id":"clokyy8dm0006qwsjcajbbgh0","_id":"clokyy8do000eqwsj32y8gju9"},{"post_id":"clokyy8dj0001qwsj9waw412a","tag_id":"clokyy8dk0002qwsj0l5me3u1","_id":"clokyy8dp000hqwsj8nvnb73i"},{"post_id":"clokyy8do000gqwsj13yvh382","tag_id":"clokyy8do000fqwsjasxpecdm","_id":"clokyy8dp000kqwsj0nhw3uvb"},{"post_id":"clokyy8dm0009qwsj4fr9fx90","tag_id":"clokyy8do000fqwsjasxpecdm","_id":"clokyy8dq000qqwsjecokb6pk"},{"post_id":"clokyy8dp000nqwsjgbxocb3i","tag_id":"clokyy8do000fqwsjasxpecdm","_id":"clokyy8dq000sqwsjeqa36czr"},{"post_id":"clokyy8dn000bqwsj9gt33o5h","tag_id":"clokyy8do000fqwsjasxpecdm","_id":"clokyy8dr000vqwsj01jb05f6"},{"post_id":"clokyy8do000dqwsj2vh07e7k","tag_id":"clokyy8do000fqwsjasxpecdm","_id":"clokyy8dr000zqwsj0ef9b9hi"},{"post_id":"clokyy8dq000pqwsjfar5aa4q","tag_id":"clokyy8do000fqwsjasxpecdm","_id":"clokyy8dt001bqwsjgw34gzwh"},{"post_id":"clokyy8dq000pqwsjfar5aa4q","tag_id":"clokyy8dr0011qwsjdywmbc62","_id":"clokyy8dt001dqwsj92lifgwe"},{"post_id":"clokyy8dq000pqwsjfar5aa4q","tag_id":"clokyy8ds0016qwsjb4c37d4g","_id":"clokyy8dt001gqwsje8zvg1fo"},{"post_id":"clokyy8dq000rqwsj5r3jepey","tag_id":"clokyy8ds0019qwsjevwpamn0","_id":"clokyy8dt001iqwsjfpxe9uvo"},{"post_id":"clokyy8dq000uqwsj46jo8rlt","tag_id":"clokyy8ds0019qwsjevwpamn0","_id":"clokyy8dt001lqwsjfky4bmth"},{"post_id":"clokyy8dt001kqwsj843kh526","tag_id":"clokyy8dt001jqwsjbfai4oxz","_id":"clokyy8du001nqwsjfqy76zi3"},{"post_id":"clokyy8dr000wqwsjaarwgrmn","tag_id":"clokyy8dt001jqwsjbfai4oxz","_id":"clokyy8du001qqwsj91e9gqqo"},{"post_id":"clokyy8du001pqwsj1k4uhzw0","tag_id":"clokyy8dt001jqwsjbfai4oxz","_id":"clokyy8du001sqwsj12wcgz9b"},{"post_id":"clokyy8dr000yqwsj41fu3ebh","tag_id":"clokyy8dt001jqwsjbfai4oxz","_id":"clokyy8du001vqwsjf63p54qk"},{"post_id":"clokyy8du001rqwsj7syrcry3","tag_id":"clokyy8dt001jqwsjbfai4oxz","_id":"clokyy8dv001xqwsj30s8eqen"},{"post_id":"clokyy8dr0010qwsj4oml40f0","tag_id":"clokyy8dt001jqwsjbfai4oxz","_id":"clokyy8dv0020qwsj99twcuuo"},{"post_id":"clokyy8ds0012qwsj7v2de3x4","tag_id":"clokyy8dt001jqwsjbfai4oxz","_id":"clokyy8dv0024qwsj21muefxx"},{"post_id":"clokyy8dv0023qwsjbe4aeis9","tag_id":"clokyy8dk0002qwsj0l5me3u1","_id":"clokyy8dw0026qwsj1o9f4y0e"},{"post_id":"clokyy8dv0023qwsjbe4aeis9","tag_id":"clokyy8dm0006qwsjcajbbgh0","_id":"clokyy8dx0029qwsj74s5gr22"},{"post_id":"clokyy8ds0014qwsjeg0uevgt","tag_id":"clokyy8dt001jqwsjbfai4oxz","_id":"clokyy8dx002bqwsj5ynk8d2b"},{"post_id":"clokyy8dv0025qwsjg5vzbwcu","tag_id":"clokyy8dk0002qwsj0l5me3u1","_id":"clokyy8dx002eqwsjhwtt76q2"},{"post_id":"clokyy8dv0025qwsjg5vzbwcu","tag_id":"clokyy8dm0006qwsjcajbbgh0","_id":"clokyy8dx002fqwsj0pbe66d4"},{"post_id":"clokyy8dw0028qwsj2vul70i7","tag_id":"clokyy8dk0002qwsj0l5me3u1","_id":"clokyy8dx002hqwsjbsur18r9"},{"post_id":"clokyy8dw0028qwsj2vul70i7","tag_id":"clokyy8dm0006qwsjcajbbgh0","_id":"clokyy8dx002iqwsj6j5wcl99"},{"post_id":"clokyy8ds0015qwsjh6j437e0","tag_id":"clokyy8dt001jqwsjbfai4oxz","_id":"clokyy8dx002kqwsj3cl16h9h"},{"post_id":"clokyy8dx002aqwsjacvpcivw","tag_id":"clokyy8dm0006qwsjcajbbgh0","_id":"clokyy8dx002lqwsjc2g1h6d9"},{"post_id":"clokyy8dx002dqwsj8w7zb4o4","tag_id":"clokyy8dm0006qwsjcajbbgh0","_id":"clokyy8dy002nqwsj5zeu2uni"},{"post_id":"clokyy8ds0017qwsj9lqm1w2r","tag_id":"clokyy8dt001jqwsjbfai4oxz","_id":"clokyy8dy002oqwsj4nxi1rgg"},{"post_id":"clokyy8ds0018qwsj3eer6n3l","tag_id":"clokyy8dt001jqwsjbfai4oxz","_id":"clokyy8dy002qqwsjerqu7lkg"},{"post_id":"clokyy8ds001aqwsjdxc70zfv","tag_id":"clokyy8dt001jqwsjbfai4oxz","_id":"clokyy8dy002rqwsjcssn2d2r"},{"post_id":"clokyy8dt001cqwsj0grhcnus","tag_id":"clokyy8dt001jqwsjbfai4oxz","_id":"clokyy8dy002tqwsj7u0qesqk"},{"post_id":"clokyy8dt001fqwsjbi7t4e7s","tag_id":"clokyy8dt001jqwsjbfai4oxz","_id":"clokyy8dy002uqwsj8czk072q"},{"post_id":"clokyy8dt001hqwsj4zoehfes","tag_id":"clokyy8dt001jqwsjbfai4oxz","_id":"clokyy8dy002wqwsjbvlcg7am"},{"post_id":"clokyy8dt001mqwsjfk9x8s37","tag_id":"clokyy8dt001jqwsjbfai4oxz","_id":"clokyy8dy002yqwsj3edrfhml"},{"post_id":"clokyy8dt001mqwsjfk9x8s37","tag_id":"clokyy8dy002vqwsjeapz6z25","_id":"clokyy8dy002zqwsjbq0bbhwl"},{"post_id":"clokyy8dv001zqwsj9hdfghdd","tag_id":"clokyy8dy0033qwsjexf1205y","_id":"clokyy8dy0037qwsj1sde9p8c"},{"post_id":"clokyy8dv0021qwsjhghcad7b","tag_id":"clokyy8dy0036qwsjbbfo2mww","_id":"clokyy8dy0038qwsj3umr87jx"},{"post_id":"clokyy8e00039qwsjfxurcxan","tag_id":"clokyy8do000fqwsjasxpecdm","_id":"clokyy8e1003bqwsj7twkfcni"},{"post_id":"clokyy8e00039qwsjfxurcxan","tag_id":"clokyy8dy002xqwsjftg07glq","_id":"clokyy8e1003dqwsj4hncbhbd"},{"post_id":"clokyy8e1003aqwsj8bnledgy","tag_id":"clokyy8do000fqwsjasxpecdm","_id":"clokyy8e1003fqwsjc7vv6zug"},{"post_id":"clokyy8e1003aqwsj8bnledgy","tag_id":"clokyy8dy002xqwsjftg07glq","_id":"clokyy8e2003hqwsj7lka58og"},{"post_id":"clokyy8e1003cqwsjc3p416e4","tag_id":"clokyy8do000fqwsjasxpecdm","_id":"clokyy8e2003jqwsjcff88w6t"},{"post_id":"clokyy8e1003cqwsjc3p416e4","tag_id":"clokyy8dy002xqwsjftg07glq","_id":"clokyy8e2003lqwsjc5zwhw8w"},{"post_id":"clokyy8e1003eqwsj3oda50v5","tag_id":"clokyy8dm0006qwsjcajbbgh0","_id":"clokyy8e3003nqwsj8mk3ej5y"},{"post_id":"clokyy8e1003gqwsj9p1scg9k","tag_id":"clokyy8do000fqwsjasxpecdm","_id":"clokyy8e3003qqwsj322l2852"},{"post_id":"clokyy8e2003iqwsj23zw9035","tag_id":"clokyy8do000fqwsjasxpecdm","_id":"clokyy8e4003uqwsjgm41brc0"},{"post_id":"clokyy8e2003iqwsj23zw9035","tag_id":"clokyy8dy002xqwsjftg07glq","_id":"clokyy8e4003xqwsj9r2t2d31"},{"post_id":"clokyy8e2003kqwsjg4hf5noa","tag_id":"clokyy8dt001jqwsjbfai4oxz","_id":"clokyy8e4003yqwsj367c01en"},{"post_id":"clokyy8e2003mqwsj9ztifql4","tag_id":"clokyy8e3003pqwsj13nx4h92","_id":"clokyy8e40040qwsj6dqray98"},{"post_id":"clokyy8e3003oqwsj76cl15kp","tag_id":"clokyy8e4003wqwsj7uh112pz","_id":"clokyy8e40041qwsj93t1f6vk"},{"post_id":"clokyy8e3003rqwsje2592e0r","tag_id":"clokyy8e4003wqwsj7uh112pz","_id":"clokyy8e40043qwsj27jq1jfb"},{"post_id":"clokyy8e3003tqwsj3w7e71hn","tag_id":"clokyy8e4003wqwsj7uh112pz","_id":"clokyy8e40045qwsj471w5z55"},{"post_id":"clokyy8e4003vqwsj77aj2fxl","tag_id":"clokyy8e4003wqwsj7uh112pz","_id":"clokyy8e40046qwsjbv0r0qdg"},{"post_id":"clp8cm0vx0000l57u7eqtbi6s","tag_id":"clp8cm0vz0001l57uazgmgnu5","_id":"clp8cm0w40002l57u85wb4ln4"},{"post_id":"clp8cm0w50003l57ubl7e5d6y","tag_id":"clp8cm0vz0001l57uazgmgnu5","_id":"clp8cm0w60005l57ubyymd5zb"},{"post_id":"clp8cm0w60004l57uckd04dmz","tag_id":"clp8cm0vz0001l57uazgmgnu5","_id":"clp8cm0w70007l57ugr0oce1c"},{"post_id":"clp8cm0w60006l57u0fvme5s2","tag_id":"clp8cm0vz0001l57uazgmgnu5","_id":"clp8cm0w70008l57u46esdf2b"},{"post_id":"clp95py9p0000e37u3ivq1as7","tag_id":"clp95py9p0001e37ubed0fc6x","_id":"clp95py9u0002e37u04o36rzx"},{"post_id":"clp95py9w0003e37uhwx0baye","tag_id":"clp95py9p0001e37ubed0fc6x","_id":"clp95py9x0004e37uhe5ggxyr"},{"post_id":"clp3h29tl0000p97ug63eajx3","tag_id":"clp95py9p0001e37ubed0fc6x","_id":"clp95py9x0005e37uao1y6l14"},{"post_id":"clp9d5a6z0007e37u1cxf6c30","tag_id":"clp9d6e5q0008e37uhxcaa8dh","_id":"clp9d6e5r0009e37ub41xctyw"},{"post_id":"clphpvdof0000aj7u9gaz5qfy","tag_id":"clokyy8dr000xqwsj9q3g4o5b","_id":"clphpvdoi0001aj7u0ov372gu"},{"post_id":"clphpvdoj0003aj7ugctear78","tag_id":"clokyy8do000fqwsjasxpecdm","_id":"clphpvdoj0007aj7u7unw5mx1"},{"post_id":"clphpvdoj0003aj7ugctear78","tag_id":"clokyy8dy002xqwsjftg07glq","_id":"clphpvdoj0008aj7u0ltm0gbj"},{"post_id":"clphpvdoj0005aj7ug6vbhr5l","tag_id":"clokyy8do000fqwsjasxpecdm","_id":"clphpvdoj0009aj7u3ktze2r0"},{"post_id":"clphpvdoj0005aj7ug6vbhr5l","tag_id":"clokyy8dy002xqwsjftg07glq","_id":"clphpvdoj000aaj7u7n244zgr"},{"post_id":"clphpvdok000baj7u3i3te2rg","tag_id":"clokyy8do000fqwsjasxpecdm","_id":"clphpvdol000daj7ubp814h2h"},{"post_id":"clphpvdok000baj7u3i3te2rg","tag_id":"clokyy8dy002xqwsjftg07glq","_id":"clphpvdol000eaj7ucw8nbcww"},{"post_id":"clphpvdok000caj7ubjq53ldn","tag_id":"clokyy8do000fqwsjasxpecdm","_id":"clphpvdol000faj7ub7tte5bg"},{"post_id":"clphpvdok000caj7ubjq53ldn","tag_id":"clokyy8dy002xqwsjftg07glq","_id":"clphpvdol000gaj7u3wqldktg"},{"post_id":"clpi1wlcr0000067ufpzn63zv","tag_id":"clp9d6e5q0008e37uhxcaa8dh","_id":"clpi1x45e00001o7u7c8k8j51"},{"post_id":"clokyy8e1003gqwsj9p1scg9k","tag_id":"clpm1qei20001hpsj09vdgxjm","_id":"clpm1qei80003hpsj7yjgb3rc"},{"post_id":"clpm1qei10000hpsj4saxbd96","tag_id":"clokyy8do000fqwsjasxpecdm","_id":"clpm1qei80004hpsjbxmcfbmj"},{"post_id":"clpm1qei10000hpsj4saxbd96","tag_id":"clpm1qei20001hpsj09vdgxjm","_id":"clpm1qei80005hpsj2mk298gs"},{"post_id":"clpoya0gr0000pksj12ah1gzj","tag_id":"clokyy8do000fqwsjasxpecdm","_id":"clpoya0gu0001pksj5tpz36bl"},{"post_id":"clpoya0gr0000pksj12ah1gzj","tag_id":"clokyy8dy002xqwsjftg07glq","_id":"clpoya0gu0002pksjh66nabe3"},{"post_id":"clp96plbb0006e37uei4tcdkw","tag_id":"clp9d6e5q0008e37uhxcaa8dh","_id":"clq0nd00w00014z7ugvhxfsgu"},{"post_id":"clq0nd00w00004z7u3a6ubwbd","tag_id":"clokyy8do000fqwsjasxpecdm","_id":"clq0nd00x00034z7uba2abgjb"},{"post_id":"clq0nd00w00004z7u3a6ubwbd","tag_id":"clokyy8dy002xqwsjftg07glq","_id":"clq0nd00x00044z7u86wqcako"},{"post_id":"clq0nd00w00024z7u843t2waw","tag_id":"clokyy8do000fqwsjasxpecdm","_id":"clq0nd00x00054z7u1coxge4a"},{"post_id":"clq0nd00w00024z7u843t2waw","tag_id":"clokyy8dy002xqwsjftg07glq","_id":"clq0nd00x00064z7ueqeb329q"}],"Tag":[{"name":"blockchain","_id":"clokyy8dk0002qwsj0l5me3u1"},{"name":"geth","_id":"clokyy8dm0006qwsjcajbbgh0"},{"name":"cryptography","_id":"clokyy8do000fqwsjasxpecdm"},{"name":"number_theory","_id":"clokyy8dr000xqwsj9q3g4o5b"},{"name":"mpc","_id":"clokyy8dr0011qwsjdywmbc62"},{"name":"ecdsa","_id":"clokyy8ds0016qwsjb4c37d4g"},{"name":"golang","_id":"clokyy8ds0019qwsjevwpamn0"},{"name":"rust","_id":"clokyy8dt001jqwsjbfai4oxz"},{"name":"cargo","_id":"clokyy8dy002vqwsjeapz6z25"},{"name":"zkp","_id":"clokyy8dy002xqwsjftg07glq"},{"name":"tool","_id":"clokyy8dy0033qwsjexf1205y"},{"name":"math","_id":"clokyy8dy0036qwsjbbfo2mww"},{"name":"rust-crate","_id":"clokyy8e3003pqwsj13nx4h92"},{"name":"rust-std","_id":"clokyy8e4003wqwsj7uh112pz"},{"name":"cpp","_id":"clp8cm0vz0001l57uazgmgnu5"},{"name":"cuda","_id":"clp95py9p0001e37ubed0fc6x"},{"name":"arithmatic","_id":"clp9d6e5q0008e37uhxcaa8dh"},{"name":"ec","_id":"clpm1qei20001hpsj09vdgxjm"}]}} \ No newline at end of file +{"meta":{"version":1,"warehouse":"4.0.2"},"models":{"Asset":[{"_id":"node_modules/hexo-theme-landscape/source/css/style.styl","path":"css/style.styl","modified":0,"renderable":1},{"_id":"node_modules/hexo-theme-landscape/source/js/jquery-3.4.1.min.js","path":"js/jquery-3.4.1.min.js","modified":0,"renderable":1},{"_id":"node_modules/hexo-theme-landscape/source/js/script.js","path":"js/script.js","modified":0,"renderable":1},{"_id":"node_modules/hexo-theme-landscape/source/fancybox/jquery.fancybox.min.css","path":"fancybox/jquery.fancybox.min.css","modified":0,"renderable":1},{"_id":"node_modules/hexo-theme-landscape/source/fancybox/jquery.fancybox.min.js","path":"fancybox/jquery.fancybox.min.js","modified":0,"renderable":1},{"_id":"node_modules/hexo-theme-landscape/source/css/fonts/FontAwesome.otf","path":"css/fonts/FontAwesome.otf","modified":0,"renderable":1},{"_id":"node_modules/hexo-theme-landscape/source/css/fonts/fontawesome-webfont.eot","path":"css/fonts/fontawesome-webfont.eot","modified":0,"renderable":1},{"_id":"node_modules/hexo-theme-landscape/source/css/fonts/fontawesome-webfont.ttf","path":"css/fonts/fontawesome-webfont.ttf","modified":0,"renderable":1},{"_id":"node_modules/hexo-theme-landscape/source/css/fonts/fontawesome-webfont.svg","path":"css/fonts/fontawesome-webfont.svg","modified":0,"renderable":1},{"_id":"node_modules/hexo-theme-landscape/source/css/fonts/fontawesome-webfont.woff","path":"css/fonts/fontawesome-webfont.woff","modified":0,"renderable":1},{"_id":"node_modules/hexo-theme-landscape/source/css/images/banner.jpg","path":"css/images/banner.jpg","modified":0,"renderable":1},{"_id":"node_modules/hexo-theme-landscape/source/css/fonts/fontawesome-webfont.woff2","path":"css/fonts/fontawesome-webfont.woff2","modified":0,"renderable":1},{"_id":"source/2017-552.pdf","path":"2017-552.pdf","modified":0,"renderable":0},{"_id":"source/images/evm.drawio.google.png","path":"images/evm.drawio.google.png","modified":0,"renderable":0},{"_id":"source/images/evm.layout.png","path":"images/evm.layout.png","modified":0,"renderable":0},{"_id":"source/images/geth_starts.drawio.png","path":"images/geth_starts.drawio.png","modified":0,"renderable":0},{"_id":"source/images/kademlia.locating.png","path":"images/kademlia.locating.png","modified":0,"renderable":0},{"_id":"source/images/kademlia.onlineprob.png","path":"images/kademlia.onlineprob.png","modified":0,"renderable":0},{"_id":"source/images/kademlia.subtree.png","path":"images/kademlia.subtree.png","modified":0,"renderable":0},{"_id":"source/images/mpt.state.ref.png","path":"images/mpt.state.ref.png","modified":0,"renderable":0},{"_id":"source/images/mpt.png","path":"images/mpt.png","modified":0,"renderable":0},{"_id":"source/images/trie.prefix.png","path":"images/trie.prefix.png","modified":0,"renderable":0},{"_id":"source/images/geth/sync.mode.jpg","path":"images/geth/sync.mode.jpg","modified":0,"renderable":0},{"_id":"source/images/two_party_ecdsa/paillier_enc.png","path":"images/two_party_ecdsa/paillier_enc.png","modified":0,"renderable":0},{"_id":"source/images/two_party_ecdsa/schnorr_ecdsa_comparison.png","path":"images/two_party_ecdsa/schnorr_ecdsa_comparison.png","modified":0,"renderable":0},{"_id":"source/images/paillier/carmichael_thorem.png","path":"images/paillier/carmichael_thorem.png","modified":0,"renderable":0},{"_id":"source/images/paillier/carmichael_thorem_2.png","path":"images/paillier/carmichael_thorem_2.png","modified":0,"renderable":0},{"_id":"source/images/paillier/homomorphic_addition.png","path":"images/paillier/homomorphic_addition.png","modified":0,"renderable":0},{"_id":"source/images/paillier/homomorphic_mul.png","path":"images/paillier/homomorphic_mul.png","modified":0,"renderable":0},{"_id":"source/images/cryptography/elliptic_curve/paring_ec.png","path":"images/cryptography/elliptic_curve/paring_ec.png","modified":0,"renderable":0},{"_id":"source/images/cryptography/elliptic_curve/point_addition.webp","path":"images/cryptography/elliptic_curve/point_addition.webp","modified":0,"renderable":0},{"_id":"source/images/cryptography/zkp/checking_qap.png","path":"images/cryptography/zkp/checking_qap.png","modified":0,"renderable":0},{"_id":"source/images/cryptography/zkp/lagrange_interpolating.png","path":"images/cryptography/zkp/lagrange_interpolating.png","modified":0,"renderable":0},{"_id":"source/images/cryptography/zkp/r1cs.png","path":"images/cryptography/zkp/r1cs.png","modified":0,"renderable":0},{"_id":"source/images/rust/macros/16.compile_process.png","path":"images/rust/macros/16.compile_process.png","modified":0,"renderable":0},{"_id":"source/images/rust/memory/trait_object_memory.png","path":"images/rust/memory/trait_object_memory.png","modified":0,"renderable":0},{"_id":"source/images/rust/ownership/move-string-2.png","path":"images/rust/ownership/move-string-2.png","modified":0,"renderable":0},{"_id":"source/images/rust/ownership/move-string.png","path":"images/rust/ownership/move-string.png","modified":0,"renderable":0},{"_id":"source/images/rust/pointers/cycle.ref.png","path":"images/rust/pointers/cycle.ref.png","modified":0,"renderable":0},{"_id":"source/images/math/fourier_series/f_x.png","path":"images/math/fourier_series/f_x.png","modified":0,"renderable":0},{"_id":"source/images/rust/pointers/rc.png","path":"images/rust/pointers/rc.png","modified":0,"renderable":0},{"_id":"source/images/cryptography/rsa/rsa_signature.png","path":"images/cryptography/rsa/rsa_signature.png","modified":0,"renderable":0},{"_id":"source/images/zkp/stark/commitment.png","path":"images/zkp/stark/commitment.png","modified":0,"renderable":0},{"_id":"source/images/zkp/stark/decommitment.png","path":"images/zkp/stark/decommitment.png","modified":0,"renderable":0},{"_id":"source/images/zkp/stark/fri.png","path":"images/zkp/stark/fri.png","modified":0,"renderable":0},{"_id":"source/images/zkp/stark/fri_example.png","path":"images/zkp/stark/fri_example.png","modified":0,"renderable":0},{"_id":"source/images/zkp/stark/trace_to_CP.png","path":"images/zkp/stark/trace_to_CP.png","modified":0,"renderable":0},{"_id":"source/images/zkp/plonk/plonk_circuit_to_trace.png","path":"images/zkp/plonk/plonk_circuit_to_trace.png","modified":0,"renderable":0},{"_id":"source/images/zkp/plonk/zero_test.png","path":"images/zkp/plonk/zero_test.png","modified":0,"renderable":0},{"_id":"source/images/cuda/cuda_stream_example.png","path":"images/cuda/cuda_stream_example.png","modified":0,"renderable":0},{"_id":"source/images/arithmatic/multiple_precision_addition.png","path":"images/arithmatic/multiple_precision_addition.png","modified":0,"renderable":0},{"_id":"source/images/arithmatic/radix_b_repesentation.png","path":"images/arithmatic/radix_b_repesentation.png","modified":0,"renderable":0},{"_id":"source/images/zkp/plonk/gate_evaluation_zero_test.png","path":"images/zkp/plonk/gate_evaluation_zero_test.png","modified":0,"renderable":0},{"_id":"source/images/zkp/plonk/permutation_check.png","path":"images/zkp/plonk/permutation_check.png","modified":0,"renderable":0},{"_id":"source/images/zkp/plonk/prod_check_lemma.png","path":"images/zkp/plonk/prod_check_lemma.png","modified":0,"renderable":0},{"_id":"source/images/zkp/plonk/prod_check_prove_verify.png","path":"images/zkp/plonk/prod_check_prove_verify.png","modified":0,"renderable":0},{"_id":"source/images/zkp/plonk/prescribed_perm_check_problem.png","path":"images/zkp/plonk/prescribed_perm_check_problem.png","modified":0,"renderable":0},{"_id":"source/images/zkp/plonk/prescribed_perm_check_problem_quadratic.png","path":"images/zkp/plonk/prescribed_perm_check_problem_quadratic.png","modified":0,"renderable":0},{"_id":"source/images/zkp/plonk/prescribed_perm_check_problem_reduce.png","path":"images/zkp/plonk/prescribed_perm_check_problem_reduce.png","modified":0,"renderable":0},{"_id":"source/images/zkp/plonk/prescribed_perm_check_problem_prove_verify.png","path":"images/zkp/plonk/prescribed_perm_check_problem_prove_verify.png","modified":0,"renderable":0},{"_id":"source/images/zkp/plonk/prescribed_perm_check_problem_complete.png","path":"images/zkp/plonk/prescribed_perm_check_problem_complete.png","modified":0,"renderable":0},{"_id":"source/images/zkp/plonk/copy_constraint_example.png","path":"images/zkp/plonk/copy_constraint_example.png","modified":0,"renderable":0},{"_id":"source/images/zkp/plonk/custom_gate.png","path":"images/zkp/plonk/custom_gate.png","modified":0,"renderable":0},{"_id":"source/images/zkp/zkevm/polygon/state_machines.png","path":"images/zkp/zkevm/polygon/state_machines.png","modified":0,"renderable":0},{"_id":"source/images/zkp/zkevm/polygon/micro_processor.png","path":"images/zkp/zkevm/polygon/micro_processor.png","modified":0,"renderable":0}],"Cache":[{"_id":"source/_posts/MPT.md","hash":"1d0394f11c3361b4474dbe49ced5c5751144fe91","modified":1699158073732},{"_id":"source/_posts/blockchain.kademlia.md","hash":"0cb0522a114bc53d79f2db4a1bf2a02c29ef1c2f","modified":1699158073732},{"_id":"source/_posts/hello-world.md","hash":"8241e671f980a667f875b9a6493f17bd333a1cfb","modified":1699158073733},{"_id":"source/_posts/geth-fine-tune.md","hash":"5431cd654f7152fd5c590891a53568f54eec4125","modified":1699158073733},{"_id":"source/images/evm.layout.png","hash":"6927cb3b922b2bcec926ec5d797e7ea8ea2b5d00","modified":1699158073736},{"_id":"source/images/kademlia.onlineprob.png","hash":"98c55aa819ef7047b3749c89eb4546f0d799a239","modified":1699158073738},{"_id":"source/images/mpt.state.ref.png","hash":"ec747cf092820c0ab5f72d0f4f07f7705bb9ecf0","modified":1699158073740},{"_id":"source/_posts/blockchain/danksharding.md","hash":"ecf0cb650e0a0e7b251b86ccfbac535a6a217522","modified":1699158077033},{"_id":"source/_posts/cryptography/bls12-381.md","hash":"f3d4cd2f87eb82929b9725644bf607469e7f7526","modified":1699158213767},{"_id":"source/_posts/cryptography/cryptography-01-primitive-group-and-field.md","hash":"b109aef9a3c4ca55a7198c284cd007716b094044","modified":1699158073732},{"_id":"source/_posts/cryptography/cryptography-03-elliptic-curve.md","hash":"3ee7e6e7b609605f7c26d5bf1f7508771d6c7a95","modified":1699158077033},{"_id":"source/_posts/cryptography/cryptography-04-digital-signature.md","hash":"f2539c849c5e052c62123a7fde737ce4c0300134","modified":1699158077033},{"_id":"source/_posts/cryptography/cryptography-02-rsa.md","hash":"6c0bb57a988b036e8b8beca7343b69c025bc4ea7","modified":1699158077033},{"_id":"source/_posts/cryptography/kzg-commitment.md","hash":"9c7b821d2c4a0cb56c70c1a0711af74118253fbd","modified":1699158077034},{"_id":"source/_posts/golang/go-reflect.md","hash":"c2808bbd3bf422ab5679c246444975107b98fb1e","modified":1699158073733},{"_id":"source/_posts/cryptography/montgomery-multiplication.md","hash":"806262fecf990ff4c95eb40d954728b635724bdd","modified":1699158213767},{"_id":"source/_posts/cryptography/paillier-encryption.md","hash":"4e43aa2d126bcb334563262563071c764c3ae19f","modified":1699158073732},{"_id":"source/_posts/cryptography/two-party-ecdsa.md","hash":"9416016bb3f47864aeb888db208f63f93ec10f71","modified":1699158077034},{"_id":"source/_posts/golang/go-similar-concepts-comparison.md","hash":"5de26ef1a5907db4264ff5e491b75aa2dfb43af1","modified":1699158073733},{"_id":"source/_posts/rust/rust-02-basics.md","hash":"be1111d868417c54b8b019938b8144e8be35df1f","modified":1699158073734},{"_id":"source/_posts/rust/rust-01-resource.md","hash":"5e2c159128ccef35da0d3a2a72b6bb7e1c12fc73","modified":1699158073734},{"_id":"source/_posts/rust/rust-03-ownership.md","hash":"7d39498532088f438d76f1d0b42892bc985c0c95","modified":1699158073734},{"_id":"source/_posts/rust/rust-04-lifetime.md","hash":"d338498d7d159a3f94687082a10fc28ada706b16","modified":1699158073734},{"_id":"source/_posts/rust/rust-05-memory-layout.md","hash":"9a8c7dac3a23bca5f7692a3f6c0c8827dfd8c7e5","modified":1699158073734},{"_id":"source/_posts/rust/rust-07-macro.md","hash":"f6e51943ab413f5462baca1327c53ac20a8afcda","modified":1699158073734},{"_id":"source/_posts/rust/rust-09-functional.md","hash":"b2e1f9a46e02c5aa49ea19394e074777558a51eb","modified":1699158073735},{"_id":"source/_posts/rust/rust-06-smart-pointer.md","hash":"909ecf0ecf594651191e0953eca17aca7fa64669","modified":1699158073734},{"_id":"source/_posts/rust/rust-08-project-management.md","hash":"2c1ab1e7b8c8510935a694e33aa2c676437f0fd1","modified":1699158073734},{"_id":"source/_posts/rust/rust-10-concurrency.md","hash":"5bba6d7c1b0e3a24f07a4899f1162a93af8ad5b1","modified":1699158073735},{"_id":"source/_posts/rust/rust-async.md","hash":"f8b896f06752fcdb3c9fa34d2ed0ba958aef9c49","modified":1699158073735},{"_id":"source/_posts/rust/rust-cargo-all-in-one.md","hash":"27eb587fe52f0461e1e30ed82b5d10a806e168f0","modified":1699158073735},{"_id":"source/_posts/rust/rust-similar-concepts-comparison.md","hash":"17c7d67efd6315828a09762c633e7f8a0c9cdee2","modified":1699158073735},{"_id":"source/_posts/rust/rust-sugar.md","hash":"dfa5a3f7bfd985118b97ae9055da496778b604e8","modified":1699158073735},{"_id":"source/_posts/rust/rust-tools.md","hash":"3019a116f156d05a95fd8fc76fa8b1380f778f24","modified":1699158073735},{"_id":"source/_posts/rust/rust-cargo-doc.md","hash":"52303be5d9e342444a4cb661fa418ab68b28d640","modified":1699158073735},{"_id":"source/_posts/zkp/stark-101.md","hash":"867f0c912e4f6abe1512848408f7afebae04b536","modified":1699158213767},{"_id":"source/_posts/zkp/understanding-plonk.md","hash":"4d501404bae2e4674c6b2d519014097c6b506807","modified":1699669060225},{"_id":"source/images/geth/sync.mode.jpg","hash":"657cc148abc14f01e7483265c67936e73f79d0e4","modified":1699158073737},{"_id":"source/_posts/tools/markdown_and_latex.md","hash":"50806cfd650f2c031a90d1860a688fa8320517a3","modified":1701410801564},{"_id":"source/_posts/math/fourier-series.md","hash":"4831e39540e229ae7d16972e299819adfff264dc","modified":1701013907630},{"_id":"source/images/two_party_ecdsa/paillier_enc.png","hash":"132cdc74eb588951d1bd347b1b6e825912bc0460","modified":1699158073744},{"_id":"source/images/paillier/homomorphic_addition.png","hash":"26b0e4a070a7e29710e2ceace09bbdb79b3a883e","modified":1699158073741},{"_id":"source/images/paillier/homomorphic_mul.png","hash":"e8a2420501140a8d48db202065aa221df92f19dc","modified":1699158073741},{"_id":"source/_posts/geth/code_analysis/geth.0.get.start.md","hash":"6cd5256bae5e43f3dfcd341e27c52eccf8848f60","modified":1699158073733},{"_id":"source/_posts/geth/code_analysis/geth.1.rpc.md","hash":"623aec4f3cd8afb85b7b30d8bfde50bb2e5a4d4a","modified":1699158073733},{"_id":"source/_posts/geth/code_analysis/geth.evm.md","hash":"ecea3e42003911dd58a2d6a9b8293f02ccb16a75","modified":1699158073733},{"_id":"source/_posts/geth/tech_docs/geth.prune.md","hash":"9ab8f315c3374f67ff7ac6f74a7fbef0ca9f7894","modified":1699158073733},{"_id":"source/_posts/geth/tech_docs/geth.sync.mode.md","hash":"7502b826b5ecbdd02f759a1780528dae344ccad7","modified":1699158073733},{"_id":"source/_posts/cryptography/zkp/zkp-demystify-zokrates.md","hash":"6ee1897511a409ed7e9e476a4375f162e70f1770","modified":1699158077035},{"_id":"source/_posts/geth/tech_docs/geth.v1.10.0.md","hash":"d303922d31b987d5b57080fb6833b84e568de342","modified":1699158073733},{"_id":"source/_posts/cryptography/zkp/zkp-a-brief-understanding.md","hash":"904b00c9a8b65b48db1482f2935fd9b18c37a8a1","modified":1699158077034},{"_id":"source/_posts/cryptography/zkp/zkp-groth16-paper-review.md","hash":"47cfa35483dae66a3c56465f217d27e738f15633","modified":1699158077035},{"_id":"source/_posts/cryptography/elliptic_curve/elliptic-curve-pairing.md","hash":"3d917772eb27821cbf36c73e73f3faa6ce8f9a54","modified":1701399040735},{"_id":"source/_posts/cryptography/zkp/zkp-under-the-hood.md","hash":"ce2b0522282e7b1676f6b884e4afad4e62ec414b","modified":1699158077035},{"_id":"source/_posts/rust/crates/rust-serde.md","hash":"9b985750a751a7bd28effe9e97147e4207a5f093","modified":1699158073734},{"_id":"source/_posts/rust/crates/rust-frequently-used-crates.md","hash":"39aec8a77f62d6c3b164ecef0663a612423afd63","modified":1699158073733},{"_id":"source/images/cryptography/elliptic_curve/paring_ec.png","hash":"85ce9f154c2c896ba28d601ef0a0bac89a82a627","modified":1699158077036},{"_id":"source/images/cryptography/elliptic_curve/point_addition.webp","hash":"ba6cd29aec4a694b53933d0824df180158f98316","modified":1699158077036},{"_id":"source/images/rust/memory/trait_object_memory.png","hash":"dd750cffa7bca5bde7856cfb65f4500de286c96f","modified":1699158073742},{"_id":"source/images/rust/ownership/move-string.png","hash":"d708edd6fb84b9d4ebe75556932f6b920934ca9a","modified":1699158073743},{"_id":"source/_posts/rust/rust_std/rust-smart-pointer-and-internal-mutibility.md","hash":"d97435f2c35ffcd4feb8a1aff787c7c20922438a","modified":1699158073735},{"_id":"source/_posts/rust/rust_std/rust-std-data-structure-1.md","hash":"34627ff9cff48a6a79a36d4e88cc4d32e118c3ae","modified":1699158073735},{"_id":"source/_posts/rust/rust_std/rust-std-data-structure-2.md","hash":"32b80b9540433ffa77a3a7969e06921eb9ddbbca","modified":1699158073735},{"_id":"source/_posts/rust/rust_std/rust-std-sync.md","hash":"bf648427835ab0d834b2701b28bdfd35088aeb2e","modified":1699158073735},{"_id":"source/images/rust/pointers/rc.png","hash":"f568f69808ef0357ea5e6d42667d863b1139cbf1","modified":1699158073744},{"_id":"source/images/evm.drawio.google.png","hash":"413a44d66153c93e2c31172d988e051a2bed9940","modified":1699158073736},{"_id":"source/images/geth_starts.drawio.png","hash":"3c426bd608121669f232aaa1b05ed7a342287fc8","modified":1699158073737},{"_id":"source/images/kademlia.subtree.png","hash":"d3198467460972151f4b2fe2b56fab0dd9e411ca","modified":1699158073738},{"_id":"source/images/trie.prefix.png","hash":"67d26d416faf0cc43ee35b4acda598e88dad2949","modified":1699158073744},{"_id":"source/images/paillier/carmichael_thorem.png","hash":"75b4d32eb2233db653bca52cdea4a624555b5ce4","modified":1699158073740},{"_id":"source/images/two_party_ecdsa/schnorr_ecdsa_comparison.png","hash":"69cbed302af467a4e99653dfb51dca45c4a5a6f3","modified":1699158073745},{"_id":"source/images/paillier/carmichael_thorem_2.png","hash":"29b103c94c36f4520ca8b675486af3d914998ac1","modified":1699158073741},{"_id":"node_modules/hexo-theme-landscape/package.json","hash":"9a94875cbf4c27fbe2e63da0496242addc6d2876","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/languages/de.yml","hash":"3ebf0775abbee928c8d7bda943c191d166ded0d3","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/languages/en.yml","hash":"3083f319b352d21d80fc5e20113ddf27889c9d11","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/languages/es.yml","hash":"76edb1171b86532ef12cfd15f5f2c1ac3949f061","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/README.md","hash":"d2772ece6d4422ccdaa0359c3e07588834044052","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/languages/fr.yml","hash":"415e1c580ced8e4ce20b3b0aeedc3610341c76fb","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/LICENSE","hash":"c480fce396b23997ee23cc535518ffaaf7f458f8","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/languages/hu.yml","hash":"284d557130bf54a74e7dcef9d42096130e4d9550","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/languages/it.yml","hash":"89b7d91306b2c1a0f3ac023b657bf974f798a1e8","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/_config.yml","hash":"b608c1f1322760dce9805285a602a95832730a2e","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/languages/ko.yml","hash":"881d6a0a101706e0452af81c580218e0bfddd9cf","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/languages/mn.yml","hash":"2e7523951072a9403ead3840ad823edd1084c116","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/languages/no.yml","hash":"965a171e70347215ec726952e63f5b47930931ef","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/languages/nl.yml","hash":"12ed59faba1fc4e8cdd1d42ab55ef518dde8039c","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/languages/ja.yml","hash":"a73e1b9c80fd6e930e2628b393bfe3fb716a21a9","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/languages/tr.yml","hash":"a1cdbfa17682d7a971de8ab8588bf57c74224b5b","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/languages/zh-CN.yml","hash":"1efd95774f401c80193eac6ee3f1794bfe93dc5a","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/languages/pt.yml","hash":"57d07b75d434fbfc33b0ddb543021cb5f53318a8","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/languages/ru.yml","hash":"4fda301bbd8b39f2c714e2c934eccc4b27c0a2b0","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/languages/zh-TW.yml","hash":"53ce3000c5f767759c7d2c4efcaa9049788599c3","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/category.ejs","hash":"765426a9c8236828dc34759e604cc2c52292835a","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/index.ejs","hash":"aa1b4456907bdb43e629be3931547e2d29ac58c8","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/archive.ejs","hash":"2703b07cc8ac64ae46d1d263f4653013c7e1666b","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/layout.ejs","hash":"0d1765036e4874500e68256fedb7470e96eeb6ee","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/page.ejs","hash":"7d80e4e36b14d30a7cd2ac1f61376d9ebf264e8b","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/post.ejs","hash":"7d80e4e36b14d30a7cd2ac1f61376d9ebf264e8b","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/scripts/fancybox.js","hash":"c857d7a5e4a5d71c743a009c5932bf84229db428","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/after-footer.ejs","hash":"414914ebb159fac1922b056b905e570ac7521925","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/archive.ejs","hash":"7cb70a7a54f8c7ae49b10d1f37c0a9b74eab8826","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/archive-post.ejs","hash":"c7a71425a946d05414c069ec91811b5c09a92c47","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/article.ejs","hash":"dfd555c00e85ffc4207c88968d12b219c1f086ec","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/tag.ejs","hash":"eaa7b4ccb2ca7befb90142e4e68995fb1ea68b2e","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/footer.ejs","hash":"3656eb692254346671abc03cb3ba1459829e0dce","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/gauges-analytics.ejs","hash":"21a1e2a3907d1a3dad1cd0ab855fe6735f233c74","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/google-analytics.ejs","hash":"2ea7442ea1e1a8ab4e41e26c563f58413b59a3d0","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/mobile-nav.ejs","hash":"e952a532dfc583930a666b9d4479c32d4a84b44e","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/header.ejs","hash":"c1acd247e14588cdf101a69460cb8319c18cd078","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/sidebar.ejs","hash":"930da35cc2d447a92e5ee8f835735e6fd2232469","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_widget/category.ejs","hash":"dd1e5af3c6af3f5d6c85dfd5ca1766faed6a0b05","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_widget/archive.ejs","hash":"beb4a86fcc82a9bdda9289b59db5a1988918bec3","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/head.ejs","hash":"f215d92a882247a7cc5ea80b241bedfcec0ea6ca","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_widget/recent_posts.ejs","hash":"60c4b012dcc656438ff59997e60367e5a21ab746","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_widget/tag.ejs","hash":"2de380865df9ab5f577f7d3bcadf44261eb5faae","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/_variables.styl","hash":"581b0cbefdaa5f894922133989dd2d3bf71ded79","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_widget/tagcloud.ejs","hash":"b4a2079101643f63993dcdb32925c9b071763b46","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/style.styl","hash":"9c451e5efd72c5bb8b56e8c2b94be731e99db05b","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/_extend.styl","hash":"222fbe6d222531d61c1ef0f868c90f747b1c2ced","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/js/script.js","hash":"998ed4c5b147e1299bf62beebf33514474f28112","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/fancybox/jquery.fancybox.min.css","hash":"1be9b79be02a1cfc5d96c4a5e0feb8f472babd95","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/post/date.ejs","hash":"f1458584b679545830b75bef2526e2f3eb931045","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/post/gallery.ejs","hash":"3d9d81a3c693ff2378ef06ddb6810254e509de5b","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/post/nav.ejs","hash":"16a904de7bceccbb36b4267565f2215704db2880","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/post/tag.ejs","hash":"2fcb0bf9c8847a644167a27824c9bb19ac74dd14","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/post/title.ejs","hash":"4d7e62574ddf46de9b41605fe3140d77b5ddb26d","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/_util/grid.styl","hash":"0bf55ee5d09f193e249083602ac5fcdb1e571aed","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/layout/_partial/post/category.ejs","hash":"c6bcd0e04271ffca81da25bcff5adf3d46f02fc0","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/_util/mixin.styl","hash":"44f32767d9fd3c1c08a60d91f181ee53c8f0dbb3","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/_partial/footer.styl","hash":"e35a060b8512031048919709a8e7b1ec0e40bc1b","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/_partial/comment.styl","hash":"79d280d8d203abb3bd933ca9b8e38c78ec684987","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/_partial/archive.styl","hash":"db15f5677dc68f1730e82190bab69c24611ca292","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/_partial/mobile.styl","hash":"a399cf9e1e1cec3e4269066e2948d7ae5854d745","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/_partial/highlight.styl","hash":"bf4e7be1968dad495b04e83c95eac14c4d0ad7c0","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/_partial/sidebar-aside.styl","hash":"890349df5145abf46ce7712010c89237900b3713","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/_partial/article.styl","hash":"80759482d07063c091e940f964a1cf6693d3d406","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/_partial/sidebar-bottom.styl","hash":"8fd4f30d319542babfd31f087ddbac550f000a8a","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/_partial/header.styl","hash":"85ab11e082f4dd86dde72bed653d57ec5381f30c","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/_partial/sidebar.styl","hash":"404ec059dc674a48b9ab89cd83f258dec4dcb24d","modified":1669606834570},{"_id":"source/images/rust/ownership/move-string-2.png","hash":"83342aed7ee254c099ada5d49ed8b9ba52a451a0","modified":1699158073743},{"_id":"source/images/cryptography/zkp/lagrange_interpolating.png","hash":"2e3a62df79b1267dd0172623e46da03801439b01","modified":1699158077038},{"_id":"source/images/rust/pointers/cycle.ref.png","hash":"ed99e2c6020833ccd5af751cf6eb2031cf47d9aa","modified":1699158073743},{"_id":"source/images/zkp/stark/commitment.png","hash":"223d7fbde2b6dc7a2d2b6cabb11fbe4763e8d217","modified":1699158213768},{"_id":"node_modules/hexo-theme-landscape/source/js/jquery-3.4.1.min.js","hash":"88523924351bac0b5d560fe0c5781e2556e7693d","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/fancybox/jquery.fancybox.min.js","hash":"6181412e73966696d08e1e5b1243a572d0f22ba6","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/fonts/fontawesome-webfont.woff","hash":"28b782240b3e76db824e12c02754a9731a167527","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/fonts/fontawesome-webfont.woff2","hash":"d6f48cba7d076fb6f2fd6ba993a75b9dc1ecbf0c","modified":1669606834570},{"_id":"source/images/cryptography/zkp/r1cs.png","hash":"c29022100d7c6581eb2a0c54cc763581d66abf6e","modified":1699158077038},{"_id":"source/images/cryptography/zkp/checking_qap.png","hash":"8f01d96d61a388b1f5d7da55da72428528ccf8e5","modified":1699158077037},{"_id":"source/images/zkp/stark/decommitment.png","hash":"22be5093a44044cb4b623251078bcfefc1fc1951","modified":1699158213768},{"_id":"source/images/zkp/stark/fri.png","hash":"18296b244a046f5f78fda1256760c88aeda09676","modified":1699158213769},{"_id":"source/images/zkp/stark/trace_to_CP.png","hash":"0655eed61ff22c0e16ab48582ffa125d9fd58f50","modified":1699158213771},{"_id":"source/images/kademlia.locating.png","hash":"702d6b779294a3c6e033cc9bde14ef8950982310","modified":1699158073738},{"_id":"node_modules/hexo-theme-landscape/source/css/fonts/FontAwesome.otf","hash":"048707bc52ac4b6563aaa383bfe8660a0ddc908c","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/fonts/fontawesome-webfont.eot","hash":"d980c2ce873dc43af460d4d572d441304499f400","modified":1669606834570},{"_id":"node_modules/hexo-theme-landscape/source/css/fonts/fontawesome-webfont.ttf","hash":"13b1eab65a983c7a73bc7997c479d66943f7c6cb","modified":1669606834570},{"_id":"source/images/rust/macros/16.compile_process.png","hash":"444080319ca2101672941346988f78ed9a37c32d","modified":1699158073742},{"_id":"source/images/math/fourier_series/f_x.png","hash":"539e659af67a921b208780e509c2e28bb8c4fe98","modified":1699158077039},{"_id":"source/images/mpt.png","hash":"700032035bdcd793f94da522330552727b00e5a3","modified":1699158073739},{"_id":"node_modules/hexo-theme-landscape/source/css/images/banner.jpg","hash":"f44aa591089fcb3ec79770a1e102fd3289a7c6a6","modified":1669606834570},{"_id":"source/images/cryptography/rsa/rsa_signature.png","hash":"44eb1a608dafaae244e487cf082aa79c2af5c19f","modified":1699158077036},{"_id":"source/images/zkp/stark/fri_example.png","hash":"b78e0f488557ac5fb59c552b934377fffe9183a9","modified":1699158213770},{"_id":"node_modules/hexo-theme-landscape/source/css/fonts/fontawesome-webfont.svg","hash":"98a8aa5cf7d62c2eff5f07ede8d844b874ef06ed","modified":1669606834570},{"_id":"source/2017-552.pdf","hash":"b1a08857c1f6532f1fbb718c48a34fd48ea7da70","modified":1699158073732},{"_id":"source/images/zkp/plonk/plonk_circuit_to_trace.png","hash":"d341c0439212156becdc0f337d1a12c8627f4592","modified":1699666066351},{"_id":"source/_posts/zkp/plonky2.md","hash":"e480bcf9c62d3c0b4ea41eecfc69794b9fcf6571","modified":1699248267137},{"_id":"public/2023/11/06/zkp/plonky2/index.html","hash":"e259a3f5949391cb60e8629f6ade7f0ae4fd3828","modified":1700633599407},{"_id":"public/2023/11/01/cryptography/montgomery-multiplication/index.html","hash":"453556ffcc134625d3032febe122c3887c08ca71","modified":1700633599407},{"_id":"public/2023/11/01/cryptography/bls12-381/index.html","hash":"f536738a60468b41f512f7d68704ed7f27068f0a","modified":1701227621497},{"_id":"public/2023/11/01/zkp/understanding-plonk/index.html","hash":"5380b76a95b4e31fdf1ba9998237a05423fa9e06","modified":1700633599407},{"_id":"public/2023/10/16/cryptography/kzg-commitment/index.html","hash":"59016382550c6d24d3ccdfbb76480308d680f063","modified":1701227621497},{"_id":"public/2023/10/12/blockchain/danksharding/index.html","hash":"5eb979dc0360ff315388066894b8942c6d84edf9","modified":1702283024204},{"_id":"public/2023/10/12/math/fourier-series/index.html","hash":"0354b9bd3456d421c8cb2e0e3012706284fd41d6","modified":1702283024204},{"_id":"public/2023/10/01/tools/markdown_and_latex/index.html","hash":"6c5cc03ab9de516dd0f52920416a36dca55d22a8","modified":1702283024204},{"_id":"public/2023/07/14/cryptography/zkp/zkp-demystify-zokrates/index.html","hash":"b0de23154aa21b9688ed24ad1cb04d01ac71688f","modified":1702283024204},{"_id":"public/2023/07/07/cryptography/zkp/zkp-groth16-paper-review/index.html","hash":"8571282e253137689219c3df12c13a6f65d6969a","modified":1702283024204},{"_id":"public/2023/06/17/cryptography/cryptography-03-elliptic-curve/index.html","hash":"640ac9a8bd1335e8793b85172fb1e5361b830fe2","modified":1702283024204},{"_id":"public/2023/06/01/rust/rust-10-concurrency/index.html","hash":"2dbbed0e17f98e4ece75f14723daaab87fab9dda","modified":1702283024204},{"_id":"public/2023/04/11/rust/rust-09-functional/index.html","hash":"428386cb66dcb64aff7a5f6a709a2883889bf776","modified":1702283024204},{"_id":"public/2023/04/01/rust/crates/rust-serde/index.html","hash":"2220e133f2a841c601142bb0bede1183906c1335","modified":1702283024204},{"_id":"public/2023/03/25/geth/tech_docs/geth.prune/index.html","hash":"e27b60acff0e964ec045c097adfa716e25e2d3da","modified":1702283024204},{"_id":"public/2023/03/05/golang/go-similar-concepts-comparison/index.html","hash":"ff14a3ab853091431b9573ff38d63e7a94527350","modified":1702283024204},{"_id":"public/2023/02/07/cryptography/two-party-ecdsa/index.html","hash":"896ffa24c7c6f7275cf0e669565ec07fe4f15dbb","modified":1702283024204},{"_id":"public/2023/01/22/MPT/index.html","hash":"514b09ffa00adffa62f4dce32a8d4ae306ca8e5e","modified":1702283024204},{"_id":"public/2023/01/01/geth-fine-tune/index.html","hash":"b3e25c8e34117cdeedaa88c9c90fd29df8f773d9","modified":1702283024204},{"_id":"public/2022/12/13/rust/crates/rust-frequently-used-crates/index.html","hash":"3c3c2b9a0eaaa6846e11c7cbe541b71e58200ec1","modified":1702283024204},{"_id":"public/2022/11/27/hello-world/index.html","hash":"a3e4e8ddc9d8cc9fa92e0c34500b84783096cea9","modified":1702283024204},{"_id":"public/2022/11/20/rust/rust-tools/index.html","hash":"56ec8e4f60c20b145b59b7774b0b36cd3ee53d3a","modified":1702283024204},{"_id":"public/2022/11/13/rust/rust-cargo-doc/index.html","hash":"3642d96d8e3919425c5f39bcd5914ff748f01bcd","modified":1702283024204},{"_id":"public/2022/11/08/geth/code_analysis/geth.1.rpc/index.html","hash":"ed49bb431ee485a1d78976a0d06d4c76973a9c05","modified":1702283024204},{"_id":"public/2022/10/25/rust/rust-05-memory-layout/index.html","hash":"594366895eb0e26a60df49b7384290707c8cb821","modified":1702283024204},{"_id":"public/2022/09/27/rust/rust-01-resource/index.html","hash":"070b3d3419fd454488fd57a6b6c756153893dff3","modified":1702283024204},{"_id":"public/archives/index.html","hash":"94ed15e3a37e7aa24999359c3e990e241ac660a7","modified":1702283024204},{"_id":"public/archives/page/2/index.html","hash":"a50cc6b434912028db5b9abb6cf514197815bb0e","modified":1702283024204},{"_id":"public/archives/page/3/index.html","hash":"fafe090cd159de1c106397c26198a488bfd20d26","modified":1702283024204},{"_id":"public/archives/page/4/index.html","hash":"258f1d35da91079ca5264fa5bfc6241b76dd8a39","modified":1702283024204},{"_id":"public/archives/page/5/index.html","hash":"4a05dd87aba90bc7bcc570c5f5cd8203ad47b997","modified":1702283024204},{"_id":"public/archives/page/6/index.html","hash":"ca8da1f9cf094356eb58dd376e7c6cd401a6fa4f","modified":1702283024204},{"_id":"public/archives/2022/index.html","hash":"9079842e6f3569af6eb31c65ca2dd7b57bbefa95","modified":1702283024204},{"_id":"public/archives/2022/page/2/index.html","hash":"44da281b1dfc952a0a1760e073af4ed90510e325","modified":1702283024204},{"_id":"public/archives/2022/09/index.html","hash":"28cf5d98effa4d170c28836a9099d0e68f0eabd8","modified":1702283024204},{"_id":"public/archives/2022/10/index.html","hash":"23347789920e52f99740e4092066f2ed18ea520c","modified":1702283024204},{"_id":"public/archives/2022/11/index.html","hash":"dccce48f731fb59ae46983c7e60c12406edeb0f5","modified":1702283024204},{"_id":"public/archives/2022/12/index.html","hash":"f5b27921f6d99260ca26f84c236b2c4b671829ee","modified":1702283024204},{"_id":"public/archives/2023/index.html","hash":"7f8e4ef812d163dd3c6fb848d6b957d94f508acd","modified":1702283024204},{"_id":"public/archives/2023/page/2/index.html","hash":"77c96e9efef06850bea97f526564e0d13c4bafc8","modified":1702283024204},{"_id":"public/archives/2023/page/3/index.html","hash":"5e4d858054d09f6823e212526f6df9d236628479","modified":1702283024204},{"_id":"public/archives/2023/page/4/index.html","hash":"058083198b84e8899d42af14bcffb047b050ea33","modified":1702283024204},{"_id":"public/archives/2023/01/index.html","hash":"a7390007d2062b308240f170bff1104eb3f97a7d","modified":1702283024204},{"_id":"public/archives/2023/02/index.html","hash":"57964ae782230b70e5109f5181933fd3955dde26","modified":1702283024204},{"_id":"public/archives/2023/03/index.html","hash":"d9e37568cdc6aeb0973446c6fec3a0a71f814152","modified":1702283024204},{"_id":"public/archives/2023/04/index.html","hash":"0f119e7ca1d94b5cf39571566b93db82a53e922b","modified":1702283024204},{"_id":"public/archives/2023/05/index.html","hash":"387f398976d2d37e77bcf68dcd5b377a158484ec","modified":1702283024204},{"_id":"public/archives/2023/06/index.html","hash":"9b1bc2e4eb048de400a60c23e0ac73acec8cde63","modified":1702283024204},{"_id":"public/archives/2023/07/index.html","hash":"7b0e30cb20af55acc18acae284039dec1ca46b51","modified":1702283024204},{"_id":"public/archives/2023/10/index.html","hash":"e2e5edd6419e2c166db4d453b73f8f5b96e21c3e","modified":1702283024204},{"_id":"public/archives/2023/11/index.html","hash":"a46e47b1020a478d8720fe3e834ea7c62a86f0f3","modified":1702283024204},{"_id":"public/tags/blockchain/index.html","hash":"605b7443aa03098a54c699a64b34cfa7433bd9e5","modified":1702283024204},{"_id":"public/tags/geth/index.html","hash":"406147d619dc774321bdc87321ff010d51d6b41e","modified":1702283024204},{"_id":"public/tags/cryptography/index.html","hash":"eda4cb9f15d750071a66e3e80ff3077b73724d41","modified":1702283024204},{"_id":"public/tags/cryptography/page/2/index.html","hash":"720ba7f6af1c9d5f1e0659ae25279242a453e588","modified":1702283024204},{"_id":"public/tags/number-theory/index.html","hash":"cba28d18c31c4f60dfd8e43d9388631403d28677","modified":1702283024204},{"_id":"public/tags/ecdsa/index.html","hash":"6f20a96f5acb6129a88691615852def0e03a392f","modified":1702283024204},{"_id":"public/tags/mpc/index.html","hash":"5a3d7b14d0dace715b361a3a8b01bc1c6d50c6cf","modified":1702283024204},{"_id":"public/tags/golang/index.html","hash":"b94e6c2904965f51843aa28a032049d8910b3732","modified":1702283024204},{"_id":"public/tags/rust/index.html","hash":"3216f068db451dbc63411eb0eaaf9f91075f59fe","modified":1702283024204},{"_id":"public/tags/rust/page/2/index.html","hash":"7ce94a328bed5e8c1d65d27e3031d760d949eef3","modified":1702283024204},{"_id":"public/tags/cargo/index.html","hash":"a2d1c2272183f7b95d702c3a2541b135872f4e95","modified":1702283024204},{"_id":"public/tags/zkp/index.html","hash":"27819920865a27f49e99ca9c9a4740028b664135","modified":1702283024204},{"_id":"public/tags/tool/index.html","hash":"7c615230d6b0c1d6f0e12af03e9b3d2eaf368044","modified":1702283024204},{"_id":"public/tags/math/index.html","hash":"f5155a76b420d4fe35fe439598a3b6a199298a9d","modified":1702283024204},{"_id":"public/tags/rust-crate/index.html","hash":"4552987c640338e92045a2057a6a023cf7583029","modified":1702283024204},{"_id":"public/tags/rust-std/index.html","hash":"570d7d79db7206044e18b28fddaca063347ea4a3","modified":1702283024204},{"_id":"public/2023/11/03/zkp/stark-101/index.html","hash":"e728dd7a064ca2a44ea93dfe966a6d3ea712a741","modified":1700633599407},{"_id":"public/2023/07/01/cryptography/zkp/zkp-under-the-hood/index.html","hash":"d933dc2b2a1bdbac038788c6a46c92ca84dbfc8c","modified":1702283024204},{"_id":"public/2023/06/27/cryptography/zkp/zkp-a-brief-understanding/index.html","hash":"a77bb9f26766c2b1e395224835fd58ee349bcc18","modified":1702283024204},{"_id":"public/2023/06/27/cryptography/elliptic_curve/elliptic-curve-pairing/index.html","hash":"51958e19ed40c90beecc6d61f4799bcab58df9a6","modified":1702283024204},{"_id":"public/2023/06/20/cryptography/cryptography-04-digital-signature/index.html","hash":"87cf1670d877edcc277f4d5f3d933048f308996a","modified":1702283024204},{"_id":"public/2023/06/10/cryptography/cryptography-02-rsa/index.html","hash":"1d5a9fea7aa5802731353d904150e40f91f1e1bd","modified":1702283024204},{"_id":"public/2023/06/08/rust/rust_std/rust-std-sync/index.html","hash":"650436c8268d58da5dc2fe3c96b79994b43a3a7a","modified":1702283024204},{"_id":"public/2023/06/03/rust/rust_std/rust-smart-pointer-and-internal-mutibility/index.html","hash":"344f77470ff6545261f4eb8e8470dff1e3943a39","modified":1702283024204},{"_id":"public/2023/06/03/cryptography/cryptography-01-primitive-group-and-field/index.html","hash":"65dc137c0f12ec31960957d8977a0aebcbfddcd7","modified":1702283024204},{"_id":"public/2023/05/02/rust/rust_std/rust-std-data-structure-2/index.html","hash":"8245ebda94413720d2d5e94f97d8d399ca0b87ca","modified":1702283024204},{"_id":"public/2023/05/01/rust/rust_std/rust-std-data-structure-1/index.html","hash":"e67fa2e108149cc8f5e831654eea689555520ff0","modified":1702283024204},{"_id":"public/2023/03/18/geth/tech_docs/geth.sync.mode/index.html","hash":"551b172dc9de61761bbf5c47f5ec5ec5c86f94e7","modified":1702283024204},{"_id":"public/2023/03/15/geth/tech_docs/geth.v1.10.0/index.html","hash":"38ba66c44c60fa43a77e3a5738ba30134c13251d","modified":1702283024204},{"_id":"public/2023/03/02/golang/go-reflect/index.html","hash":"daf45c37c6c7cbd8e7a19be3b65615b2f0184851","modified":1702283024204},{"_id":"public/2023/02/23/cryptography/paillier-encryption/index.html","hash":"aefa9d39d385385d948764d30775539bcd7384ff","modified":1702283024204},{"_id":"public/2023/01/15/blockchain.kademlia/index.html","hash":"8d45243f395519f93e18a554bd3348857edf6a06","modified":1702283024204},{"_id":"public/2023/01/13/rust/rust-async/index.html","hash":"d5df4ab53ae2e7ff67cdd5119c3fdca5950117d6","modified":1702283024204},{"_id":"public/2023/01/08/geth/code_analysis/geth.evm/index.html","hash":"7d9eef3e1be997eb8442fd2f4bbf3cdd1de3eb0f","modified":1702283024204},{"_id":"public/2022/12/28/rust/rust-07-macro/index.html","hash":"1a42ab081cdce0734c933a0f26f1ebdfed83550d","modified":1702283024204},{"_id":"public/2022/12/06/rust/rust-cargo-all-in-one/index.html","hash":"e2bb545576b9ac935f2e5c8d9eddf789c296af39","modified":1702283024204},{"_id":"public/2022/11/23/rust/rust-similar-concepts-comparison/index.html","hash":"1cb7caf668d1e2dededda346732965afc882fb42","modified":1702283024204},{"_id":"public/2022/11/17/rust/rust-sugar/index.html","hash":"beec4813cb5271a73e43c44d7364a7827f3e1753","modified":1702283024204},{"_id":"public/2022/11/06/rust/rust-08-project-management/index.html","hash":"d57e92577442da4c3c3274ea58949b8fc96eb265","modified":1702283024204},{"_id":"public/2022/11/01/geth/code_analysis/geth.0.get.start/index.html","hash":"ef8ed821727c55e7f4fddbf15f6cabbf7705d494","modified":1702283024204},{"_id":"public/2022/10/30/rust/rust-06-smart-pointer/index.html","hash":"e2d6b952bd2f4ea72bc6d5051df98da2ceb89c42","modified":1702283024204},{"_id":"public/2022/10/18/rust/rust-04-lifetime/index.html","hash":"f6af715f33cdb8ebdbb94a16c9517ca6858d5a47","modified":1702283024204},{"_id":"public/2022/10/11/rust/rust-03-ownership/index.html","hash":"b1636058e9a9e6b89e104caad2f787f892e9950f","modified":1702283024204},{"_id":"public/2022/10/04/rust/rust-02-basics/index.html","hash":"4dedce4d393796b62e4582c13016999a29796776","modified":1702283024204},{"_id":"public/index.html","hash":"cf5dd165d63af746c113ffb7d64ccb339c0185f0","modified":1702352970990},{"_id":"public/page/2/index.html","hash":"44a6915019fd3db4481388581462eb53c864b09d","modified":1702381556218},{"_id":"public/page/3/index.html","hash":"dbfddfe01ae4b74a7774232e24ed29e733ca3ba9","modified":1702283024204},{"_id":"public/page/4/index.html","hash":"73ad2b1f5db5437178cf2a58c20880739997bede","modified":1702283024204},{"_id":"public/page/5/index.html","hash":"13c4940f12e6f0b05ec51f1ebe3b7a42f2b6f183","modified":1702283024204},{"_id":"public/page/6/index.html","hash":"9c6507867088d15e8f9cc5416b072da144ddcf6e","modified":1702283024204},{"_id":"public/images/zkp/plonk/plonk_circuit_to_trace.png","hash":"d341c0439212156becdc0f337d1a12c8627f4592","modified":1699667020264},{"_id":"source/_posts/math/goldilocks-field.md","hash":"0d28d6301c8bd16bcc6e126e751af964ecea4128","modified":1700834519863},{"_id":"source/_posts/cuda/inline-ptx-assembly-in-cuda.md","hash":"a2eb51caa1376faaa14e33e8ec527ae0d26b0042","modified":1700619356342},{"_id":"source/_posts/cpp/cpp-notes.md","hash":"3fae4c9374b0ebea986d77d96f1347697f1ebca9","modified":1700019450981},{"_id":"source/images/zkp/plonk/zero_test.png","hash":"23686fabe59c9bec98ff658ce1782886fbd5c5b3","modified":1699668979336},{"_id":"public/2023/11/18/math/goldilocks-field/index.html","hash":"2e06f4f6cfa64d42c201cba3d65db662e9518fe7","modified":1701138383617},{"_id":"public/2023/11/15/cpp/cpp-notes/index.html","hash":"bfa9974c0641ff74b5845671c5df8b1da374952d","modified":1700553651994},{"_id":"public/2023/11/15/cuda/inline-ptx-assembly-in-cuda/index.html","hash":"ca85357b31121e56a4cf31f85392b122576e7a07","modified":1702283024204},{"_id":"public/images/zkp/plonk/zero_test.png","hash":"23686fabe59c9bec98ff658ce1782886fbd5c5b3","modified":1700277062226},{"_id":"source/_posts/cpp/generics_and_macros.md","hash":"35f6098a6a30a9fcdca9995fb324fea72d6933d2","modified":1700706616168},{"_id":"source/_posts/cpp/key_words.md","hash":"33ffe7f7a6d05de363fd530dea3c46d49b0080a1","modified":1700723138353},{"_id":"source/_posts/cpp/types.md","hash":"6af7e4b22d1f762de43cb1e8639d69cf873100d3","modified":1700569616654},{"_id":"source/_posts/cpp/std.md","hash":"e0c17144c4eeb4af63face571c963de8fb081d82","modified":1700571950461},{"_id":"public/2023/11/21/cpp/generics_and_macros/index.html","hash":"7a4275b62559015405568a8ee6e8cce000d833f9","modified":1702283024204},{"_id":"public/2023/11/21/cpp/key_words/index.html","hash":"ba2cd56f969ea04d401db7af8383100a585033cd","modified":1702283024204},{"_id":"public/2023/11/21/cpp/std/index.html","hash":"e0254a767a4556cac5fe94b814a452d6f346498f","modified":1702283024204},{"_id":"public/2023/11/15/cpp/types/index.html","hash":"da88c592cd346be4592a71b46fd32f8342927fb9","modified":1702283024204},{"_id":"public/archives/2023/page/5/index.html","hash":"d65fe390b8df690364e3b3900db925b68a162455","modified":1702283024204},{"_id":"public/archives/2023/11/page/2/index.html","hash":"19d8717ae0ca381e1b53e3db2b5045bade73c094","modified":1702283024204},{"_id":"public/tags/cpp/index.html","hash":"b9921797e58a7ef9f9185a8b3dfb4fbad6f3818f","modified":1702283024204},{"_id":"source/_posts/cuda/concepts_and_keywords.md","hash":"d1fe9b6dd6dbfba1eef681c639df5d49a33e911f","modified":1700724603676},{"_id":"source/_posts/cuda/cuda_api_and_tools.md","hash":"845fef81ac8e65bdc6b6c18f1795ea248e368b4d","modified":1700725285289},{"_id":"source/images/cuda/cuda_stream_example.png","hash":"cb321f532bac228ef187e82210ad2c2586292b0e","modified":1700620752287},{"_id":"source/_posts/arithmatic/efficient_implementations.md","hash":"5a2d092c2fefd8ba70212e9f75697045089600b9","modified":1700637615345},{"_id":"source/_posts/arithmatic/tricks.md","hash":"b51d383cef8fa22b5fae0af5ecae419589bf9a8f","modified":1702016313813},{"_id":"public/2023/11/22/cuda/cuda_api_and_tools/index.html","hash":"bc7785310e3fd6b66d5ab9f07ec56be34f99a523","modified":1702283024204},{"_id":"public/2023/11/22/arithmatic/efficient_implementations/index.html","hash":"327bf3b78c8a02f955d0b6f5b493cfe2a41cbb8f","modified":1702283024204},{"_id":"public/2023/11/22/arithmatic/tricks/index.html","hash":"cce5cddf2b571bf7b01a629452a162ad7cfeb1f7","modified":1702283024204},{"_id":"public/archives/page/7/index.html","hash":"f3584f6a8de495c89d934d6a5ccecda31282b0f1","modified":1702283024204},{"_id":"public/tags/cuda/index.html","hash":"6dda011c800cea67d6028029d4d224a19fb11d3c","modified":1702283024204},{"_id":"public/tags/arithmatic/index.html","hash":"75a16681f4f1231f7559cdf7ceabdbe9e2d976df","modified":1702283024204},{"_id":"public/2023/11/22/cuda/concepts_and_keywords/index.html","hash":"b4cddfafe726dcee8cc0a89b800f20272ff9600a","modified":1702283024204},{"_id":"public/page/7/index.html","hash":"eae3b9f46a731c38357915ac651086a4cf0af55a","modified":1702283024204},{"_id":"public/images/cuda/cuda_stream_example.png","hash":"cb321f532bac228ef187e82210ad2c2586292b0e","modified":1700633599407},{"_id":"source/images/arithmatic/multiple_precision_addition.png","hash":"9f1f24e8f95d44e62e98277b48bac948ae91285e","modified":1700635373358},{"_id":"source/images/arithmatic/radix_b_repesentation.png","hash":"9ce9e52f7a9a2ecd8b416e1cf8d11308bb604a8a","modified":1700634163557},{"_id":"public/images/arithmatic/multiple_precision_addition.png","hash":"9f1f24e8f95d44e62e98277b48bac948ae91285e","modified":1700745044711},{"_id":"public/images/arithmatic/radix_b_repesentation.png","hash":"9ce9e52f7a9a2ecd8b416e1cf8d11308bb604a8a","modified":1700745044711},{"_id":"source/_posts/cryptography/zkp/sgx_multi_prover.md","hash":"7cb2dec1b24e1f31a79ef8ed292dec2920733581","modified":1700834715647},{"_id":"source/_posts/arithmatic/montgomery-multiplication.md","hash":"067016492be4574514e2aa6b773f87af55b35442","modified":1702003544046},{"_id":"source/_posts/cryptography/zkp/stark-101.md","hash":"867f0c912e4f6abe1512848408f7afebae04b536","modified":1699157905566},{"_id":"source/_posts/cryptography/zkp/plonky2.md","hash":"31d92b6cfa6cd6db20f8474144f740c57a0c7e9a","modified":1701676923117},{"_id":"source/_posts/cryptography/zkp/zk-bench.md","hash":"595402d134d08adef02d80ba28de119e7a84947e","modified":1700834726446},{"_id":"source/_posts/cryptography/zkp/understanding-plonk.md","hash":"0fdbd1e6ae465944722bff3124e587af1ec31576","modified":1701228285714},{"_id":"public/2023/11/24/cryptography/zkp/zk-bench/index.html","hash":"c90de35829e35478749d060aa8f31790de9a68ce","modified":1702283024204},{"_id":"public/2023/11/24/cryptography/zkp/sgx_multi_prover/index.html","hash":"17af084e22fd2e7c4f43b1c285092c7caa614a2a","modified":1702283024204},{"_id":"public/2023/11/06/cryptography/zkp/plonky2/index.html","hash":"bc87f9693d8e7142e0a6697ab0315800511d175b","modified":1701962639979},{"_id":"public/2023/11/01/arithmatic/montgomery-multiplication/index.html","hash":"11e20bf895491889fdbaae51754c6370b39f5805","modified":1701661940748},{"_id":"public/2023/11/01/cryptography/zkp/understanding-plonk/index.html","hash":"b840bb54b8f6746718ba169c352e38333336e049","modified":1702283024204},{"_id":"public/2023/11/03/cryptography/zkp/stark-101/index.html","hash":"1ecd81fc0dad55c2c3c15c58b63027faeef4ab75","modified":1702283024204},{"_id":"source/_posts/arithmatic/goldilocks-field.md","hash":"00d471ce337f6d98e577533d2f400bfc3b2ef0ea","modified":1701164677469},{"_id":"source/images/zkp/plonk/gate_evaluation_zero_test.png","hash":"a62a302d7dd13cc9d19913819fa4e950b483ed89","modified":1701141838168},{"_id":"source/images/zkp/plonk/permutation_check.png","hash":"04d7e45bd261bb24a0e0e41beb3e53ded818b43b","modified":1701142333439},{"_id":"public/2023/11/18/arithmatic/goldilocks-field/index.html","hash":"4ea47edb4de9299e305dcbd1cb59ae77437067cc","modified":1702283024204},{"_id":"public/images/zkp/plonk/gate_evaluation_zero_test.png","hash":"a62a302d7dd13cc9d19913819fa4e950b483ed89","modified":1701158595615},{"_id":"public/images/zkp/plonk/permutation_check.png","hash":"04d7e45bd261bb24a0e0e41beb3e53ded818b43b","modified":1701158595615},{"_id":"source/images/zkp/plonk/copy_constraint_example.png","hash":"ccace19afe01e3991b688eef27da922f13b69365","modified":1701185887943},{"_id":"source/images/zkp/plonk/custom_gate.png","hash":"35c2febb4bd8044f48cf5d42744ccc508503d436","modified":1701227590315},{"_id":"source/images/zkp/plonk/prod_check_lemma.png","hash":"e6e29ac8e9eae8cdeff6e8b9909904c612d6ccb4","modified":1701165709852},{"_id":"source/images/zkp/plonk/prod_check_prove_verify.png","hash":"e6de2efae2fdbb98cc49d6bae5965ed311412e4a","modified":1701165956061},{"_id":"source/images/zkp/plonk/prescribed_perm_check_problem_prove_verify.png","hash":"58c1e25df25c2ed3cfe5da6eefe4f543890a3036","modified":1701167756501},{"_id":"source/images/zkp/plonk/prescribed_perm_check_problem_complete.png","hash":"10bf6b8740b39541a2062928bc4a48d57d4a4d96","modified":1701167850819},{"_id":"source/images/zkp/plonk/prescribed_perm_check_problem_reduce.png","hash":"1abcac63be58b40a4191defa3a7335632ffdc3da","modified":1701167649980},{"_id":"source/images/zkp/plonk/prescribed_perm_check_problem_quadratic.png","hash":"ed9cfcc17dd706abcf381f05578d00c00dab3214","modified":1701167336035},{"_id":"source/images/zkp/plonk/prescribed_perm_check_problem.png","hash":"26b7e37f8ca360fef0504099e7e6a8fb5423fe53","modified":1701167178038},{"_id":"public/images/zkp/plonk/copy_constraint_example.png","hash":"ccace19afe01e3991b688eef27da922f13b69365","modified":1701227621497},{"_id":"public/images/zkp/plonk/prod_check_lemma.png","hash":"e6e29ac8e9eae8cdeff6e8b9909904c612d6ccb4","modified":1701227621497},{"_id":"public/images/zkp/plonk/custom_gate.png","hash":"35c2febb4bd8044f48cf5d42744ccc508503d436","modified":1701227621497},{"_id":"public/images/zkp/plonk/prod_check_prove_verify.png","hash":"e6de2efae2fdbb98cc49d6bae5965ed311412e4a","modified":1701227621497},{"_id":"public/images/zkp/plonk/prescribed_perm_check_problem_prove_verify.png","hash":"58c1e25df25c2ed3cfe5da6eefe4f543890a3036","modified":1701227621497},{"_id":"public/images/zkp/plonk/prescribed_perm_check_problem_complete.png","hash":"10bf6b8740b39541a2062928bc4a48d57d4a4d96","modified":1701227621497},{"_id":"public/images/zkp/plonk/prescribed_perm_check_problem_reduce.png","hash":"1abcac63be58b40a4191defa3a7335632ffdc3da","modified":1701227621497},{"_id":"public/images/zkp/plonk/prescribed_perm_check_problem_quadratic.png","hash":"ed9cfcc17dd706abcf381f05578d00c00dab3214","modified":1701227621497},{"_id":"public/images/zkp/plonk/prescribed_perm_check_problem.png","hash":"26b7e37f8ca360fef0504099e7e6a8fb5423fe53","modified":1701227621497},{"_id":"source/_posts/cryptography/elliptic_curve/bls12-381.md","hash":"be92f8e12ea7890f02772cb2ee19f0b460ed06e7","modified":1701575637439},{"_id":"source/_posts/cryptography/zkp/kzg-commitment.md","hash":"acbf08b806cc9ff247eab5eb676e6b0deff10edd","modified":1702289247075},{"_id":"public/2023/11/05/cryptography/zkp/kzg-commitment/index.html","hash":"d8c8710aec26c17a2b5dfe12e1f6c0bf7c7d685d","modified":1701575782113},{"_id":"public/tags/ec/index.html","hash":"f49c00f9fe13c32261802fb18a11b46205016cb8","modified":1702283024204},{"_id":"public/archives/2023/12/index.html","hash":"fdc18dd671388cb2bd5cd64eee0b9d0ccfde8fd4","modified":1702283024204},{"_id":"public/2023/12/01/cryptography/elliptic_curve/bls12-381/index.html","hash":"ef8d5bc46ca215ec0f308cb8a885c04e43e632e6","modified":1702283024204},{"_id":"public/2023/12/03/cryptography/zkp/kzg-commitment/index.html","hash":"c40ae91dda0c79a493a2e1919a1e7a0c007cf51f","modified":1702352970990},{"_id":"public/2023/12/07/arithmatic/montgomery-multiplication/index.html","hash":"f329b0ab8951d3abb3d8bab03100b4aec5ac7269","modified":1702283024204},{"_id":"source/_posts/cryptography/zkp/polygon_zkevm.md","hash":"54c0dbd616412eb600eea236afaa6340abda0a23","modified":1702283699530},{"_id":"source/_posts/cryptography/zkp/plonky2-code-analysis.md","hash":"690fafd34484a6e20e707de2066665aaf9b5689c","modified":1702381537866},{"_id":"source/images/zkp/zkevm/polygon/state_machines.png","hash":"d367f14e982243b0db74d15ec1f71cc4c91a6b0d","modified":1702282985940},{"_id":"public/2023/12/11/cryptography/zkp/polygon_zkevm/index.html","hash":"0707191693ef92757d38a27f3b788b5dd6f1b286","modified":1702352970990},{"_id":"public/2023/11/06/cryptography/zkp/plonky2-code-analysis/index.html","hash":"40150cb8c969f6476ae838ec3520e635ac823486","modified":1702381556218},{"_id":"public/tags/zkp/page/2/index.html","hash":"8e6da933936be3a044a678518ddf863c4645ec7a","modified":1702283024204},{"_id":"public/images/zkp/zkevm/polygon/state_machines.png","hash":"d367f14e982243b0db74d15ec1f71cc4c91a6b0d","modified":1702283024204},{"_id":"source/images/zkp/zkevm/polygon/micro_processor.png","hash":"0aef9a284fe051e40d79c80d0b79021f34aaadde","modified":1702283676469},{"_id":"public/images/zkp/zkevm/polygon/micro_processor.png","hash":"0aef9a284fe051e40d79c80d0b79021f34aaadde","modified":1702352970990}],"Category":[],"Data":[],"Page":[],"Post":[{"title":"MPT","date":"2023-01-22T09:25:36.000Z","_content":"\n# trie\na trie, also called prefix tree, is a type of k-ary search tree. These keys are most often strings, with links between nodes defined not by the entire key, but by individual characters. In order to access a key, the trie is traversed depth-first, following the links between nodes, which represent each character in the key.\n\n![prefix trie](/images/trie.prefix.png)\nIn general, the nodes of a Trie look like this:\n```\n[ [Ia, Ib, … I*], value]\n```\n[Ia, Ib, ... I*] is the index array of the node, which takes the next character in the key as the index, and each element I* points to the corresponding child node. value represents the value\n\n# MPT\nEach block of Ethereum contains three MPT trees, respectively\n\n- Transaction tree\n- Receipt tree\n- State tree\n\nIn the figure below are two block headers, where state root, tx root receipt root stores the roots of the three trees, and the second block shows when the data of account 175 changes (27 -> 45). Only need to store 3 nodes related to this account, and the data in the old block can still be accessed normally. (This is somewhat similar to the implementation of an immutable data structure in a functional programming language.) The detailed structure is\n\n![state reference](/images/mpt.state.ref.png)\n\n- use []byte as key, other than string\n- nibble: the smallest unit of the key type (4 bit)\n- Use hashes to refer to nodes instead of memory pointers\n\nthere are two types of node: full nodes (fullNode) and short nodes (shortNode). Full nodes have 17 elements, while shortNode nodes have two elements. Their schematic expressions are as follows\n```\nfullNode: [i0, i1, i2, … i15, hash] \nshortNode： [ [k0, k1, … kn], hash ] // first element is an array\n```\nif the hash pointing to a value, it is a leaf node; if pointing another node, a non leaf node. shortNode contains extension and leaf node. full node is branch node.\n\n![mpt](/images/mpt.png)\n\nUse the upper 4 bits of the first byte of the []byte value composed of nibbles as storage flag. The 0th bit stores the parity information, and the 1st bit stores the type represented by the value\n|hex char| bits | pointing to | odd/even | 2nd niddle padding |\n| -----|:----:|:----:|:----:|-------|\n|0| 0000 | node | even | no |\n|1| 0001 | node | odd | yes |\n|2| 0010 | value | even | no |\n|3| 0011 | value | odd | yes |\n\nthis encoding method is only used when accessing the database. After reading into memory, the key is directly stored in []byte type\n\nIn the trie module, there is a `Database` object, which you can understand as a cache layer of the underlying database. In actual use, the Trie object uses the Database as a database. However, the important function of Database is not caching, but the reference counting of node data during caching, and provides Database.Reference and Database.Dereference to reference and dereference a trie node. If the reference count of a node becomes 0, the node will be deleted from memory, so it will not be written to the real database\n\n# reference\n- [github](https://github.com/agiletechvn/go-ethereum-code-analysis/blob/master/trie-analysis.md)\n- [yangzhe's blog](http://yangzhe.me/2019/01/12/ethereum-trie-part-1/)\n- [yangzhe's blod](http://yangzhe.me/2019/01/18/ethereum-trie-part-2/)","source":"_posts/MPT.md","raw":"---\ntitle: MPT\ndate: 2023-01-22 17:25:36\ntags: [blockchain, geth]\n---\n\n# trie\na trie, also called prefix tree, is a type of k-ary search tree. These keys are most often strings, with links between nodes defined not by the entire key, but by individual characters. In order to access a key, the trie is traversed depth-first, following the links between nodes, which represent each character in the key.\n\n![prefix trie](/images/trie.prefix.png)\nIn general, the nodes of a Trie look like this:\n```\n[ [Ia, Ib, … I*], value]\n```\n[Ia, Ib, ... I*] is the index array of the node, which takes the next character in the key as the index, and each element I* points to the corresponding child node. value represents the value\n\n# MPT\nEach block of Ethereum contains three MPT trees, respectively\n\n- Transaction tree\n- Receipt tree\n- State tree\n\nIn the figure below are two block headers, where state root, tx root receipt root stores the roots of the three trees, and the second block shows when the data of account 175 changes (27 -> 45). Only need to store 3 nodes related to this account, and the data in the old block can still be accessed normally. (This is somewhat similar to the implementation of an immutable data structure in a functional programming language.) The detailed structure is\n\n![state reference](/images/mpt.state.ref.png)\n\n- use []byte as key, other than string\n- nibble: the smallest unit of the key type (4 bit)\n- Use hashes to refer to nodes instead of memory pointers\n\nthere are two types of node: full nodes (fullNode) and short nodes (shortNode). Full nodes have 17 elements, while shortNode nodes have two elements. Their schematic expressions are as follows\n```\nfullNode: [i0, i1, i2, … i15, hash] \nshortNode： [ [k0, k1, … kn], hash ] // first element is an array\n```\nif the hash pointing to a value, it is a leaf node; if pointing another node, a non leaf node. shortNode contains extension and leaf node. full node is branch node.\n\n![mpt](/images/mpt.png)\n\nUse the upper 4 bits of the first byte of the []byte value composed of nibbles as storage flag. The 0th bit stores the parity information, and the 1st bit stores the type represented by the value\n|hex char| bits | pointing to | odd/even | 2nd niddle padding |\n| -----|:----:|:----:|:----:|-------|\n|0| 0000 | node | even | no |\n|1| 0001 | node | odd | yes |\n|2| 0010 | value | even | no |\n|3| 0011 | value | odd | yes |\n\nthis encoding method is only used when accessing the database. After reading into memory, the key is directly stored in []byte type\n\nIn the trie module, there is a `Database` object, which you can understand as a cache layer of the underlying database. In actual use, the Trie object uses the Database as a database. However, the important function of Database is not caching, but the reference counting of node data during caching, and provides Database.Reference and Database.Dereference to reference and dereference a trie node. If the reference count of a node becomes 0, the node will be deleted from memory, so it will not be written to the real database\n\n# reference\n- [github](https://github.com/agiletechvn/go-ethereum-code-analysis/blob/master/trie-analysis.md)\n- [yangzhe's blog](http://yangzhe.me/2019/01/12/ethereum-trie-part-1/)\n- [yangzhe's blod](http://yangzhe.me/2019/01/18/ethereum-trie-part-2/)","slug":"MPT","published":1,"updated":"2023-11-05T04:21:13.732Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8dh0000qwsj6ebuelv1","content":"

trie

a trie, also called prefix tree, is a type of k-ary search tree. These keys are most often strings, with links between nodes defined not by the entire key, but by individual characters. In order to access a key, the trie is traversed depth-first, following the links between nodes, which represent each character in the key.

\n

\"prefix
In general, the nodes of a Trie look like this:

\n
1
[ [Ia, Ib, … I*], value]
\n

[Ia, Ib, … I*] is the index array of the node, which takes the next character in the key as the index, and each element I* points to the corresponding child node. value represents the value

\n

MPT

Each block of Ethereum contains three MPT trees, respectively

\n
    \n
  • Transaction tree
    • \n
  • Receipt tree
    • \n
  • State tree
    • \n
\n

In the figure below are two block headers, where state root, tx root receipt root stores the roots of the three trees, and the second block shows when the data of account 175 changes (27 -> 45). Only need to store 3 nodes related to this account, and the data in the old block can still be accessed normally. (This is somewhat similar to the implementation of an immutable data structure in a functional programming language.) The detailed structure is

\n

\"state

\n
    \n
  • use []byte as key, other than string
    • \n
  • nibble: the smallest unit of the key type (4 bit)
    • \n
  • Use hashes to refer to nodes instead of memory pointers
    • \n
\n

there are two types of node: full nodes (fullNode) and short nodes (shortNode). Full nodes have 17 elements, while shortNode nodes have two elements. Their schematic expressions are as follows

\n
1
2
fullNode: [i0, i1, i2, … i15, hash]  
shortNode： [ [k0, k1, … kn], hash ] // first element is an array
\n

if the hash pointing to a value, it is a leaf node; if pointing another node, a non leaf node. shortNode contains extension and leaf node. full node is branch node.

\n

\"mpt\"

\n

Use the upper 4 bits of the first byte of the []byte value composed of nibbles as storage flag. The 0th bit stores the parity information, and the 1st bit stores the type represented by the value

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
hex charbitspointing toodd/even2nd niddle padding
00000nodeevenno
10001nodeoddyes
20010valueevenno
30011valueoddyes
\n

this encoding method is only used when accessing the database. After reading into memory, the key is directly stored in []byte type

\n

In the trie module, there is a Database object, which you can understand as a cache layer of the underlying database. In actual use, the Trie object uses the Database as a database. However, the important function of Database is not caching, but the reference counting of node data during caching, and provides Database.Reference and Database.Dereference to reference and dereference a trie node. If the reference count of a node becomes 0, the node will be deleted from memory, so it will not be written to the real database

\n

reference

\n","site":{"data":{}},"excerpt":"","more":"

trie

a trie, also called prefix tree, is a type of k-ary search tree. These keys are most often strings, with links between nodes defined not by the entire key, but by individual characters. In order to access a key, the trie is traversed depth-first, following the links between nodes, which represent each character in the key.

\n

\"prefix
In general, the nodes of a Trie look like this:

\n
1
[ [Ia, Ib, … I*], value]
\n

[Ia, Ib, … I*] is the index array of the node, which takes the next character in the key as the index, and each element I* points to the corresponding child node. value represents the value

\n

MPT

Each block of Ethereum contains three MPT trees, respectively

\n
    \n
  • Transaction tree
    • \n
  • Receipt tree
    • \n
  • State tree
    • \n
\n

In the figure below are two block headers, where state root, tx root receipt root stores the roots of the three trees, and the second block shows when the data of account 175 changes (27 -> 45). Only need to store 3 nodes related to this account, and the data in the old block can still be accessed normally. (This is somewhat similar to the implementation of an immutable data structure in a functional programming language.) The detailed structure is

\n

\"state

\n
    \n
  • use []byte as key, other than string
    • \n
  • nibble: the smallest unit of the key type (4 bit)
    • \n
  • Use hashes to refer to nodes instead of memory pointers
    • \n
\n

there are two types of node: full nodes (fullNode) and short nodes (shortNode). Full nodes have 17 elements, while shortNode nodes have two elements. Their schematic expressions are as follows

\n
1
2
fullNode: [i0, i1, i2, … i15, hash]  
shortNode： [ [k0, k1, … kn], hash ] // first element is an array
\n

if the hash pointing to a value, it is a leaf node; if pointing another node, a non leaf node. shortNode contains extension and leaf node. full node is branch node.

\n

\"mpt\"

\n

Use the upper 4 bits of the first byte of the []byte value composed of nibbles as storage flag. The 0th bit stores the parity information, and the 1st bit stores the type represented by the value

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
hex charbitspointing toodd/even2nd niddle padding
00000nodeevenno
10001nodeoddyes
20010valueevenno
30011valueoddyes
\n

this encoding method is only used when accessing the database. After reading into memory, the key is directly stored in []byte type

\n

In the trie module, there is a Database object, which you can understand as a cache layer of the underlying database. In actual use, the Trie object uses the Database as a database. However, the important function of Database is not caching, but the reference counting of node data during caching, and provides Database.Reference and Database.Dereference to reference and dereference a trie node. If the reference count of a node becomes 0, the node will be deleted from memory, so it will not be written to the real database

\n

reference

\n"},{"title":"understanding of kademlia protocol","date":"2023-01-15T03:00:30.000Z","_content":"\n# Introduction\nKademlia, a peer-to-peer distributed hash table(DHT). Some other DHT techniques are Chord. In the Kad network, each node has a unique 160-bit ID (ETH account address is 20 bytes, which is 160 bits) pairs are stored in nodes whose ID is 'close' to the key for some notion of closeness. \n\n# system description\nKad effectively treats nodes as leaves in a binary tree, with each nodes's position determined by the shortest unique prefix of its ID. Figure 1 shows the position of a node with unique prefix 0011 in an example.\n\n![Figure 1](/images/kademlia.subtree.png)\nfor any given node, the binary tree is divided into a series of successively lower subtrees that don't contain the node.\n The highest-level subtree is composed of the other half of the whole tree that does not contain itself; the next level of subtree is composed of the remaining half that does not contain itself; and so on, until the complete tree is split into n subtrees. As shown in the figure, the part contained by the dotted line is the subtree of node 0011.\n\nif there is at least one node knwon in each subtree (in total, at least n nodes), a recursive routing algorithm can be used to reach any node within the binary tree. Figure 2 shows an example of node 0011 locating node 1110 by successively querying the best node it knows of to find contacts in lower and lower subtrees; finaly the lookup converges to the target node.\n\n![Figure 2](/images/kademlia.locating.png)\n\n## node distance: XOR metric\nNode's id is 160 bit. Keys are also 160 bit. The Kad algorithm uses an XOR operation to calculate the distance between nodes.\nd(x,y) = x XOR y\n\n## node state\nfor each 0 <= i < 160, every node keeps k triples (as a list) for nodes of distance between 2^i and 2^i+1 from itself. it is called k-buckets.\neach k-bucket is kept sorted by time last seen--least recently seen node at the head, most-recently seen at the tail.\nfor small values of i, the k-buckets will generally be empty (as no approriate nodes will exist).\n\n The K value mentioned here is a system-level constant (must be an even number). It is set by the software system using Kad (for example, the Kad network used by BT download, K is set to 8). \n\n## update of k bucket\nThere are mainly the following ways to update the K bucket:\n\n- Actively collect nodes: Any node can initiate a FIND_NODE (query node) request to refresh the node information in the K-bucket.\n- Passive collection node: When receiving requests from other nodes (such as: FIND_NODE, FIND_VALUE), it will add the node ID of the other party to a certain K-bucket.\n- Detect invalid nodes: By initiating a PING request, determine whether a node in the K-bucket is online, and then clean up which offline nodes in the K-bucket.\n\nThe update principle is to insert the latest node into the tail of the queue, and not to remove the online nodes in the queue.\n\nAccording to the K-bucket update principle, nodes with a long online time are more likely to remain in the K-bucket list. Therefore, by keeping the nodes with a long online time in the K-bucket, Kad will significantly increase the number of nodes in the K-bucket. it can defend against DOS attacks to a certain extent, because only when the old node fails, Kad will update the K bucket information, which avoids flooding routing information through the addition of new nodes\n![probability of continuous online agains onlie duration](/images/kademlia.onlineprob.png)\n\n## RPC method\nThe Kademlia protocol includes four remote RPC operations: PING, STORE, FIND_NODE, FIND_VALUE.\n\n- PING probes a node to see if it is online.\n- STORE instructs a node to store a pair for later retrieval.\n- FIND_NODE takes a 160bit ID as an argument. The receiver of this operation returns the (IP address, UDP port, Node ID) information of K nodes that it knows are closer to the target ID. The information of these nodes can be obtained from a single K-bucket, or from multiple K-buckets. In either case, the receiver will return the information of K nodes to the operation initiator. \n- The FIND_VALUE operation is similar to the FIND_NODE operation, with one exception. if the RPC receipients has received a STORE RPC for the key, it just returns the stored value.\n\n## node lookup\nKad participant must locate the k closest nodes to some given node ID. Kad employs a recursive algorithm for node lookups. It recursively send FIND_NODE requests to \\alpha (out of k) closest nodes it knows of. \n\n## find a pair\nto find a pair, a node starts by performing a lookup to find the k nodes with IDs closest to the key. However, value lookups use FIND_VLAUE rather than FIND_NODE RPCs.\n\n## node join\nto join the network, a node u must have a contact to an already participating node w.\n\n## routing table\n\n# references\n- [kademlia paper](https://pdos.csail.mit.edu/~petar/papers/maymounkov-kademlia-lncs.pdf)\n- [go implementation](https://github.com/libp2p/go-libp2p-kad-dht)\n- [kademlia stanford](https://codethechange.stanford.edu/guides/guide_kademlia.html)\n- [zhihu](https://zhuanlan.zhihu.com/p/388994038)","source":"_posts/blockchain.kademlia.md","raw":"---\ntitle: understanding of kademlia protocol\ndate: 2023-01-15 11:00:30\ntags: [blockchain]\n---\n\n# Introduction\nKademlia, a peer-to-peer distributed hash table(DHT). Some other DHT techniques are Chord. In the Kad network, each node has a unique 160-bit ID (ETH account address is 20 bytes, which is 160 bits) pairs are stored in nodes whose ID is 'close' to the key for some notion of closeness. \n\n# system description\nKad effectively treats nodes as leaves in a binary tree, with each nodes's position determined by the shortest unique prefix of its ID. Figure 1 shows the position of a node with unique prefix 0011 in an example.\n\n![Figure 1](/images/kademlia.subtree.png)\nfor any given node, the binary tree is divided into a series of successively lower subtrees that don't contain the node.\n The highest-level subtree is composed of the other half of the whole tree that does not contain itself; the next level of subtree is composed of the remaining half that does not contain itself; and so on, until the complete tree is split into n subtrees. As shown in the figure, the part contained by the dotted line is the subtree of node 0011.\n\nif there is at least one node knwon in each subtree (in total, at least n nodes), a recursive routing algorithm can be used to reach any node within the binary tree. Figure 2 shows an example of node 0011 locating node 1110 by successively querying the best node it knows of to find contacts in lower and lower subtrees; finaly the lookup converges to the target node.\n\n![Figure 2](/images/kademlia.locating.png)\n\n## node distance: XOR metric\nNode's id is 160 bit. Keys are also 160 bit. The Kad algorithm uses an XOR operation to calculate the distance between nodes.\nd(x,y) = x XOR y\n\n## node state\nfor each 0 <= i < 160, every node keeps k triples (as a list) for nodes of distance between 2^i and 2^i+1 from itself. it is called k-buckets.\neach k-bucket is kept sorted by time last seen--least recently seen node at the head, most-recently seen at the tail.\nfor small values of i, the k-buckets will generally be empty (as no approriate nodes will exist).\n\n The K value mentioned here is a system-level constant (must be an even number). It is set by the software system using Kad (for example, the Kad network used by BT download, K is set to 8). \n\n## update of k bucket\nThere are mainly the following ways to update the K bucket:\n\n- Actively collect nodes: Any node can initiate a FIND_NODE (query node) request to refresh the node information in the K-bucket.\n- Passive collection node: When receiving requests from other nodes (such as: FIND_NODE, FIND_VALUE), it will add the node ID of the other party to a certain K-bucket.\n- Detect invalid nodes: By initiating a PING request, determine whether a node in the K-bucket is online, and then clean up which offline nodes in the K-bucket.\n\nThe update principle is to insert the latest node into the tail of the queue, and not to remove the online nodes in the queue.\n\nAccording to the K-bucket update principle, nodes with a long online time are more likely to remain in the K-bucket list. Therefore, by keeping the nodes with a long online time in the K-bucket, Kad will significantly increase the number of nodes in the K-bucket. it can defend against DOS attacks to a certain extent, because only when the old node fails, Kad will update the K bucket information, which avoids flooding routing information through the addition of new nodes\n![probability of continuous online agains onlie duration](/images/kademlia.onlineprob.png)\n\n## RPC method\nThe Kademlia protocol includes four remote RPC operations: PING, STORE, FIND_NODE, FIND_VALUE.\n\n- PING probes a node to see if it is online.\n- STORE instructs a node to store a pair for later retrieval.\n- FIND_NODE takes a 160bit ID as an argument. The receiver of this operation returns the (IP address, UDP port, Node ID) information of K nodes that it knows are closer to the target ID. The information of these nodes can be obtained from a single K-bucket, or from multiple K-buckets. In either case, the receiver will return the information of K nodes to the operation initiator. \n- The FIND_VALUE operation is similar to the FIND_NODE operation, with one exception. if the RPC receipients has received a STORE RPC for the key, it just returns the stored value.\n\n## node lookup\nKad participant must locate the k closest nodes to some given node ID. Kad employs a recursive algorithm for node lookups. It recursively send FIND_NODE requests to \\alpha (out of k) closest nodes it knows of. \n\n## find a pair\nto find a pair, a node starts by performing a lookup to find the k nodes with IDs closest to the key. However, value lookups use FIND_VLAUE rather than FIND_NODE RPCs.\n\n## node join\nto join the network, a node u must have a contact to an already participating node w.\n\n## routing table\n\n# references\n- [kademlia paper](https://pdos.csail.mit.edu/~petar/papers/maymounkov-kademlia-lncs.pdf)\n- [go implementation](https://github.com/libp2p/go-libp2p-kad-dht)\n- [kademlia stanford](https://codethechange.stanford.edu/guides/guide_kademlia.html)\n- [zhihu](https://zhuanlan.zhihu.com/p/388994038)","slug":"blockchain.kademlia","published":1,"updated":"2023-11-05T04:21:13.732Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8dj0001qwsj9waw412a","content":"

Introduction

Kademlia, a peer-to-peer distributed hash table(DHT). Some other DHT techniques are Chord. In the Kad network, each node has a unique 160-bit ID (ETH account address is 20 bytes, which is 160 bits) <key,value> pairs are stored in nodes whose ID is ‘close’ to the key for some notion of closeness.

\n

system description

Kad effectively treats nodes as leaves in a binary tree, with each nodes’s position determined by the shortest unique prefix of its ID. Figure 1 shows the position of a node with unique prefix 0011 in an example.

\n

\"Figure
for any given node, the binary tree is divided into a series of successively lower subtrees that don’t contain the node.
The highest-level subtree is composed of the other half of the whole tree that does not contain itself; the next level of subtree is composed of the remaining half that does not contain itself; and so on, until the complete tree is split into n subtrees. As shown in the figure, the part contained by the dotted line is the subtree of node 0011.

\n

if there is at least one node knwon in each subtree (in total, at least n nodes), a recursive routing algorithm can be used to reach any node within the binary tree. Figure 2 shows an example of node 0011 locating node 1110 by successively querying the best node it knows of to find contacts in lower and lower subtrees; finaly the lookup converges to the target node.

\n

\"Figure

\n

node distance: XOR metric

Node’s id is 160 bit. Keys are also 160 bit. The Kad algorithm uses an XOR operation to calculate the distance between nodes.
d(x,y) = x XOR y

\n

node state

for each 0 <= i < 160, every node keeps k <IP address, UDP port, node ID> triples (as a list) for nodes of distance between 2^i and 2^i+1 from itself. it is called k-buckets.
each k-bucket is kept sorted by time last seen–least recently seen node at the head, most-recently seen at the tail.
for small values of i, the k-buckets will generally be empty (as no approriate nodes will exist).

\n

The K value mentioned here is a system-level constant (must be an even number). It is set by the software system using Kad (for example, the Kad network used by BT download, K is set to 8).

\n

update of k bucket

There are mainly the following ways to update the K bucket:

\n
    \n
  • Actively collect nodes: Any node can initiate a FIND_NODE (query node) request to refresh the node information in the K-bucket.
    • \n
  • Passive collection node: When receiving requests from other nodes (such as: FIND_NODE, FIND_VALUE), it will add the node ID of the other party to a certain K-bucket.
    • \n
  • Detect invalid nodes: By initiating a PING request, determine whether a node in the K-bucket is online, and then clean up which offline nodes in the K-bucket.
    • \n
\n

The update principle is to insert the latest node into the tail of the queue, and not to remove the online nodes in the queue.

\n

According to the K-bucket update principle, nodes with a long online time are more likely to remain in the K-bucket list. Therefore, by keeping the nodes with a long online time in the K-bucket, Kad will significantly increase the number of nodes in the K-bucket. it can defend against DOS attacks to a certain extent, because only when the old node fails, Kad will update the K bucket information, which avoids flooding routing information through the addition of new nodes
\"probability

\n

RPC method

The Kademlia protocol includes four remote RPC operations: PING, STORE, FIND_NODE, FIND_VALUE.

\n
    \n
  • PING probes a node to see if it is online.
    • \n
  • STORE instructs a node to store a <key,value> pair for later retrieval.
    • \n
  • FIND_NODE takes a 160bit ID as an argument. The receiver of this operation returns the (IP address, UDP port, Node ID) information of K nodes that it knows are closer to the target ID. The information of these nodes can be obtained from a single K-bucket, or from multiple K-buckets. In either case, the receiver will return the information of K nodes to the operation initiator.
    • \n
  • The FIND_VALUE operation is similar to the FIND_NODE operation, with one exception. if the RPC receipients has received a STORE RPC for the key, it just returns the stored value.
    • \n
\n

node lookup

Kad participant must locate the k closest nodes to some given node ID. Kad employs a recursive algorithm for node lookups. It recursively send FIND_NODE requests to \\alpha (out of k) closest nodes it knows of.

\n

find a <key,value> pair

to find a <key,value> pair, a node starts by performing a lookup to find the k nodes with IDs closest to the key. However, value lookups use FIND_VLAUE rather than FIND_NODE RPCs.

\n

node join

to join the network, a node u must have a contact to an already participating node w.

\n

routing table

references

\n","site":{"data":{}},"excerpt":"","more":"

Introduction

Kademlia, a peer-to-peer distributed hash table(DHT). Some other DHT techniques are Chord. In the Kad network, each node has a unique 160-bit ID (ETH account address is 20 bytes, which is 160 bits) <key,value> pairs are stored in nodes whose ID is ‘close’ to the key for some notion of closeness.

\n

system description

Kad effectively treats nodes as leaves in a binary tree, with each nodes’s position determined by the shortest unique prefix of its ID. Figure 1 shows the position of a node with unique prefix 0011 in an example.

\n

\"Figure
for any given node, the binary tree is divided into a series of successively lower subtrees that don’t contain the node.
The highest-level subtree is composed of the other half of the whole tree that does not contain itself; the next level of subtree is composed of the remaining half that does not contain itself; and so on, until the complete tree is split into n subtrees. As shown in the figure, the part contained by the dotted line is the subtree of node 0011.

\n

if there is at least one node knwon in each subtree (in total, at least n nodes), a recursive routing algorithm can be used to reach any node within the binary tree. Figure 2 shows an example of node 0011 locating node 1110 by successively querying the best node it knows of to find contacts in lower and lower subtrees; finaly the lookup converges to the target node.

\n

\"Figure

\n

node distance: XOR metric

Node’s id is 160 bit. Keys are also 160 bit. The Kad algorithm uses an XOR operation to calculate the distance between nodes.
d(x,y) = x XOR y

\n

node state

for each 0 <= i < 160, every node keeps k <IP address, UDP port, node ID> triples (as a list) for nodes of distance between 2^i and 2^i+1 from itself. it is called k-buckets.
each k-bucket is kept sorted by time last seen–least recently seen node at the head, most-recently seen at the tail.
for small values of i, the k-buckets will generally be empty (as no approriate nodes will exist).

\n

The K value mentioned here is a system-level constant (must be an even number). It is set by the software system using Kad (for example, the Kad network used by BT download, K is set to 8).

\n

update of k bucket

There are mainly the following ways to update the K bucket:

\n
    \n
  • Actively collect nodes: Any node can initiate a FIND_NODE (query node) request to refresh the node information in the K-bucket.
    • \n
  • Passive collection node: When receiving requests from other nodes (such as: FIND_NODE, FIND_VALUE), it will add the node ID of the other party to a certain K-bucket.
    • \n
  • Detect invalid nodes: By initiating a PING request, determine whether a node in the K-bucket is online, and then clean up which offline nodes in the K-bucket.
    • \n
\n

The update principle is to insert the latest node into the tail of the queue, and not to remove the online nodes in the queue.

\n

According to the K-bucket update principle, nodes with a long online time are more likely to remain in the K-bucket list. Therefore, by keeping the nodes with a long online time in the K-bucket, Kad will significantly increase the number of nodes in the K-bucket. it can defend against DOS attacks to a certain extent, because only when the old node fails, Kad will update the K bucket information, which avoids flooding routing information through the addition of new nodes
\"probability

\n

RPC method

The Kademlia protocol includes four remote RPC operations: PING, STORE, FIND_NODE, FIND_VALUE.

\n
    \n
  • PING probes a node to see if it is online.
    • \n
  • STORE instructs a node to store a <key,value> pair for later retrieval.
    • \n
  • FIND_NODE takes a 160bit ID as an argument. The receiver of this operation returns the (IP address, UDP port, Node ID) information of K nodes that it knows are closer to the target ID. The information of these nodes can be obtained from a single K-bucket, or from multiple K-buckets. In either case, the receiver will return the information of K nodes to the operation initiator.
    • \n
  • The FIND_VALUE operation is similar to the FIND_NODE operation, with one exception. if the RPC receipients has received a STORE RPC for the key, it just returns the stored value.
    • \n
\n

node lookup

Kad participant must locate the k closest nodes to some given node ID. Kad employs a recursive algorithm for node lookups. It recursively send FIND_NODE requests to \\alpha (out of k) closest nodes it knows of.

\n

find a <key,value> pair

to find a <key,value> pair, a node starts by performing a lookup to find the k nodes with IDs closest to the key. However, value lookups use FIND_VLAUE rather than FIND_NODE RPCs.

\n

node join

to join the network, a node u must have a contact to an already participating node w.

\n

routing table

references

\n"},{"title":"geth_fine_tune","date":"2023-01-01T08:29:43.000Z","_content":"\n\nIf we're a full node on mainnet without --cache specified, bump default cache allowance\nctx.Set(utils.CacheFlag.Name, strconv.Itoa(4096))","source":"_posts/geth-fine-tune.md","raw":"---\ntitle: geth_fine_tune\ndate: 2023-01-01 16:29:43\ntags:\n---\n\n\nIf we're a full node on mainnet without --cache specified, bump default cache allowance\nctx.Set(utils.CacheFlag.Name, strconv.Itoa(4096))","slug":"geth-fine-tune","published":1,"updated":"2023-11-05T04:21:13.733Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8dl0003qwsj0la65sux","content":"

If we’re a full node on mainnet without –cache specified, bump default cache allowance
ctx.Set(utils.CacheFlag.Name, strconv.Itoa(4096))

\n","site":{"data":{}},"excerpt":"","more":"

If we’re a full node on mainnet without –cache specified, bump default cache allowance
ctx.Set(utils.CacheFlag.Name, strconv.Itoa(4096))

\n"},{"title":"how to use hexo","date":"2022-11-27T03:47:56.000Z","_content":"Welcome to [Hexo](https://hexo.io/)! This is your very first post. Check [documentation](https://hexo.io/docs/) for more info. If you get any problems when using Hexo, you can find the answer in [troubleshooting](https://hexo.io/docs/troubleshooting.html) or you can ask me on [GitHub](https://github.com/hexojs/hexo/issues).\n\n## Quick Start\n\n### Create a new post\n\n``` bash\n$ hexo new \"My New Post\"\n```\n\nMore info: [Writing](https://hexo.io/docs/writing.html)\n\n### Run server\n\n``` bash\n$ hexo server\n```\n\nMore info: [Server](https://hexo.io/docs/server.html)\n\n### Generate static files\n\n``` bash\n$ hexo generate\n```\n\nMore info: [Generating](https://hexo.io/docs/generating.html)\n\n### Deploy to remote sites\n\n``` bash\n$ hexo deploy\n```\n\nMore info: [Deployment](https://hexo.io/docs/one-command-deployment.html)\n","source":"_posts/hello-world.md","raw":"---\ntitle: how to use hexo\ndate: 2022-11-27 11:47:56\n---\nWelcome to [Hexo](https://hexo.io/)! This is your very first post. Check [documentation](https://hexo.io/docs/) for more info. If you get any problems when using Hexo, you can find the answer in [troubleshooting](https://hexo.io/docs/troubleshooting.html) or you can ask me on [GitHub](https://github.com/hexojs/hexo/issues).\n\n## Quick Start\n\n### Create a new post\n\n``` bash\n$ hexo new \"My New Post\"\n```\n\nMore info: [Writing](https://hexo.io/docs/writing.html)\n\n### Run server\n\n``` bash\n$ hexo server\n```\n\nMore info: [Server](https://hexo.io/docs/server.html)\n\n### Generate static files\n\n``` bash\n$ hexo generate\n```\n\nMore info: [Generating](https://hexo.io/docs/generating.html)\n\n### Deploy to remote sites\n\n``` bash\n$ hexo deploy\n```\n\nMore info: [Deployment](https://hexo.io/docs/one-command-deployment.html)\n","slug":"hello-world","published":1,"updated":"2023-11-05T04:21:13.733Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8dl0004qwsj06vjfxz2","content":"

Welcome to Hexo! This is your very first post. Check documentation for more info. If you get any problems when using Hexo, you can find the answer in troubleshooting or you can ask me on GitHub.

\n

Quick Start

Create a new post

1
$ hexo new "My New Post"
\n\n

More info: Writing

\n

Run server

1
$ hexo server
\n\n

More info: Server

\n

Generate static files

1
$ hexo generate
\n\n

More info: Generating

\n

Deploy to remote sites

1
$ hexo deploy
\n\n

More info: Deployment

\n","site":{"data":{}},"excerpt":"","more":"

Welcome to Hexo! This is your very first post. Check documentation for more info. If you get any problems when using Hexo, you can find the answer in troubleshooting or you can ask me on GitHub.

\n

Quick Start

Create a new post

1
$ hexo new "My New Post"
\n\n

More info: Writing

\n

Run server

1
$ hexo server
\n\n

More info: Server

\n

Generate static files

1
$ hexo generate
\n\n

More info: Generating

\n

Deploy to remote sites

1
$ hexo deploy
\n\n

More info: Deployment

\n"},{"title":"danksharding","date":"2023-10-12T06:36:58.000Z","_content":"\n\n\n\n## introduction\nDanksharding is central to Ethereum’s rollup-centric roadmap. The idea is to provide space for large blobs of data, which are verifiable and available, without attempting to interpret them.\n\n\n## Preliminaries\nAll references to EC, in this report, refer to BLS12_381 [1]. We are mainly concerned with the EC Abelian-group \\\\(\\mathbb{G_{1}}\\\\) and the EC scalar-field \\\\(F_r\\\\). Both are of size \\\\(r < 2256\\\\). Note that \\\\(2^{32}\\\\) divides the size of the multiplicative-group in \\\\(F_r\\\\)\n\n## Data organization\nThe input data consists of n = 256 shard-blobs. Each shard-blob is a vector of m = 4096 field-elements referred-to as symbols. The data symbols are organized in an n × m input\nmatrix\n\\\\[\n \\tag{1}\n D_{n x m}^{in} =\n\\begin{bmatrix}\n\\displaylines{\n d(0,0) & d(0,1) & \\dots & d(0,m-1)\\\\\\\\\n d(1,0) & d(1,1) & \\dots & d(1,m-1)\\\\\\\\\n \\vdots & \\vdots & \\ddots & \\vdots\\\\\\\\\n d(n-1,0) & d(n-1,1) & \\dots & d(n-1,m-1)\n}\n\\end{bmatrix}\n\\\\]\nwhere each row is one of the shard-blob vectors [2]\nIn order to enable interpolation and polynomial-commitment to the data, we will pro-\nceed to treat the data symbols as polynomial evaluations.\n\nLet us thus associate each domain location in the input matrix with a field-element pair \\\\(u_{\\mu}, \\omega_{\\eta}\\\\), where \\\\(\\mu \\in [0, n−1], \\eta \\in [0, m−1]\\\\) correspond to the row and column indexes, respectively.The row field-element is defined as \\\\( u_{\\mu} \\equiv u^{rbo(\\mu)}\\\\), where u is a 2n’th root-of-unity such that \\\\(u^{2n} = 1\\\\). The column field-element is defined as \\\\( \\omega_{\\eta} \\equiv \\omega^{rbo(\\eta)}\\\\), where \\\\(\\omega\\\\) is a 2m’th root-of-unity such that \\\\(\\omega^{2m} = 1\\\\). _Using reverse-bit-order ordering rather than natural-ordering allows accessing cosets in block (consecutive) rather than interleaved manner_\n\n## coefficients extraction\nTaking the data symbols to be evaluations of a 2D-polynomial or 1D-product-polynomials with row degree n−1 and column degree m−1 uniquely defines the polynomials’ coefficients.\n\n### 2D coeficients extraction\nThe 2D-polynomial representing the input data can be expressed as\n\\\\[\\tag{2} d(x,y) \\equiv \\sum_{i=0}^{n-1}\\sum_{j=0}^{m-1} \\hat{c}[i,j] x^{i}y^{j}\\\\]\nPlugging an evaluation from (1) into (2) results in the following:\n\n\\\\[\\tag{3} d(u_{\\mu},\\omega_{\\eta}) \\equiv \\sum_{i=0}^{n-1}\\sum_{j=0}^{m-1} \\hat{c}[i,j] {u_{\\mu}}^{i}{\\omega_{\\eta}}^{j}\\\\]\n## references\n[1] Ben Edgington. Bls12-381 for the rest of us. https://hackmd.io/@benjaminion/bls12-381.\n[2] Dankrad Feist. Data availability encoding. https://notes.ethereum.org/@dankrad/danksharding_encoding","source":"_posts/blockchain/danksharding.md","raw":"---\ntitle: danksharding\ndate: 2023-10-12 14:36:58\ntags: [blockchain]\n---\n\n\n\n\n## introduction\nDanksharding is central to Ethereum’s rollup-centric roadmap. The idea is to provide space for large blobs of data, which are verifiable and available, without attempting to interpret them.\n\n\n## Preliminaries\nAll references to EC, in this report, refer to BLS12_381 [1]. We are mainly concerned with the EC Abelian-group \\\\(\\mathbb{G_{1}}\\\\) and the EC scalar-field \\\\(F_r\\\\). Both are of size \\\\(r < 2256\\\\). Note that \\\\(2^{32}\\\\) divides the size of the multiplicative-group in \\\\(F_r\\\\)\n\n## Data organization\nThe input data consists of n = 256 shard-blobs. Each shard-blob is a vector of m = 4096 field-elements referred-to as symbols. The data symbols are organized in an n × m input\nmatrix\n\\\\[\n \\tag{1}\n D_{n x m}^{in} =\n\\begin{bmatrix}\n\\displaylines{\n d(0,0) & d(0,1) & \\dots & d(0,m-1)\\\\\\\\\n d(1,0) & d(1,1) & \\dots & d(1,m-1)\\\\\\\\\n \\vdots & \\vdots & \\ddots & \\vdots\\\\\\\\\n d(n-1,0) & d(n-1,1) & \\dots & d(n-1,m-1)\n}\n\\end{bmatrix}\n\\\\]\nwhere each row is one of the shard-blob vectors [2]\nIn order to enable interpolation and polynomial-commitment to the data, we will pro-\nceed to treat the data symbols as polynomial evaluations.\n\nLet us thus associate each domain location in the input matrix with a field-element pair \\\\(u_{\\mu}, \\omega_{\\eta}\\\\), where \\\\(\\mu \\in [0, n−1], \\eta \\in [0, m−1]\\\\) correspond to the row and column indexes, respectively.The row field-element is defined as \\\\( u_{\\mu} \\equiv u^{rbo(\\mu)}\\\\), where u is a 2n’th root-of-unity such that \\\\(u^{2n} = 1\\\\). The column field-element is defined as \\\\( \\omega_{\\eta} \\equiv \\omega^{rbo(\\eta)}\\\\), where \\\\(\\omega\\\\) is a 2m’th root-of-unity such that \\\\(\\omega^{2m} = 1\\\\). _Using reverse-bit-order ordering rather than natural-ordering allows accessing cosets in block (consecutive) rather than interleaved manner_\n\n## coefficients extraction\nTaking the data symbols to be evaluations of a 2D-polynomial or 1D-product-polynomials with row degree n−1 and column degree m−1 uniquely defines the polynomials’ coefficients.\n\n### 2D coeficients extraction\nThe 2D-polynomial representing the input data can be expressed as\n\\\\[\\tag{2} d(x,y) \\equiv \\sum_{i=0}^{n-1}\\sum_{j=0}^{m-1} \\hat{c}[i,j] x^{i}y^{j}\\\\]\nPlugging an evaluation from (1) into (2) results in the following:\n\n\\\\[\\tag{3} d(u_{\\mu},\\omega_{\\eta}) \\equiv \\sum_{i=0}^{n-1}\\sum_{j=0}^{m-1} \\hat{c}[i,j] {u_{\\mu}}^{i}{\\omega_{\\eta}}^{j}\\\\]\n## references\n[1] Ben Edgington. Bls12-381 for the rest of us. https://hackmd.io/@benjaminion/bls12-381.\n[2] Dankrad Feist. Data availability encoding. https://notes.ethereum.org/@dankrad/danksharding_encoding","slug":"blockchain/danksharding","published":1,"updated":"2023-11-05T04:21:17.033Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8dm0005qwsja95zaawe","content":"\n\n\n

introduction

Danksharding is central to Ethereum’s rollup-centric roadmap. The idea is to provide space for large blobs of data, which are verifiable and available, without attempting to interpret them.

\n

Preliminaries

All references to EC, in this report, refer to BLS12_381 [1]. We are mainly concerned with the EC Abelian-group \\(\\mathbb{G_{1}}\\) and the EC scalar-field \\(F_r\\). Both are of size \\(r < 2256\\). Note that \\(2^{32}\\) divides the size of the multiplicative-group in \\(F_r\\)

\n

Data organization

The input data consists of n = 256 shard-blobs. Each shard-blob is a vector of m = 4096 field-elements referred-to as symbols. The data symbols are organized in an n × m input
matrix
\\[
\\tag{1}
D_{n x m}^{in} =
\\begin{bmatrix}
\\displaylines{
d(0,0) & d(0,1) & \\dots & d(0,m-1)\\\\
d(1,0) & d(1,1) & \\dots & d(1,m-1)\\\\
\\vdots & \\vdots & \\ddots & \\vdots\\\\
d(n-1,0) & d(n-1,1) & \\dots & d(n-1,m-1)
}
\\end{bmatrix}
\\]
where each row is one of the shard-blob vectors [2]
In order to enable interpolation and polynomial-commitment to the data, we will pro-
ceed to treat the data symbols as polynomial evaluations.

\n

Let us thus associate each domain location in the input matrix with a field-element pair \\(u_{\\mu}, \\omega_{\\eta}\\), where \\(\\mu \\in [0, n−1], \\eta \\in [0, m−1]\\) correspond to the row and column indexes, respectively.The row field-element is defined as \\( u_{\\mu} \\equiv u^{rbo(\\mu)}\\), where u is a 2n’th root-of-unity such that \\(u^{2n} = 1\\). The column field-element is defined as \\( \\omega_{\\eta} \\equiv \\omega^{rbo(\\eta)}\\), where \\(\\omega\\) is a 2m’th root-of-unity such that \\(\\omega^{2m} = 1\\). Using reverse-bit-order ordering rather than natural-ordering allows accessing cosets in block (consecutive) rather than interleaved manner

\n

coefficients extraction

Taking the data symbols to be evaluations of a 2D-polynomial or 1D-product-polynomials with row degree n−1 and column degree m−1 uniquely defines the polynomials’ coefficients.

\n

2D coeficients extraction

The 2D-polynomial representing the input data can be expressed as
\\[\\tag{2} d(x,y) \\equiv \\sum_{i=0}^{n-1}\\sum_{j=0}^{m-1} \\hat{c}[i,j] x^{i}y^{j}\\]
Plugging an evaluation from (1) into (2) results in the following:

\n

\\[\\tag{3} d(u_{\\mu},\\omega_{\\eta}) \\equiv \\sum_{i=0}^{n-1}\\sum_{j=0}^{m-1} \\hat{c}[i,j] {u_{\\mu}}^{i}{\\omega_{\\eta}}^{j}\\]

\n

references

[1] Ben Edgington. Bls12-381 for the rest of us. https://hackmd.io/@benjaminion/bls12-381.
[2] Dankrad Feist. Data availability encoding. https://notes.ethereum.org/@dankrad/danksharding_encoding

\n","site":{"data":{}},"excerpt":"","more":"\n\n\n

introduction

Danksharding is central to Ethereum’s rollup-centric roadmap. The idea is to provide space for large blobs of data, which are verifiable and available, without attempting to interpret them.

\n

Preliminaries

All references to EC, in this report, refer to BLS12_381 [1]. We are mainly concerned with the EC Abelian-group \\(\\mathbb{G_{1}}\\) and the EC scalar-field \\(F_r\\). Both are of size \\(r < 2256\\). Note that \\(2^{32}\\) divides the size of the multiplicative-group in \\(F_r\\)

\n

Data organization

The input data consists of n = 256 shard-blobs. Each shard-blob is a vector of m = 4096 field-elements referred-to as symbols. The data symbols are organized in an n × m input
matrix
\\[
\\tag{1}
D_{n x m}^{in} =
\\begin{bmatrix}
\\displaylines{
d(0,0) & d(0,1) & \\dots & d(0,m-1)\\\\
d(1,0) & d(1,1) & \\dots & d(1,m-1)\\\\
\\vdots & \\vdots & \\ddots & \\vdots\\\\
d(n-1,0) & d(n-1,1) & \\dots & d(n-1,m-1)
}
\\end{bmatrix}
\\]
where each row is one of the shard-blob vectors [2]
In order to enable interpolation and polynomial-commitment to the data, we will pro-
ceed to treat the data symbols as polynomial evaluations.

\n

Let us thus associate each domain location in the input matrix with a field-element pair \\(u_{\\mu}, \\omega_{\\eta}\\), where \\(\\mu \\in [0, n−1], \\eta \\in [0, m−1]\\) correspond to the row and column indexes, respectively.The row field-element is defined as \\( u_{\\mu} \\equiv u^{rbo(\\mu)}\\), where u is a 2n’th root-of-unity such that \\(u^{2n} = 1\\). The column field-element is defined as \\( \\omega_{\\eta} \\equiv \\omega^{rbo(\\eta)}\\), where \\(\\omega\\) is a 2m’th root-of-unity such that \\(\\omega^{2m} = 1\\). Using reverse-bit-order ordering rather than natural-ordering allows accessing cosets in block (consecutive) rather than interleaved manner

\n

coefficients extraction

Taking the data symbols to be evaluations of a 2D-polynomial or 1D-product-polynomials with row degree n−1 and column degree m−1 uniquely defines the polynomials’ coefficients.

\n

2D coeficients extraction

The 2D-polynomial representing the input data can be expressed as
\\[\\tag{2} d(x,y) \\equiv \\sum_{i=0}^{n-1}\\sum_{j=0}^{m-1} \\hat{c}[i,j] x^{i}y^{j}\\]
Plugging an evaluation from (1) into (2) results in the following:

\n

\\[\\tag{3} d(u_{\\mu},\\omega_{\\eta}) \\equiv \\sum_{i=0}^{n-1}\\sum_{j=0}^{m-1} \\hat{c}[i,j] {u_{\\mu}}^{i}{\\omega_{\\eta}}^{j}\\]

\n

references

[1] Ben Edgington. Bls12-381 for the rest of us. https://hackmd.io/@benjaminion/bls12-381.
[2] Dankrad Feist. Data availability encoding. https://notes.ethereum.org/@dankrad/danksharding_encoding

\n"},{"title":"cryptography (1) group & field","date":"2023-06-03T06:29:26.000Z","_content":"\n\n\n\n\n## Group\na group is a set of elements \\\\( G \\\\) together with an operation \\\\( \\circ \\\\). a group has the following properties\n1. the group operation \\\\( \\circ \\\\) is closed. that is, for all \\\\( a,b, \\in G \\\\), it holds that \\\\( a \\circ b = c \\in G \\\\)\n2. the group operation is associative. That is, \\\\( a \\circ (b \\circ c) = (a \\circ b) \\circ c \\\\) for all \\\\( a,b,c \\in G \\\\)\n3. there is an element \\\\( 1 \\in G \\\\), called the neutral element (or identity element), such that \\\\( a \\circ \\ = 1 \\circ a\\\\) for all \\\\( a \\in G \\\\)\n4. For each \\\\( a \\in G \\\\) there exists an element \\\\( a^{-1} \\in G\\\\), called the inverse of a, such that \\\\( a \\circ a^{-1} = 1 \\\\).\n5. a group G is abelian (or commutative) if, furthermore, \\\\( a \\circ b = b \\circ a \\\\) for all \\\\( a, b \\in G \\\\) \n\n### Example of Group\nthe set of integers \\\\( Z_{m} = 0,1,...,m-1 \\\\) and the operation addition modulo \\\\( m \\\\) forma group with the neutral element 0. every element \\\\( a \\\\) has an inverse \\\\( -a \\\\). note that this set does not form a group with the operation multiplication gecause mose eleents \\\\( a \\\\) do not have an inverse such that \\\\( a a^{-1} = 1 \\bmod m\\\\).\n\nIn order to have all four basic arithmetic operations (i.e. addition, subtraction, multiplication, division) in one structure, we need a set which contains an additive and a multiplicative group. this is what we call a field.\n\n## Field\nA field is a set of elements with the following properties\n- all elements of \\\\( F\\\\) form an additive group with the group operation \\\\( + \\\\) and the neutral element \\\\( 0 \\\\)\n- all element of \\\\( F \\\\) except \\\\( 0 \\\\) form a multiplicative group with the gorup operation \\\\( x \\\\) and the neutral element \\\\( 1 \\\\) .\n- when the two group operations are mixed, the distributivety law holds, i.e., for all \\\\( a,b,c \\in F: a(b+c) = (ab) + (ac) \\\\) \n\n### Example of Field\nthe set \\\\( R \\\\) of real numbers is a field with the neutral element 0 for the additive group and the neutral element 1 for the multiplicative group. every real number \\\\( a \\\\) has an additive inverse, namely \\\\( -a \\\\), and every nonzero element \\\\( a \\\\) has a multiplicative inverse \\\\( 1 \\div a \\\\)\n\n## Galois Field\nIn cryptography, we are almose always interested in fields with a **finite** number of elements, which we call finite fields or `Galois field`. the number of elements in the field is called the `order` or `cardinality` of the field. Of fundamental importance is the following theorem\n***\na field with order `m` only exists if `m` is a prime power, i.e, \\\\( m = p^{n} \\\\), for some positive integer `n` and prime integer `p`. `p` is called the characteristic of the finite field\n***\n\nthe theorem implies that there are, for instance, finite fields with 11 elements (\\\\( 11^{1} \\\\)), or with 81 elements \\\\( 81 = 3^{4} \\\\) or with 256 elements (\\\\( 256 = 2^{8} \\\\)). \n\n## Prime Field\nthe most intuitive examples of finite fields are fields of prime order, i.e., fields with \\\\( n =1 \\\\). elements of the field \\\\( GF(p) \\\\) can be represented by integers \\\\( 0,1,..., p-1 \\\\).\n\n## Extension Field `GF(2^m)`\nIn AES the finite field contains 256 elements and is denoted as \\\\( GF(2^8) \\\\).\nHowever, if the order of a finite field is not prime, and \\\\( 2^8 \\\\) is clearly not a prime, the addition and multiplication operation cannot be represented by addition and multiplication of integers modulo \\\\( 2^8 \\\\). such fields with \\\\( m >1 \\\\) are called `extension field`. in order to deal with extension fields we need (1) a different notation for field elements and (2) different rules for performing arithmetic with the elements. elements of extension fields can be represented as `polynomials`, and computation in the extension field is achieved by performing a certain type of `polynomial arithmetic`.\n\nIn the field \\\\( GF(2^8) \\\\), which is used in AES, each element \\\\( A \\in GF(2^8) \\\\) is thus represented as: \n\\\\[ A(x) = a_{7}x^7 + ...+a_{1}x+a_{0}, a_{i} \\in GF(2) = {0,1} \\\\]\nIt is also important to observe that every polynomial can simply be stored in digital form as an 8-bit vector\n\\\\[ A = (a_7,a_6,a_5,a_4,a_3,a_2,a_1,a_0) \\\\]\n\n### addition and subtraction in `GF(2^m)`\naddition and subtraction are simply achieved by performing polynomial addition and subtraction. note that we perform modulo 2 addition (or subtraction) with the coefficients.\n\\\\[ A(x) = x^7 + x^6 + x^4 + 1 \\\\]\n\\\\[ B(x) = x^4 + x^2+ 1 \\\\]\n\\\\[ A(x) + B(x) = x^7 + x^6+ x^2 \\\\]\n\n### multiplication in `GF(2^m)`\nfirstly, two elements (represented by their polynomials) of a finite field `GF(2^m)` are multiplied usig the standard polynomial multiplication rule \\\\[ A(x) \\dot B(x) = C(x) \\\\]. In general, the product polynomial \\\\[ C(x) \\\\] will have a degree higher than `m-1` and has to be reduced. to do that, the product of the multiplication is divided by a certain polyhomial, and we consider only the remainder after the polynomial division. we need **irreducible** polynomials for the module reduction. irreducible polynomials are roughly comparable to prime numbers. their only factors are 1 and polynomial itself.\nThus, every field `GF(2^m)` requires an irreducible polynomial `P(x)` of degree `m`. \nFor AES, the irreducible polynomial is \n\\\\[ P(x) = x^8 + x^4+ x^3 +x + 1 \\\\]\nthe main algorithm for computing multiplicative inverse is the extended Euclidean algorithm, which is introduced in other posts.","source":"_posts/cryptography/cryptography-01-primitive-group-and-field.md","raw":"---\ntitle: cryptography (1) group & field\ndate: 2023-06-03 14:29:26\ntags: [cryptography]\n---\n\n\n\n\n\n## Group\na group is a set of elements \\\\( G \\\\) together with an operation \\\\( \\circ \\\\). a group has the following properties\n1. the group operation \\\\( \\circ \\\\) is closed. that is, for all \\\\( a,b, \\in G \\\\), it holds that \\\\( a \\circ b = c \\in G \\\\)\n2. the group operation is associative. That is, \\\\( a \\circ (b \\circ c) = (a \\circ b) \\circ c \\\\) for all \\\\( a,b,c \\in G \\\\)\n3. there is an element \\\\( 1 \\in G \\\\), called the neutral element (or identity element), such that \\\\( a \\circ \\ = 1 \\circ a\\\\) for all \\\\( a \\in G \\\\)\n4. For each \\\\( a \\in G \\\\) there exists an element \\\\( a^{-1} \\in G\\\\), called the inverse of a, such that \\\\( a \\circ a^{-1} = 1 \\\\).\n5. a group G is abelian (or commutative) if, furthermore, \\\\( a \\circ b = b \\circ a \\\\) for all \\\\( a, b \\in G \\\\) \n\n### Example of Group\nthe set of integers \\\\( Z_{m} = 0,1,...,m-1 \\\\) and the operation addition modulo \\\\( m \\\\) forma group with the neutral element 0. every element \\\\( a \\\\) has an inverse \\\\( -a \\\\). note that this set does not form a group with the operation multiplication gecause mose eleents \\\\( a \\\\) do not have an inverse such that \\\\( a a^{-1} = 1 \\bmod m\\\\).\n\nIn order to have all four basic arithmetic operations (i.e. addition, subtraction, multiplication, division) in one structure, we need a set which contains an additive and a multiplicative group. this is what we call a field.\n\n## Field\nA field is a set of elements with the following properties\n- all elements of \\\\( F\\\\) form an additive group with the group operation \\\\( + \\\\) and the neutral element \\\\( 0 \\\\)\n- all element of \\\\( F \\\\) except \\\\( 0 \\\\) form a multiplicative group with the gorup operation \\\\( x \\\\) and the neutral element \\\\( 1 \\\\) .\n- when the two group operations are mixed, the distributivety law holds, i.e., for all \\\\( a,b,c \\in F: a(b+c) = (ab) + (ac) \\\\) \n\n### Example of Field\nthe set \\\\( R \\\\) of real numbers is a field with the neutral element 0 for the additive group and the neutral element 1 for the multiplicative group. every real number \\\\( a \\\\) has an additive inverse, namely \\\\( -a \\\\), and every nonzero element \\\\( a \\\\) has a multiplicative inverse \\\\( 1 \\div a \\\\)\n\n## Galois Field\nIn cryptography, we are almose always interested in fields with a **finite** number of elements, which we call finite fields or `Galois field`. the number of elements in the field is called the `order` or `cardinality` of the field. Of fundamental importance is the following theorem\n***\na field with order `m` only exists if `m` is a prime power, i.e, \\\\( m = p^{n} \\\\), for some positive integer `n` and prime integer `p`. `p` is called the characteristic of the finite field\n***\n\nthe theorem implies that there are, for instance, finite fields with 11 elements (\\\\( 11^{1} \\\\)), or with 81 elements \\\\( 81 = 3^{4} \\\\) or with 256 elements (\\\\( 256 = 2^{8} \\\\)). \n\n## Prime Field\nthe most intuitive examples of finite fields are fields of prime order, i.e., fields with \\\\( n =1 \\\\). elements of the field \\\\( GF(p) \\\\) can be represented by integers \\\\( 0,1,..., p-1 \\\\).\n\n## Extension Field `GF(2^m)`\nIn AES the finite field contains 256 elements and is denoted as \\\\( GF(2^8) \\\\).\nHowever, if the order of a finite field is not prime, and \\\\( 2^8 \\\\) is clearly not a prime, the addition and multiplication operation cannot be represented by addition and multiplication of integers modulo \\\\( 2^8 \\\\). such fields with \\\\( m >1 \\\\) are called `extension field`. in order to deal with extension fields we need (1) a different notation for field elements and (2) different rules for performing arithmetic with the elements. elements of extension fields can be represented as `polynomials`, and computation in the extension field is achieved by performing a certain type of `polynomial arithmetic`.\n\nIn the field \\\\( GF(2^8) \\\\), which is used in AES, each element \\\\( A \\in GF(2^8) \\\\) is thus represented as: \n\\\\[ A(x) = a_{7}x^7 + ...+a_{1}x+a_{0}, a_{i} \\in GF(2) = {0,1} \\\\]\nIt is also important to observe that every polynomial can simply be stored in digital form as an 8-bit vector\n\\\\[ A = (a_7,a_6,a_5,a_4,a_3,a_2,a_1,a_0) \\\\]\n\n### addition and subtraction in `GF(2^m)`\naddition and subtraction are simply achieved by performing polynomial addition and subtraction. note that we perform modulo 2 addition (or subtraction) with the coefficients.\n\\\\[ A(x) = x^7 + x^6 + x^4 + 1 \\\\]\n\\\\[ B(x) = x^4 + x^2+ 1 \\\\]\n\\\\[ A(x) + B(x) = x^7 + x^6+ x^2 \\\\]\n\n### multiplication in `GF(2^m)`\nfirstly, two elements (represented by their polynomials) of a finite field `GF(2^m)` are multiplied usig the standard polynomial multiplication rule \\\\[ A(x) \\dot B(x) = C(x) \\\\]. In general, the product polynomial \\\\[ C(x) \\\\] will have a degree higher than `m-1` and has to be reduced. to do that, the product of the multiplication is divided by a certain polyhomial, and we consider only the remainder after the polynomial division. we need **irreducible** polynomials for the module reduction. irreducible polynomials are roughly comparable to prime numbers. their only factors are 1 and polynomial itself.\nThus, every field `GF(2^m)` requires an irreducible polynomial `P(x)` of degree `m`. \nFor AES, the irreducible polynomial is \n\\\\[ P(x) = x^8 + x^4+ x^3 +x + 1 \\\\]\nthe main algorithm for computing multiplicative inverse is the extended Euclidean algorithm, which is introduced in other posts.","slug":"cryptography/cryptography-01-primitive-group-and-field","published":1,"updated":"2023-11-05T04:21:13.732Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8dm0009qwsj4fr9fx90","content":"\n\n\n\n\n

Group

a group is a set of elements \\( G \\) together with an operation \\( \\circ \\). a group has the following properties

\n
    \n
  1. the group operation \\( \\circ \\) is closed. that is, for all \\( a,b, \\in G \\), it holds that \\( a \\circ b = c \\in G \\)
    2. \n
  2. the group operation is associative. That is, \\( a \\circ (b \\circ c) = (a \\circ b) \\circ c \\) for all \\( a,b,c \\in G \\)
    3. \n
  3. there is an element \\( 1 \\in G \\), called the neutral element (or identity element), such that \\( a \\circ \\ = 1 \\circ a\\) for all \\( a \\in G \\)
    4. \n
  4. For each \\( a \\in G \\) there exists an element \\( a^{-1} \\in G\\), called the inverse of a, such that \\( a \\circ a^{-1} = 1 \\).
    5. \n
  5. a group G is abelian (or commutative) if, furthermore, \\( a \\circ b = b \\circ a \\) for all \\( a, b \\in G \\)
    6. \n
\n

Example of Group

the set of integers \\( Z_{m} = 0,1,…,m-1 \\) and the operation addition modulo \\( m \\) forma group with the neutral element 0. every element \\( a \\) has an inverse \\( -a \\). note that this set does not form a group with the operation multiplication gecause mose eleents \\( a \\) do not have an inverse such that \\( a a^{-1} = 1 \\bmod m\\).

\n

In order to have all four basic arithmetic operations (i.e. addition, subtraction, multiplication, division) in one structure, we need a set which contains an additive and a multiplicative group. this is what we call a field.

\n

Field

A field is a set of elements with the following properties

\n
    \n
  • all elements of \\( F\\) form an additive group with the group operation \\( + \\) and the neutral element \\( 0 \\)
    • \n
  • all element of \\( F \\) except \\( 0 \\) form a multiplicative group with the gorup operation \\( x \\) and the neutral element \\( 1 \\) .
    • \n
  • when the two group operations are mixed, the distributivety law holds, i.e., for all \\( a,b,c \\in F: a(b+c) = (ab) + (ac) \\)
    • \n
\n

Example of Field

the set \\( R \\) of real numbers is a field with the neutral element 0 for the additive group and the neutral element 1 for the multiplicative group. every real number \\( a \\) has an additive inverse, namely \\( -a \\), and every nonzero element \\( a \\) has a multiplicative inverse \\( 1 \\div a \\)

\n

Galois Field

In cryptography, we are almose always interested in fields with a finite number of elements, which we call finite fields or Galois field. the number of elements in the field is called the order or cardinality of the field. Of fundamental importance is the following theorem

\n
\n

a field with order m only exists if m is a prime power, i.e, \\( m = p^{n} \\), for some positive integer n and prime integer p. p is called the characteristic of the finite field

\n
\n

the theorem implies that there are, for instance, finite fields with 11 elements (\\( 11^{1} \\)), or with 81 elements \\( 81 = 3^{4} \\) or with 256 elements (\\( 256 = 2^{8} \\)).

\n

Prime Field

the most intuitive examples of finite fields are fields of prime order, i.e., fields with \\( n =1 \\). elements of the field \\( GF(p) \\) can be represented by integers \\( 0,1,…, p-1 \\).

\n

Extension Field GF(2^m)

In AES the finite field contains 256 elements and is denoted as \\( GF(2^8) \\).
However, if the order of a finite field is not prime, and \\( 2^8 \\) is clearly not a prime, the addition and multiplication operation cannot be represented by addition and multiplication of integers modulo \\( 2^8 \\). such fields with \\( m >1 \\) are called extension field. in order to deal with extension fields we need (1) a different notation for field elements and (2) different rules for performing arithmetic with the elements. elements of extension fields can be represented as polynomials, and computation in the extension field is achieved by performing a certain type of polynomial arithmetic.

\n

In the field \\( GF(2^8) \\), which is used in AES, each element \\( A \\in GF(2^8) \\) is thus represented as:
\\[ A(x) = a_{7}x^7 + …+a_{1}x+a_{0}, a_{i} \\in GF(2) = {0,1} \\]
It is also important to observe that every polynomial can simply be stored in digital form as an 8-bit vector
\\[ A = (a_7,a_6,a_5,a_4,a_3,a_2,a_1,a_0) \\]

\n

addition and subtraction in GF(2^m)

addition and subtraction are simply achieved by performing polynomial addition and subtraction. note that we perform modulo 2 addition (or subtraction) with the coefficients.
\\[ A(x) = x^7 + x^6 + x^4 + 1 \\]
\\[ B(x) = x^4 + x^2+ 1 \\]
\\[ A(x) + B(x) = x^7 + x^6+ x^2 \\]

\n

multiplication in GF(2^m)

firstly, two elements (represented by their polynomials) of a finite field GF(2^m) are multiplied usig the standard polynomial multiplication rule \\[ A(x) \\dot B(x) = C(x) \\]. In general, the product polynomial \\[ C(x) \\] will have a degree higher than m-1 and has to be reduced. to do that, the product of the multiplication is divided by a certain polyhomial, and we consider only the remainder after the polynomial division. we need irreducible polynomials for the module reduction. irreducible polynomials are roughly comparable to prime numbers. their only factors are 1 and polynomial itself.
Thus, every field GF(2^m) requires an irreducible polynomial P(x) of degree m.
For AES, the irreducible polynomial is
\\[ P(x) = x^8 + x^4+ x^3 +x + 1 \\]
the main algorithm for computing multiplicative inverse is the extended Euclidean algorithm, which is introduced in other posts.

\n","site":{"data":{}},"excerpt":"","more":"\n\n\n\n\n

Group

a group is a set of elements \\( G \\) together with an operation \\( \\circ \\). a group has the following properties

\n
    \n
  1. the group operation \\( \\circ \\) is closed. that is, for all \\( a,b, \\in G \\), it holds that \\( a \\circ b = c \\in G \\)
    2. \n
  2. the group operation is associative. That is, \\( a \\circ (b \\circ c) = (a \\circ b) \\circ c \\) for all \\( a,b,c \\in G \\)
    3. \n
  3. there is an element \\( 1 \\in G \\), called the neutral element (or identity element), such that \\( a \\circ \\ = 1 \\circ a\\) for all \\( a \\in G \\)
    4. \n
  4. For each \\( a \\in G \\) there exists an element \\( a^{-1} \\in G\\), called the inverse of a, such that \\( a \\circ a^{-1} = 1 \\).
    5. \n
  5. a group G is abelian (or commutative) if, furthermore, \\( a \\circ b = b \\circ a \\) for all \\( a, b \\in G \\)
    6. \n
\n

Example of Group

the set of integers \\( Z_{m} = 0,1,…,m-1 \\) and the operation addition modulo \\( m \\) forma group with the neutral element 0. every element \\( a \\) has an inverse \\( -a \\). note that this set does not form a group with the operation multiplication gecause mose eleents \\( a \\) do not have an inverse such that \\( a a^{-1} = 1 \\bmod m\\).

\n

In order to have all four basic arithmetic operations (i.e. addition, subtraction, multiplication, division) in one structure, we need a set which contains an additive and a multiplicative group. this is what we call a field.

\n

Field

A field is a set of elements with the following properties

\n
    \n
  • all elements of \\( F\\) form an additive group with the group operation \\( + \\) and the neutral element \\( 0 \\)
    • \n
  • all element of \\( F \\) except \\( 0 \\) form a multiplicative group with the gorup operation \\( x \\) and the neutral element \\( 1 \\) .
    • \n
  • when the two group operations are mixed, the distributivety law holds, i.e., for all \\( a,b,c \\in F: a(b+c) = (ab) + (ac) \\)
    • \n
\n

Example of Field

the set \\( R \\) of real numbers is a field with the neutral element 0 for the additive group and the neutral element 1 for the multiplicative group. every real number \\( a \\) has an additive inverse, namely \\( -a \\), and every nonzero element \\( a \\) has a multiplicative inverse \\( 1 \\div a \\)

\n

Galois Field

In cryptography, we are almose always interested in fields with a finite number of elements, which we call finite fields or Galois field. the number of elements in the field is called the order or cardinality of the field. Of fundamental importance is the following theorem

\n
\n

a field with order m only exists if m is a prime power, i.e, \\( m = p^{n} \\), for some positive integer n and prime integer p. p is called the characteristic of the finite field

\n
\n

the theorem implies that there are, for instance, finite fields with 11 elements (\\( 11^{1} \\)), or with 81 elements \\( 81 = 3^{4} \\) or with 256 elements (\\( 256 = 2^{8} \\)).

\n

Prime Field

the most intuitive examples of finite fields are fields of prime order, i.e., fields with \\( n =1 \\). elements of the field \\( GF(p) \\) can be represented by integers \\( 0,1,…, p-1 \\).

\n

Extension Field GF(2^m)

In AES the finite field contains 256 elements and is denoted as \\( GF(2^8) \\).
However, if the order of a finite field is not prime, and \\( 2^8 \\) is clearly not a prime, the addition and multiplication operation cannot be represented by addition and multiplication of integers modulo \\( 2^8 \\). such fields with \\( m >1 \\) are called extension field. in order to deal with extension fields we need (1) a different notation for field elements and (2) different rules for performing arithmetic with the elements. elements of extension fields can be represented as polynomials, and computation in the extension field is achieved by performing a certain type of polynomial arithmetic.

\n

In the field \\( GF(2^8) \\), which is used in AES, each element \\( A \\in GF(2^8) \\) is thus represented as:
\\[ A(x) = a_{7}x^7 + …+a_{1}x+a_{0}, a_{i} \\in GF(2) = {0,1} \\]
It is also important to observe that every polynomial can simply be stored in digital form as an 8-bit vector
\\[ A = (a_7,a_6,a_5,a_4,a_3,a_2,a_1,a_0) \\]

\n

addition and subtraction in GF(2^m)

addition and subtraction are simply achieved by performing polynomial addition and subtraction. note that we perform modulo 2 addition (or subtraction) with the coefficients.
\\[ A(x) = x^7 + x^6 + x^4 + 1 \\]
\\[ B(x) = x^4 + x^2+ 1 \\]
\\[ A(x) + B(x) = x^7 + x^6+ x^2 \\]

\n

multiplication in GF(2^m)

firstly, two elements (represented by their polynomials) of a finite field GF(2^m) are multiplied usig the standard polynomial multiplication rule \\[ A(x) \\dot B(x) = C(x) \\]. In general, the product polynomial \\[ C(x) \\] will have a degree higher than m-1 and has to be reduced. to do that, the product of the multiplication is divided by a certain polyhomial, and we consider only the remainder after the polynomial division. we need irreducible polynomials for the module reduction. irreducible polynomials are roughly comparable to prime numbers. their only factors are 1 and polynomial itself.
Thus, every field GF(2^m) requires an irreducible polynomial P(x) of degree m.
For AES, the irreducible polynomial is
\\[ P(x) = x^8 + x^4+ x^3 +x + 1 \\]
the main algorithm for computing multiplicative inverse is the extended Euclidean algorithm, which is introduced in other posts.

\n"},{"title":"cryptography (3) elliptic curve","date":"2023-06-17T06:29:26.000Z","_content":"\n\n\n\n## elliptic curve definition\nfor cryptographic use, we need to conside the curve not over the real numbers but over a finite field. the most popular fields `GF(p)`, where all arithmetic is performed modulo a prime p.\n***\nthe elliptic curve over \\\\( Z_{p}, p>3 \\\\), is the set of all pairs \\\\( (x,y) \\in Z_{p} \\\\) which fulfill\n \\\\[ y^2 \\equiv x^3 + a \\cdot x + b \\bmod p \\\\]\ntogether with an imaginary point of infinity \\\\( \\mathcal{O} \\\\), where\n \\\\[ a,b \\in Z_{p} \\\\]\n and the condition \\\\( 4 \\cdot a^3 + 27 \\cdot b^2 \\neq 0 \\bmod p \\\\)\n***\nthe definition of elliptic curve requires that the curve is nonsingular. Geometrically speaking, this means that the plot has no self-intersections or vertices, which is achieved if the discriminant of the curve \\\\( -16(4a^3) + 27b^2 \\\\) is nonzero.\n\n## operations on elliptic curve\n![point addition](/images/cryptography/elliptic_curve/point_addition.webp)\nlet's denote the group operation with the addition symbol `+`. \"addition\" means that given two points and their coordinates, say \\\\( P = (x_1, y_1) \\\\) and \\\\( Q = (x_2, y_2) \\\\), we have to compute the coordidnate of a third point \\\\( R (x_3, y_3) \\\\). the construction works as follows: draw a line through P and Q and obtain a third point of intersection between the elliptic curve and the line (-R). Mirror this third intersection point along the x-axis. this mirored point is, by definition, the point R. point doubling `(P+P = 2P)` is a tangent line through the point P, and the rest is same.\nthe fomulae for Point Addition (P+Q) and Point Doubling (2P) is as below \n***\n \\\\[ x_3 = s^2 - x_1 -x_2 \\bmod p \\\\]\n \\\\[ y_3 = s(x_1 - x_3) - y_1 \\bmod p\\\\]\n where \n\\begin{equation}\ns = \n\\begin{cases}\n\\frac{y_2-y_1}{x_2-x_1} \\bmod p & if P \\neq Q (\\text{point addition}) \\cr\n\\frac{3x_{1}^{2} + a}{2y_1} \\bmod p & if P =Q (\\text{point doubling})\n\\end{cases}\n\\end{equation}\n***\nnote that the parameter s is the slope of the line through P and Q in the case of point addition, or the slope of the tangent through P in the case of point doubling.\nFurthmore, we define an abstract point at infinity as the neutral element \\\\( \\mathcal{O} \\\\). \n\\\\[ P + \\mathcal{O} = P\\\\]\naccording the group definition, we can now also define the inverse `-P` of any group element P as\n\\\\[ P + (-P) = \\mathcal{O}\\\\]\nTherefore, the inverse of \\\\( P = (x_p, y_p) \\\\) is just \\\\( P = (x_p, -y_p)\\\\). since \\\\( -y_p \\equiv p - y_p\\\\), hence\n\\\\[ -P = (x_p, p-y_p) \\\\]\n\n## DLP(discrete log problem) with Elliptic curve\nto set up DL cryptosystems it is important to know the order of the group.\nHasse's theorem\n***\ngiven an elliptic curve E modulo p, the number of points on the curve is denoted by #E and is bounded by:\n\\\\[ p+1-2\\sqrt{p} \\leq \\\\#E \\leq p+1+2\\sqrt{p} \\\\]\n***\nElliptic Curved Discrete Logarithm Problem (ECDLP)\n***\nGiven an elliptic curve E. We consider a primitive elemnt P and another element T. The DL problem is finding the integer d, where \\\\( 1 \\leq d \\leq \\\\#E \\\\), such that:\n\\\\[ P + P + ....+ P = dP = T \\\\]\n***\nIn cryptosystems, d is the private key which is an integer, while the public key T is a point on the curve with coordinates \\\\( T = (x_T, y_T)\\\\)\nUsually a **square-and-multiply** algorithm is used to calculate point multiplication (the algorithm detail is not coverred in this post). \n","source":"_posts/cryptography/cryptography-03-elliptic-curve.md","raw":"---\ntitle: cryptography (3) elliptic curve\ndate: 2023-06-17 14:29:26\ntags: [cryptography]\n---\n\n\n\n\n## elliptic curve definition\nfor cryptographic use, we need to conside the curve not over the real numbers but over a finite field. the most popular fields `GF(p)`, where all arithmetic is performed modulo a prime p.\n***\nthe elliptic curve over \\\\( Z_{p}, p>3 \\\\), is the set of all pairs \\\\( (x,y) \\in Z_{p} \\\\) which fulfill\n \\\\[ y^2 \\equiv x^3 + a \\cdot x + b \\bmod p \\\\]\ntogether with an imaginary point of infinity \\\\( \\mathcal{O} \\\\), where\n \\\\[ a,b \\in Z_{p} \\\\]\n and the condition \\\\( 4 \\cdot a^3 + 27 \\cdot b^2 \\neq 0 \\bmod p \\\\)\n***\nthe definition of elliptic curve requires that the curve is nonsingular. Geometrically speaking, this means that the plot has no self-intersections or vertices, which is achieved if the discriminant of the curve \\\\( -16(4a^3) + 27b^2 \\\\) is nonzero.\n\n## operations on elliptic curve\n![point addition](/images/cryptography/elliptic_curve/point_addition.webp)\nlet's denote the group operation with the addition symbol `+`. \"addition\" means that given two points and their coordinates, say \\\\( P = (x_1, y_1) \\\\) and \\\\( Q = (x_2, y_2) \\\\), we have to compute the coordidnate of a third point \\\\( R (x_3, y_3) \\\\). the construction works as follows: draw a line through P and Q and obtain a third point of intersection between the elliptic curve and the line (-R). Mirror this third intersection point along the x-axis. this mirored point is, by definition, the point R. point doubling `(P+P = 2P)` is a tangent line through the point P, and the rest is same.\nthe fomulae for Point Addition (P+Q) and Point Doubling (2P) is as below \n***\n \\\\[ x_3 = s^2 - x_1 -x_2 \\bmod p \\\\]\n \\\\[ y_3 = s(x_1 - x_3) - y_1 \\bmod p\\\\]\n where \n\\begin{equation}\ns = \n\\begin{cases}\n\\frac{y_2-y_1}{x_2-x_1} \\bmod p & if P \\neq Q (\\text{point addition}) \\cr\n\\frac{3x_{1}^{2} + a}{2y_1} \\bmod p & if P =Q (\\text{point doubling})\n\\end{cases}\n\\end{equation}\n***\nnote that the parameter s is the slope of the line through P and Q in the case of point addition, or the slope of the tangent through P in the case of point doubling.\nFurthmore, we define an abstract point at infinity as the neutral element \\\\( \\mathcal{O} \\\\). \n\\\\[ P + \\mathcal{O} = P\\\\]\naccording the group definition, we can now also define the inverse `-P` of any group element P as\n\\\\[ P + (-P) = \\mathcal{O}\\\\]\nTherefore, the inverse of \\\\( P = (x_p, y_p) \\\\) is just \\\\( P = (x_p, -y_p)\\\\). since \\\\( -y_p \\equiv p - y_p\\\\), hence\n\\\\[ -P = (x_p, p-y_p) \\\\]\n\n## DLP(discrete log problem) with Elliptic curve\nto set up DL cryptosystems it is important to know the order of the group.\nHasse's theorem\n***\ngiven an elliptic curve E modulo p, the number of points on the curve is denoted by #E and is bounded by:\n\\\\[ p+1-2\\sqrt{p} \\leq \\\\#E \\leq p+1+2\\sqrt{p} \\\\]\n***\nElliptic Curved Discrete Logarithm Problem (ECDLP)\n***\nGiven an elliptic curve E. We consider a primitive elemnt P and another element T. The DL problem is finding the integer d, where \\\\( 1 \\leq d \\leq \\\\#E \\\\), such that:\n\\\\[ P + P + ....+ P = dP = T \\\\]\n***\nIn cryptosystems, d is the private key which is an integer, while the public key T is a point on the curve with coordinates \\\\( T = (x_T, y_T)\\\\)\nUsually a **square-and-multiply** algorithm is used to calculate point multiplication (the algorithm detail is not coverred in this post). \n","slug":"cryptography/cryptography-03-elliptic-curve","published":1,"updated":"2023-11-05T04:21:17.033Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8dn000bqwsj9gt33o5h","content":"\n\n\n\n

elliptic curve definition

for cryptographic use, we need to conside the curve not over the real numbers but over a finite field. the most popular fields GF(p), where all arithmetic is performed modulo a prime p.

\n
\n

the elliptic curve over \\( Z_{p}, p>3 \\), is the set of all pairs \\( (x,y) \\in Z_{p} \\) which fulfill
\\[ y^2 \\equiv x^3 + a \\cdot x + b \\bmod p \\]
together with an imaginary point of infinity \\( \\mathcal{O} \\), where
\\[ a,b \\in Z_{p} \\]
and the condition \\( 4 \\cdot a^3 + 27 \\cdot b^2 \\neq 0 \\bmod p \\)

\n
\n

the definition of elliptic curve requires that the curve is nonsingular. Geometrically speaking, this means that the plot has no self-intersections or vertices, which is achieved if the discriminant of the curve \\( -16(4a^3) + 27b^2 \\) is nonzero.

\n

operations on elliptic curve

\"point
let’s denote the group operation with the addition symbol +. “addition” means that given two points and their coordinates, say \\( P = (x_1, y_1) \\) and \\( Q = (x_2, y_2) \\), we have to compute the coordidnate of a third point \\( R (x_3, y_3) \\). the construction works as follows: draw a line through P and Q and obtain a third point of intersection between the elliptic curve and the line (-R). Mirror this third intersection point along the x-axis. this mirored point is, by definition, the point R. point doubling (P+P = 2P) is a tangent line through the point P, and the rest is same.
the fomulae for Point Addition (P+Q) and Point Doubling (2P) is as below

\n
\n

\\[ x_3 = s^2 - x_1 -x_2 \\bmod p \\]
\\[ y_3 = s(x_1 - x_3) - y_1 \\bmod p\\]
where
\\begin{equation}
s =
\\begin{cases}
\\frac{y_2-y_1}{x_2-x_1} \\bmod p & if P \\neq Q (\\text{point addition}) \\cr
\\frac{3x_{1}^{2} + a}{2y_1} \\bmod p & if P =Q (\\text{point doubling})
\\end{cases}
\\end{equation}

\n
\n

note that the parameter s is the slope of the line through P and Q in the case of point addition, or the slope of the tangent through P in the case of point doubling.
Furthmore, we define an abstract point at infinity as the neutral element \\( \\mathcal{O} \\).
\\[ P + \\mathcal{O} = P\\]
according the group definition, we can now also define the inverse -P of any group element P as
\\[ P + (-P) = \\mathcal{O}\\]
Therefore, the inverse of \\( P = (x_p, y_p) \\) is just \\( P = (x_p, -y_p)\\). since \\( -y_p \\equiv p - y_p\\), hence
\\[ -P = (x_p, p-y_p) \\]

\n

DLP(discrete log problem) with Elliptic curve

to set up DL cryptosystems it is important to know the order of the group.
Hasse’s theorem

\n
\n

given an elliptic curve E modulo p, the number of points on the curve is denoted by #E and is bounded by:
\\[ p+1-2\\sqrt{p} \\leq \\#E \\leq p+1+2\\sqrt{p} \\]

\n
\n

Elliptic Curved Discrete Logarithm Problem (ECDLP)

\n
\n

Given an elliptic curve E. We consider a primitive elemnt P and another element T. The DL problem is finding the integer d, where \\( 1 \\leq d \\leq \\#E \\), such that:
\\[ P + P + ….+ P = dP = T \\]

\n
\n

In cryptosystems, d is the private key which is an integer, while the public key T is a point on the curve with coordinates \\( T = (x_T, y_T)\\)
Usually a square-and-multiply algorithm is used to calculate point multiplication (the algorithm detail is not coverred in this post).

\n","site":{"data":{}},"excerpt":"","more":"\n\n\n\n

elliptic curve definition

for cryptographic use, we need to conside the curve not over the real numbers but over a finite field. the most popular fields GF(p), where all arithmetic is performed modulo a prime p.

\n
\n

the elliptic curve over \\( Z_{p}, p>3 \\), is the set of all pairs \\( (x,y) \\in Z_{p} \\) which fulfill
\\[ y^2 \\equiv x^3 + a \\cdot x + b \\bmod p \\]
together with an imaginary point of infinity \\( \\mathcal{O} \\), where
\\[ a,b \\in Z_{p} \\]
and the condition \\( 4 \\cdot a^3 + 27 \\cdot b^2 \\neq 0 \\bmod p \\)

\n
\n

the definition of elliptic curve requires that the curve is nonsingular. Geometrically speaking, this means that the plot has no self-intersections or vertices, which is achieved if the discriminant of the curve \\( -16(4a^3) + 27b^2 \\) is nonzero.

\n

operations on elliptic curve

\"point
let’s denote the group operation with the addition symbol +. “addition” means that given two points and their coordinates, say \\( P = (x_1, y_1) \\) and \\( Q = (x_2, y_2) \\), we have to compute the coordidnate of a third point \\( R (x_3, y_3) \\). the construction works as follows: draw a line through P and Q and obtain a third point of intersection between the elliptic curve and the line (-R). Mirror this third intersection point along the x-axis. this mirored point is, by definition, the point R. point doubling (P+P = 2P) is a tangent line through the point P, and the rest is same.
the fomulae for Point Addition (P+Q) and Point Doubling (2P) is as below

\n
\n

\\[ x_3 = s^2 - x_1 -x_2 \\bmod p \\]
\\[ y_3 = s(x_1 - x_3) - y_1 \\bmod p\\]
where
\\begin{equation}
s =
\\begin{cases}
\\frac{y_2-y_1}{x_2-x_1} \\bmod p & if P \\neq Q (\\text{point addition}) \\cr
\\frac{3x_{1}^{2} + a}{2y_1} \\bmod p & if P =Q (\\text{point doubling})
\\end{cases}
\\end{equation}

\n
\n

note that the parameter s is the slope of the line through P and Q in the case of point addition, or the slope of the tangent through P in the case of point doubling.
Furthmore, we define an abstract point at infinity as the neutral element \\( \\mathcal{O} \\).
\\[ P + \\mathcal{O} = P\\]
according the group definition, we can now also define the inverse -P of any group element P as
\\[ P + (-P) = \\mathcal{O}\\]
Therefore, the inverse of \\( P = (x_p, y_p) \\) is just \\( P = (x_p, -y_p)\\). since \\( -y_p \\equiv p - y_p\\), hence
\\[ -P = (x_p, p-y_p) \\]

\n

DLP(discrete log problem) with Elliptic curve

to set up DL cryptosystems it is important to know the order of the group.
Hasse’s theorem

\n
\n

given an elliptic curve E modulo p, the number of points on the curve is denoted by #E and is bounded by:
\\[ p+1-2\\sqrt{p} \\leq \\#E \\leq p+1+2\\sqrt{p} \\]

\n
\n

Elliptic Curved Discrete Logarithm Problem (ECDLP)

\n
\n

Given an elliptic curve E. We consider a primitive elemnt P and another element T. The DL problem is finding the integer d, where \\( 1 \\leq d \\leq \\#E \\), such that:
\\[ P + P + ….+ P = dP = T \\]

\n
\n

In cryptosystems, d is the private key which is an integer, while the public key T is a point on the curve with coordinates \\( T = (x_T, y_T)\\)
Usually a square-and-multiply algorithm is used to calculate point multiplication (the algorithm detail is not coverred in this post).

\n"},{"title":"cryptography (4) digital signature","date":"2023-06-20T06:29:26.000Z","_content":"\n\n\n## Elgamal Digital Signature Scheme\nThe Elgammal signature scheme is based on the difficulty of computing discrete logarithms. Unlike RSA, where encryption and digital signature are almoste identical operations, the Elgamal digital signature is quite different from the encryption scheme with teh same name.\n\n### key generation\n***\n1. Choose a large prime \\\\(p\\\\).\n2. Choose a primitive element \\\\(\\alpha\\\\) of \\\\(Z_{p}^{\\ast}\\\\), or a subgroup of \\\\(Z_{p}^{\\ast}\\\\).\n3. Choose a random integer \\\\(d \\in {2,3,...,p-2}\\\\)\n4. Compute \\\\(\\beta = \\alpha^{d} \\bmod p\\\\)\n***\nThe public key is now formed by \\\\(k_{pub} = (p, \\alpha, \\beta)\\\\), and the private key by \\\\(k_{pr}=d\\\\)\n\\\\(Z_{p}^{\\ast}\\\\) is the set of integers who are smaller than \\\\(p\\\\) and coprime to \\\\(p\\\\)\n\n### signature and verification\nUsign the private ey and parameters of the public key, the signature\n\\\\[sig_{k_{pr}}(x, k_{E}) = (r,s)\\\\]\n\\\\(x\\\\) is the message. \\\\(k_{E}\\\\) is a random value, which forms an ephemeral private key\n***\n**Elgamal Signature Generation**\n1. choose a random ephemeral key \\\\(k_{E} \\in {0,1,2,..,p-2}\\\\) such that \\\\(gcd(k_{E}, p-1) = 1\\\\)\n2. compute the signatue parameters:\n\\\\[r \\equiv \\alpha^{k_{E}} \\bmod p\\\\]\n\\\\[s \\equiv (x - d \\cdot r)k_{E}^{-1} \\bmod p-1\\\\]\n***\non the receiving side, the signature is verified as \\\\(ver_{k_{pub}}(x,(r,s))\\\\) using the public key, the signature and the message.\n***\n**Elgamal Signature Verification**\n1. comput the value \n\\\\[t \\equiv \\beta^{r} \\cdot r^s \\bmod p\\\\]\n2. the verification follows form\n\\begin{equation}\nt = \n\\begin{cases}\n\\equiv \\alpha^{x} \\bmod p & => \\text{valid signature} \\cr\n\\not\\equiv \\alpha^{x} \\bmod p & => \\text{invalid signature}\n\\end{cases}\n\\end{equation}\n***\n\n## Digital Signature Algorithm (DSA)\nThe native Elgamal signature algorithm described above is rarely used in practice. Instead, a much more popular variant is used, known as the Digital Signature Algorithm (DSA). It is a federal US government standard for digital signatures and was proposed by NIST (National Institute of Standards and Technology). Its main advantages over the Elgamal signature scheme are that the signature is only 320-bit long and that some of the attacks\n that can threaten the Elgamal scheme are not applicable.\n### key generation\n***\n**key Generation for DSA**\n1. Generate a prime \\\\(p\\\\) with \\\\(2^1023 < p < 2^1024\\\\)\n2. find a prime divisor \\\\(q\\\\) of \\\\(p-1\\\\) \\\\(2^159 < q < 2^160\\\\)\n3. Find an element \\\\(\\alpha\\\\) with \\\\( ord(\\alpha) = q\\\\), i.e., \\alpha genertes the subgroup with \\\\(q\\\\) elements.\n4. choose a random integer \\\\(d\\\\) with \\\\(0 < d < q\\\\).\n5. compute \\\\(\\beta \\equiv \\alpha^{d} \\bmod p\\\\).\nthe keys are now:\n\\\\(k_{pub} = (p, q, \\alpha, \\beta)\\\\)\n\\\\(k_{pr}= (d)\\\\)\n***\nThe central idea of DSA is that there are two cyclic groups involved. One is the large cyclic group \\\\(Z_{p}*{\\ast}\\\\), the order of which has bit length of 1024 bit. The second one is in the 160-bit subgroup of \\\\(Z_{p}^{\\ast}\\\\). this set-up yields shorter signature.\n\n### Signature and Verification\nAs in the Elgamal signatue scheme, the DSA signature consists of a pair of integers \\\\((r,s)\\\\). Since each of the two parameters is only 160-bit long, the total signature length is 320 bit. \n***\n**DSA signature generation**\n1. choose an integer as random ephemeral key \\\\(k_{E}\\\\) with \\\\(0 < k_{E} < q\\\\).\n2. compute \\\\(r \\equiv (\\alpha^{k_{E}} \\bmod p) \\bmod q\\\\)\n3. compute \\\\(s \\equiv (SHA(x) + d \\cdot r)k_{E}^{-1} \\bmod q \\\\)\n***\n\nThe signature verification process is as follows:\n***\n**DSA signature verification**\n1. compute auxilary value \\\\(w \\equiv s^{-1} \\bmod q\\\\).\n2. compute auxilary value \\\\(u_{1} \\equiv w \\cdot SHA(x) \\bmod q\\\\).\n3. compute auxilary value \\\\(u_{2} \\equiv w \\cdot r \\bmod q\\\\).\n4. compute \\\\(v \\equiv (\\alpha^{u_1} \\cdot \\beta^{u_2} \\bmod p) \\bmod q\\\\).\n5. the verification \\\\(ver_{k_{pub}}(x, (r,s))\\\\) folows from\n\\begin{equation}\nv = \n\\begin{cases}\n\\equiv r \\bmod q & => \\text{valid signature} \\cr\n\\not\\equiv r \\bmod q & => \\text{invalid signature}\n\\end{cases}\n\\end{equation}\n***\n\n## Elliptic Curve Digital Signature Algorithm (ECDSA)\nElliptic curves have several advantages over RSA and over DL schemes like Elgamal or DSA. In particular, in abscence of strong attacks against elliptic curve cryptosystems (ECC), bit lengths in the range of 160-256 bit can be chosen which provide security equivalent to 1024-3072 bit RSA and DL scheme. The shorter bit length of ECC often results in shorter processing time and in shorter signatures. \nThe ECDSA standard is defined for elliptic curves over prime fields \\\\(Z_{p}\\\\) adn Galois fields \\\\(GF(2^m)\\\\). the former is often preferred in practice, and we only introduce this one in what follows\n### key generation\n***\n**Key Generation for ECDSA**\n1. Use and elliptic curve E with \n- modulus p\n- coefficients a and b\n- a point A which generates a cyclic group of prime order q.\n2. choose a random integer d with \\\\(0 < d < q\\\\)\n3. compute \\\\(B = dA\\\\).\nThe keys are now\n\\\\(k_{pub} = (p,a,b,q,A,B)\\\\)\n\\\\(k_{pr} = (d)\\\\)\n***\n\n### Signature and Verification\n***\n**ECDSA Signature Generation**\n1. choose an integer as random ephemeral key \\\\(k_{E}\\\\) with \\\\( 0 < k_{E} < q\\\\).\n2. compute \\\\(R = k_{E}A\\\\)\n3. Let \\\\(r = x_{R}\\\\)\n4. compute \\\\(s \\equiv (h(x) + d \\cdot r)k_{E}^{-1} \\bmod q\\\\)\n***\nthe signature verification process is as follows\n***\n**ECDSA Signature Verification**\n1. Compute auxiliary value \\\\(w \\equiv s^{-1} \\bmod q\\\\)\n2. compute auxilary value \\\\(u_1 \\equiv w \\cdot h(x) \\bmod q\\\\)\n3. compute auxiliary value \\\\(u_2 = w \\cdot r \\bmod q\\\\)\n4. compute \\\\(P = u_1 A + u_2 B\\\\).\n5. the verification \\\\(ver{k_{pub}}(x, (r,s))\\\\) follows from\n\\begin{equation}\nx_{P} = \n\\begin{cases}\n\\equiv r \\bmod q & => \\text{valid signature} \\cr\n\\not\\equiv r \\bmod q & => \\text{invalid signature}\n\\end{cases}\n\\end{equation}\n***\nThe point multiplication, which is in most cases by the far the most arithmetic intensive operation, can be precomputed by choosing the ephemeral key ahead of time.","source":"_posts/cryptography/cryptography-04-digital-signature.md","raw":"---\ntitle: cryptography (4) digital signature\ndate: 2023-06-20 14:29:26\ntags: [cryptography]\n---\n\n\n\n## Elgamal Digital Signature Scheme\nThe Elgammal signature scheme is based on the difficulty of computing discrete logarithms. Unlike RSA, where encryption and digital signature are almoste identical operations, the Elgamal digital signature is quite different from the encryption scheme with teh same name.\n\n### key generation\n***\n1. Choose a large prime \\\\(p\\\\).\n2. Choose a primitive element \\\\(\\alpha\\\\) of \\\\(Z_{p}^{\\ast}\\\\), or a subgroup of \\\\(Z_{p}^{\\ast}\\\\).\n3. Choose a random integer \\\\(d \\in {2,3,...,p-2}\\\\)\n4. Compute \\\\(\\beta = \\alpha^{d} \\bmod p\\\\)\n***\nThe public key is now formed by \\\\(k_{pub} = (p, \\alpha, \\beta)\\\\), and the private key by \\\\(k_{pr}=d\\\\)\n\\\\(Z_{p}^{\\ast}\\\\) is the set of integers who are smaller than \\\\(p\\\\) and coprime to \\\\(p\\\\)\n\n### signature and verification\nUsign the private ey and parameters of the public key, the signature\n\\\\[sig_{k_{pr}}(x, k_{E}) = (r,s)\\\\]\n\\\\(x\\\\) is the message. \\\\(k_{E}\\\\) is a random value, which forms an ephemeral private key\n***\n**Elgamal Signature Generation**\n1. choose a random ephemeral key \\\\(k_{E} \\in {0,1,2,..,p-2}\\\\) such that \\\\(gcd(k_{E}, p-1) = 1\\\\)\n2. compute the signatue parameters:\n\\\\[r \\equiv \\alpha^{k_{E}} \\bmod p\\\\]\n\\\\[s \\equiv (x - d \\cdot r)k_{E}^{-1} \\bmod p-1\\\\]\n***\non the receiving side, the signature is verified as \\\\(ver_{k_{pub}}(x,(r,s))\\\\) using the public key, the signature and the message.\n***\n**Elgamal Signature Verification**\n1. comput the value \n\\\\[t \\equiv \\beta^{r} \\cdot r^s \\bmod p\\\\]\n2. the verification follows form\n\\begin{equation}\nt = \n\\begin{cases}\n\\equiv \\alpha^{x} \\bmod p & => \\text{valid signature} \\cr\n\\not\\equiv \\alpha^{x} \\bmod p & => \\text{invalid signature}\n\\end{cases}\n\\end{equation}\n***\n\n## Digital Signature Algorithm (DSA)\nThe native Elgamal signature algorithm described above is rarely used in practice. Instead, a much more popular variant is used, known as the Digital Signature Algorithm (DSA). It is a federal US government standard for digital signatures and was proposed by NIST (National Institute of Standards and Technology). Its main advantages over the Elgamal signature scheme are that the signature is only 320-bit long and that some of the attacks\n that can threaten the Elgamal scheme are not applicable.\n### key generation\n***\n**key Generation for DSA**\n1. Generate a prime \\\\(p\\\\) with \\\\(2^1023 < p < 2^1024\\\\)\n2. find a prime divisor \\\\(q\\\\) of \\\\(p-1\\\\) \\\\(2^159 < q < 2^160\\\\)\n3. Find an element \\\\(\\alpha\\\\) with \\\\( ord(\\alpha) = q\\\\), i.e., \\alpha genertes the subgroup with \\\\(q\\\\) elements.\n4. choose a random integer \\\\(d\\\\) with \\\\(0 < d < q\\\\).\n5. compute \\\\(\\beta \\equiv \\alpha^{d} \\bmod p\\\\).\nthe keys are now:\n\\\\(k_{pub} = (p, q, \\alpha, \\beta)\\\\)\n\\\\(k_{pr}= (d)\\\\)\n***\nThe central idea of DSA is that there are two cyclic groups involved. One is the large cyclic group \\\\(Z_{p}*{\\ast}\\\\), the order of which has bit length of 1024 bit. The second one is in the 160-bit subgroup of \\\\(Z_{p}^{\\ast}\\\\). this set-up yields shorter signature.\n\n### Signature and Verification\nAs in the Elgamal signatue scheme, the DSA signature consists of a pair of integers \\\\((r,s)\\\\). Since each of the two parameters is only 160-bit long, the total signature length is 320 bit. \n***\n**DSA signature generation**\n1. choose an integer as random ephemeral key \\\\(k_{E}\\\\) with \\\\(0 < k_{E} < q\\\\).\n2. compute \\\\(r \\equiv (\\alpha^{k_{E}} \\bmod p) \\bmod q\\\\)\n3. compute \\\\(s \\equiv (SHA(x) + d \\cdot r)k_{E}^{-1} \\bmod q \\\\)\n***\n\nThe signature verification process is as follows:\n***\n**DSA signature verification**\n1. compute auxilary value \\\\(w \\equiv s^{-1} \\bmod q\\\\).\n2. compute auxilary value \\\\(u_{1} \\equiv w \\cdot SHA(x) \\bmod q\\\\).\n3. compute auxilary value \\\\(u_{2} \\equiv w \\cdot r \\bmod q\\\\).\n4. compute \\\\(v \\equiv (\\alpha^{u_1} \\cdot \\beta^{u_2} \\bmod p) \\bmod q\\\\).\n5. the verification \\\\(ver_{k_{pub}}(x, (r,s))\\\\) folows from\n\\begin{equation}\nv = \n\\begin{cases}\n\\equiv r \\bmod q & => \\text{valid signature} \\cr\n\\not\\equiv r \\bmod q & => \\text{invalid signature}\n\\end{cases}\n\\end{equation}\n***\n\n## Elliptic Curve Digital Signature Algorithm (ECDSA)\nElliptic curves have several advantages over RSA and over DL schemes like Elgamal or DSA. In particular, in abscence of strong attacks against elliptic curve cryptosystems (ECC), bit lengths in the range of 160-256 bit can be chosen which provide security equivalent to 1024-3072 bit RSA and DL scheme. The shorter bit length of ECC often results in shorter processing time and in shorter signatures. \nThe ECDSA standard is defined for elliptic curves over prime fields \\\\(Z_{p}\\\\) adn Galois fields \\\\(GF(2^m)\\\\). the former is often preferred in practice, and we only introduce this one in what follows\n### key generation\n***\n**Key Generation for ECDSA**\n1. Use and elliptic curve E with \n- modulus p\n- coefficients a and b\n- a point A which generates a cyclic group of prime order q.\n2. choose a random integer d with \\\\(0 < d < q\\\\)\n3. compute \\\\(B = dA\\\\).\nThe keys are now\n\\\\(k_{pub} = (p,a,b,q,A,B)\\\\)\n\\\\(k_{pr} = (d)\\\\)\n***\n\n### Signature and Verification\n***\n**ECDSA Signature Generation**\n1. choose an integer as random ephemeral key \\\\(k_{E}\\\\) with \\\\( 0 < k_{E} < q\\\\).\n2. compute \\\\(R = k_{E}A\\\\)\n3. Let \\\\(r = x_{R}\\\\)\n4. compute \\\\(s \\equiv (h(x) + d \\cdot r)k_{E}^{-1} \\bmod q\\\\)\n***\nthe signature verification process is as follows\n***\n**ECDSA Signature Verification**\n1. Compute auxiliary value \\\\(w \\equiv s^{-1} \\bmod q\\\\)\n2. compute auxilary value \\\\(u_1 \\equiv w \\cdot h(x) \\bmod q\\\\)\n3. compute auxiliary value \\\\(u_2 = w \\cdot r \\bmod q\\\\)\n4. compute \\\\(P = u_1 A + u_2 B\\\\).\n5. the verification \\\\(ver{k_{pub}}(x, (r,s))\\\\) follows from\n\\begin{equation}\nx_{P} = \n\\begin{cases}\n\\equiv r \\bmod q & => \\text{valid signature} \\cr\n\\not\\equiv r \\bmod q & => \\text{invalid signature}\n\\end{cases}\n\\end{equation}\n***\nThe point multiplication, which is in most cases by the far the most arithmetic intensive operation, can be precomputed by choosing the ephemeral key ahead of time.","slug":"cryptography/cryptography-04-digital-signature","published":1,"updated":"2023-11-05T04:21:17.033Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8do000dqwsj2vh07e7k","content":"\n\n\n

Elgamal Digital Signature Scheme

The Elgammal signature scheme is based on the difficulty of computing discrete logarithms. Unlike RSA, where encryption and digital signature are almoste identical operations, the Elgamal digital signature is quite different from the encryption scheme with teh same name.

\n

key generation

\n
    \n
  1. Choose a large prime \\(p\\).
    2. \n
  2. Choose a primitive element \\(\\alpha\\) of \\(Z_{p}^{\\ast}\\), or a subgroup of \\(Z_{p}^{\\ast}\\).
    3. \n
  3. Choose a random integer \\(d \\in {2,3,…,p-2}\\)
    4. \n
  4. Compute \\(\\beta = \\alpha^{d} \\bmod p\\)
    5. \n
\n
\n

The public key is now formed by \\(k_{pub} = (p, \\alpha, \\beta)\\), and the private key by \\(k_{pr}=d\\)
\\(Z_{p}^{\\ast}\\) is the set of integers who are smaller than \\(p\\) and coprime to \\(p\\)

\n

signature and verification

Usign the private ey and parameters of the public key, the signature
\\[sig_{k_{pr}}(x, k_{E}) = (r,s)\\]
\\(x\\) is the message. \\(k_{E}\\) is a random value, which forms an ephemeral private key

\n
\n

Elgamal Signature Generation

\n
    \n
  1. choose a random ephemeral key \\(k_{E} \\in {0,1,2,..,p-2}\\) such that \\(gcd(k_{E}, p-1) = 1\\)
    2. \n
  2. compute the signatue parameters:
    \\[r \\equiv \\alpha^{k_{E}} \\bmod p\\]
    \\[s \\equiv (x - d \\cdot r)k_{E}^{-1} \\bmod p-1\\]
    3. \n
\n
\n

on the receiving side, the signature is verified as \\(ver_{k_{pub}}(x,(r,s))\\) using the public key, the signature and the message.

\n
\n

Elgamal Signature Verification

\n
    \n
  1. comput the value
    \\[t \\equiv \\beta^{r} \\cdot r^s \\bmod p\\]
    2. \n
  2. the verification follows form
    \\begin{equation}
    t =
    \\begin{cases}
    \\equiv \\alpha^{x} \\bmod p & => \\text{valid signature} \\cr
    \\not\\equiv \\alpha^{x} \\bmod p & => \\text{invalid signature}
    \\end{cases}
    \\end{equation}
    3. \n
\n
\n

Digital Signature Algorithm (DSA)

The native Elgamal signature algorithm described above is rarely used in practice. Instead, a much more popular variant is used, known as the Digital Signature Algorithm (DSA). It is a federal US government standard for digital signatures and was proposed by NIST (National Institute of Standards and Technology). Its main advantages over the Elgamal signature scheme are that the signature is only 320-bit long and that some of the attacks
that can threaten the Elgamal scheme are not applicable.

\n

key generation

\n

key Generation for DSA

\n
    \n
  1. Generate a prime \\(p\\) with \\(2^1023 < p < 2^1024\\)
    2. \n
  2. find a prime divisor \\(q\\) of \\(p-1\\) \\(2^159 < q < 2^160\\)
    3. \n
  3. Find an element \\(\\alpha\\) with \\( ord(\\alpha) = q\\), i.e., \\alpha genertes the subgroup with \\(q\\) elements.
    4. \n
  4. choose a random integer \\(d\\) with \\(0 < d < q\\).
    5. \n
  5. compute \\(\\beta \\equiv \\alpha^{d} \\bmod p\\).
    the keys are now:
    \\(k_{pub} = (p, q, \\alpha, \\beta)\\)
    \\(k_{pr}= (d)\\)
    6. \n
\n
\n

The central idea of DSA is that there are two cyclic groups involved. One is the large cyclic group \\(Z_{p}*{\\ast}\\), the order of which has bit length of 1024 bit. The second one is in the 160-bit subgroup of \\(Z_{p}^{\\ast}\\). this set-up yields shorter signature.

\n

Signature and Verification

As in the Elgamal signatue scheme, the DSA signature consists of a pair of integers \\((r,s)\\). Since each of the two parameters is only 160-bit long, the total signature length is 320 bit.

\n
\n

DSA signature generation

\n
    \n
  1. choose an integer as random ephemeral key \\(k_{E}\\) with \\(0 < k_{E} < q\\).
    2. \n
  2. compute \\(r \\equiv (\\alpha^{k_{E}} \\bmod p) \\bmod q\\)
    3. \n
  3. compute \\(s \\equiv (SHA(x) + d \\cdot r)k_{E}^{-1} \\bmod q \\)
    4. \n
\n
\n

The signature verification process is as follows:

\n
\n

DSA signature verification

\n
    \n
  1. compute auxilary value \\(w \\equiv s^{-1} \\bmod q\\).
    2. \n
  2. compute auxilary value \\(u_{1} \\equiv w \\cdot SHA(x) \\bmod q\\).
    3. \n
  3. compute auxilary value \\(u_{2} \\equiv w \\cdot r \\bmod q\\).
    4. \n
  4. compute \\(v \\equiv (\\alpha^{u_1} \\cdot \\beta^{u_2} \\bmod p) \\bmod q\\).
    5. \n
  5. the verification \\(ver_{k_{pub}}(x, (r,s))\\) folows from
    \\begin{equation}
    v =
    \\begin{cases}
    \\equiv r \\bmod q & => \\text{valid signature} \\cr
    \\not\\equiv r \\bmod q & => \\text{invalid signature}
    \\end{cases}
    \\end{equation}
    6. \n
\n
\n

Elliptic Curve Digital Signature Algorithm (ECDSA)

Elliptic curves have several advantages over RSA and over DL schemes like Elgamal or DSA. In particular, in abscence of strong attacks against elliptic curve cryptosystems (ECC), bit lengths in the range of 160-256 bit can be chosen which provide security equivalent to 1024-3072 bit RSA and DL scheme. The shorter bit length of ECC often results in shorter processing time and in shorter signatures.
The ECDSA standard is defined for elliptic curves over prime fields \\(Z_{p}\\) adn Galois fields \\(GF(2^m)\\). the former is often preferred in practice, and we only introduce this one in what follows

\n

key generation

\n

Key Generation for ECDSA

\n
    \n
  1. Use and elliptic curve E with
    2. \n
\n
    \n
  • modulus p
    • \n
  • coefficients a and b
    • \n
  • a point A which generates a cyclic group of prime order q.
    • \n
\n
    \n
  1. choose a random integer d with \\(0 < d < q\\)
    2. \n
  2. compute \\(B = dA\\).
    The keys are now
    \\(k_{pub} = (p,a,b,q,A,B)\\)
    \\(k_{pr} = (d)\\)
    3. \n
\n
\n

Signature and Verification

\n

ECDSA Signature Generation

\n
    \n
  1. choose an integer as random ephemeral key \\(k_{E}\\) with \\( 0 < k_{E} < q\\).
    2. \n
  2. compute \\(R = k_{E}A\\)
    3. \n
  3. Let \\(r = x_{R}\\)
    4. \n
  4. compute \\(s \\equiv (h(x) + d \\cdot r)k_{E}^{-1} \\bmod q\\)
    5. \n
\n
\n

the signature verification process is as follows

\n
\n

ECDSA Signature Verification

\n
    \n
  1. Compute auxiliary value \\(w \\equiv s^{-1} \\bmod q\\)
    2. \n
  2. compute auxilary value \\(u_1 \\equiv w \\cdot h(x) \\bmod q\\)
    3. \n
  3. compute auxiliary value \\(u_2 = w \\cdot r \\bmod q\\)
    4. \n
  4. compute \\(P = u_1 A + u_2 B\\).
    5. \n
  5. the verification \\(ver{k_{pub}}(x, (r,s))\\) follows from
    \\begin{equation}
    x_{P} =
    \\begin{cases}
    \\equiv r \\bmod q & => \\text{valid signature} \\cr
    \\not\\equiv r \\bmod q & => \\text{invalid signature}
    \\end{cases}
    \\end{equation}
    6. \n
\n
\n

The point multiplication, which is in most cases by the far the most arithmetic intensive operation, can be precomputed by choosing the ephemeral key ahead of time.

\n","site":{"data":{}},"excerpt":"","more":"\n\n\n

Elgamal Digital Signature Scheme

The Elgammal signature scheme is based on the difficulty of computing discrete logarithms. Unlike RSA, where encryption and digital signature are almoste identical operations, the Elgamal digital signature is quite different from the encryption scheme with teh same name.

\n

key generation

\n
    \n
  1. Choose a large prime \\(p\\).
    2. \n
  2. Choose a primitive element \\(\\alpha\\) of \\(Z_{p}^{\\ast}\\), or a subgroup of \\(Z_{p}^{\\ast}\\).
    3. \n
  3. Choose a random integer \\(d \\in {2,3,…,p-2}\\)
    4. \n
  4. Compute \\(\\beta = \\alpha^{d} \\bmod p\\)
    5. \n
\n
\n

The public key is now formed by \\(k_{pub} = (p, \\alpha, \\beta)\\), and the private key by \\(k_{pr}=d\\)
\\(Z_{p}^{\\ast}\\) is the set of integers who are smaller than \\(p\\) and coprime to \\(p\\)

\n

signature and verification

Usign the private ey and parameters of the public key, the signature
\\[sig_{k_{pr}}(x, k_{E}) = (r,s)\\]
\\(x\\) is the message. \\(k_{E}\\) is a random value, which forms an ephemeral private key

\n
\n

Elgamal Signature Generation

\n
    \n
  1. choose a random ephemeral key \\(k_{E} \\in {0,1,2,..,p-2}\\) such that \\(gcd(k_{E}, p-1) = 1\\)
    2. \n
  2. compute the signatue parameters:
    \\[r \\equiv \\alpha^{k_{E}} \\bmod p\\]
    \\[s \\equiv (x - d \\cdot r)k_{E}^{-1} \\bmod p-1\\]
    3. \n
\n
\n

on the receiving side, the signature is verified as \\(ver_{k_{pub}}(x,(r,s))\\) using the public key, the signature and the message.

\n
\n

Elgamal Signature Verification

\n
    \n
  1. comput the value
    \\[t \\equiv \\beta^{r} \\cdot r^s \\bmod p\\]
    2. \n
  2. the verification follows form
    \\begin{equation}
    t =
    \\begin{cases}
    \\equiv \\alpha^{x} \\bmod p & => \\text{valid signature} \\cr
    \\not\\equiv \\alpha^{x} \\bmod p & => \\text{invalid signature}
    \\end{cases}
    \\end{equation}
    3. \n
\n
\n

Digital Signature Algorithm (DSA)

The native Elgamal signature algorithm described above is rarely used in practice. Instead, a much more popular variant is used, known as the Digital Signature Algorithm (DSA). It is a federal US government standard for digital signatures and was proposed by NIST (National Institute of Standards and Technology). Its main advantages over the Elgamal signature scheme are that the signature is only 320-bit long and that some of the attacks
that can threaten the Elgamal scheme are not applicable.

\n

key generation

\n

key Generation for DSA

\n
    \n
  1. Generate a prime \\(p\\) with \\(2^1023 < p < 2^1024\\)
    2. \n
  2. find a prime divisor \\(q\\) of \\(p-1\\) \\(2^159 < q < 2^160\\)
    3. \n
  3. Find an element \\(\\alpha\\) with \\( ord(\\alpha) = q\\), i.e., \\alpha genertes the subgroup with \\(q\\) elements.
    4. \n
  4. choose a random integer \\(d\\) with \\(0 < d < q\\).
    5. \n
  5. compute \\(\\beta \\equiv \\alpha^{d} \\bmod p\\).
    the keys are now:
    \\(k_{pub} = (p, q, \\alpha, \\beta)\\)
    \\(k_{pr}= (d)\\)
    6. \n
\n
\n

The central idea of DSA is that there are two cyclic groups involved. One is the large cyclic group \\(Z_{p}*{\\ast}\\), the order of which has bit length of 1024 bit. The second one is in the 160-bit subgroup of \\(Z_{p}^{\\ast}\\). this set-up yields shorter signature.

\n

Signature and Verification

As in the Elgamal signatue scheme, the DSA signature consists of a pair of integers \\((r,s)\\). Since each of the two parameters is only 160-bit long, the total signature length is 320 bit.

\n
\n

DSA signature generation

\n
    \n
  1. choose an integer as random ephemeral key \\(k_{E}\\) with \\(0 < k_{E} < q\\).
    2. \n
  2. compute \\(r \\equiv (\\alpha^{k_{E}} \\bmod p) \\bmod q\\)
    3. \n
  3. compute \\(s \\equiv (SHA(x) + d \\cdot r)k_{E}^{-1} \\bmod q \\)
    4. \n
\n
\n

The signature verification process is as follows:

\n
\n

DSA signature verification

\n
    \n
  1. compute auxilary value \\(w \\equiv s^{-1} \\bmod q\\).
    2. \n
  2. compute auxilary value \\(u_{1} \\equiv w \\cdot SHA(x) \\bmod q\\).
    3. \n
  3. compute auxilary value \\(u_{2} \\equiv w \\cdot r \\bmod q\\).
    4. \n
  4. compute \\(v \\equiv (\\alpha^{u_1} \\cdot \\beta^{u_2} \\bmod p) \\bmod q\\).
    5. \n
  5. the verification \\(ver_{k_{pub}}(x, (r,s))\\) folows from
    \\begin{equation}
    v =
    \\begin{cases}
    \\equiv r \\bmod q & => \\text{valid signature} \\cr
    \\not\\equiv r \\bmod q & => \\text{invalid signature}
    \\end{cases}
    \\end{equation}
    6. \n
\n
\n

Elliptic Curve Digital Signature Algorithm (ECDSA)

Elliptic curves have several advantages over RSA and over DL schemes like Elgamal or DSA. In particular, in abscence of strong attacks against elliptic curve cryptosystems (ECC), bit lengths in the range of 160-256 bit can be chosen which provide security equivalent to 1024-3072 bit RSA and DL scheme. The shorter bit length of ECC often results in shorter processing time and in shorter signatures.
The ECDSA standard is defined for elliptic curves over prime fields \\(Z_{p}\\) adn Galois fields \\(GF(2^m)\\). the former is often preferred in practice, and we only introduce this one in what follows

\n

key generation

\n

Key Generation for ECDSA

\n
    \n
  1. Use and elliptic curve E with
    2. \n
\n
    \n
  • modulus p
    • \n
  • coefficients a and b
    • \n
  • a point A which generates a cyclic group of prime order q.
    • \n
\n
    \n
  1. choose a random integer d with \\(0 < d < q\\)
    2. \n
  2. compute \\(B = dA\\).
    The keys are now
    \\(k_{pub} = (p,a,b,q,A,B)\\)
    \\(k_{pr} = (d)\\)
    3. \n
\n
\n

Signature and Verification

\n

ECDSA Signature Generation

\n
    \n
  1. choose an integer as random ephemeral key \\(k_{E}\\) with \\( 0 < k_{E} < q\\).
    2. \n
  2. compute \\(R = k_{E}A\\)
    3. \n
  3. Let \\(r = x_{R}\\)
    4. \n
  4. compute \\(s \\equiv (h(x) + d \\cdot r)k_{E}^{-1} \\bmod q\\)
    5. \n
\n
\n

the signature verification process is as follows

\n
\n

ECDSA Signature Verification

\n
    \n
  1. Compute auxiliary value \\(w \\equiv s^{-1} \\bmod q\\)
    2. \n
  2. compute auxilary value \\(u_1 \\equiv w \\cdot h(x) \\bmod q\\)
    3. \n
  3. compute auxiliary value \\(u_2 = w \\cdot r \\bmod q\\)
    4. \n
  4. compute \\(P = u_1 A + u_2 B\\).
    5. \n
  5. the verification \\(ver{k_{pub}}(x, (r,s))\\) follows from
    \\begin{equation}
    x_{P} =
    \\begin{cases}
    \\equiv r \\bmod q & => \\text{valid signature} \\cr
    \\not\\equiv r \\bmod q & => \\text{invalid signature}
    \\end{cases}
    \\end{equation}
    6. \n
\n
\n

The point multiplication, which is in most cases by the far the most arithmetic intensive operation, can be precomputed by choosing the ephemeral key ahead of time.

\n"},{"title":"cryptography (2) RSA cryptosystem","date":"2023-06-10T06:29:26.000Z","_content":"\n\n\n\n## introduction\nthe security of RSA relies on the difficulty of factoring a product of two large primes(the integer factorization problem). it is firstly presented in 1978 in [1].\n\n## Euclidean Algorithm\nthe gcd of two positive integers \\\\(r_0\\\\) and \\\\(r_1\\\\) is denoted by \n\\\\[gcd(r_0, r_1)\\\\]\nthe Eucliedean algorithm is used to calculate gcd, based on as simple observation that\n\\\\[gcd(r_0, r_1) = gcd(r_0-r_1, r_1)\\\\]\nwhere we assume \\\\(r_0 > r_1\\\\)\nThis property can easily be proven: Let \\\\(gcd(r_0, r_1) = g\\\\), since \\\\(g\\\\) divides both \\\\(r_0\\\\) and \\\\(r_1\\\\), we can write \\\\(r_0 = g \\cdot x\\\\) and \\\\(r_1 = g \\cdot y\\\\), where \\\\(x>y\\\\) and \\\\(x\\\\) and \\\\(y\\\\) are coprime integers, i.e., they do not have common factors. Moreover, it is easy to show that \\\\((x-y)\\\\) and \\\\(y\\\\) are also coprime. it follows from here that\n\\\\[gcd(r_0 - r_1, r_1) = gcd(g \\cdot (x-y), g\\cdot y) = g\\\\]\n\nit also follow immediately that we can apply the process iteratively:\n\\\\[gcd(r_0 - r_1, r_1) = gcd(r_0 - mr_1, r_1) \\\\]\nas long as \\\\((r_0 - mr_1) >0\\\\). the algorithm use the fewest number of steps if we choose maximum value of \\\\(m\\\\). this is the case if we compute:\n\\\\[gcd(r_0, r_1) = gcd( r_1, r_0 \\bmod r_1) \\\\]\nthis process can be applied recursively until we obtain finally \\\\[gcd(r_l, 0) = r_l \\\\]\nthe euclidean algorithm is very efficient, even with the very long numbers. the number of iterations is close to the number of digits of the input operands. that means, for instance, that the number of iterations of a gcd involving 1024-bit nubmers is 1024 times a constant.\n## Extended Euclidean Algorithm\nan extension of the Euclidean algorithm allows us to compute ***modular inverse***. in addition to computing the gcd, the ***extended Euclidean algorithm*** computes a linear combination of the form\n\\\\[gcd(r_0, r_1) = s \\cdot r_0 + t\\cdot r_1 \\\\]\nwhere s and t are integer coefficients. this equation is ofthen referred to as ***Diophantine equation***\n\nthe detail of the algorithm can be foud in section 6.3.2 of the book understanding cryptography by Christof Paar. Here presents the general idea by using an example.\nlet \\\\(r_0 =973 , r_1 = 301\\\\). during the steps of Euclidean Algorithm, we obtain \\\\(973 = 3\\cdot301 + 70\\\\)\nwhich is \\\\[r_0 = q_1 \\cdot r_1 + r_2\\\\] \nrearrange:\n\\\\[r_2 = r_0 + (-q_1) \\cdot r_1\\\\] \nreplacing (r_0, r_1) and iteratively by (r_1, r_2), ... (r_{i-1}, r_{i}), util \\\\(r_{i+1} = 0\\\\)\nthen \\\\(r_{i}\\\\) is \\\\(gcd(r_0,r_1)\\\\), and can have a representation of \n\\\\[gcd(r_0, r_1) = s\\cdot r_0 + t\\cdot r_1 \\\\]. \nsince the inverse only exists if \\\\(gcd(r_0, r_1)=1\\\\). we obtain\n\\\\[ s\\cdot r_0 + t\\cdot r_1 = 1\\\\]\ntaking this equation modulo \\\\(r_0\\\\) we obtain\n\\\\[ s\\cdot 0 + t\\cdot r_1 \\equiv 1 \\bmod r_0\\\\]\n\\\\[ t\\cdot r_1 \\equiv 1 \\bmod r_0\\\\]\nt is the definition of the inverse of \\\\(r_1\\\\)\n\nThus, if we need to compute an inverse \\\\(a^{-1} \\bmod m\\\\), we apply EEA with the input parameters \\\\(m\\\\) and \\\\(a\\\\)\n## Euler's Phi Function\nwe consider the ring \\\\( Z_m\\\\) i.e., the set of integers \\\\({0,1,...,m-1}\\\\). we are interested in teh problem of knowing how many numbers in this set are relatively prime to m. this quantity is given by Euler's phi function, which is \\\\(\\Phi(m)\\\\)\n\n***\nlet m have the following canonical factorization\n\\\\[ m = p_{1}^{e_1} \\cdot p_{2}^{e_2} \\cdot ... \\cdot p_{n}^{e_n}\\\\]\nwhere the \\\\(p_i\\\\) are distinct prime numbers and \\\\( e_i\\\\) are positive integers, then\n\n\\\\[ \\Phi(m) = \\prod_{i=1}^{n}(p_{i}^{e_i} - p_{i}^{e_i -1} ) \\\\]\n***\nit is important to stress that we need to know the factoorization of m in order to calculate Euler's phi function.\n\n## Fermat's little theorem\nFermat's little theorem states that if p is a prime number, then for any integer a, the number \n\\\\(a^{p}-a \\\\) is an integer multiple of p. In the notation of modular arithmetic, this is expressed as\n\\\\[ a^{p} \\equiv a \\bmod p\\\\]\nthe theorem can be stated in the form also,\n\\\\[ a^{p-1} \\equiv 1 \\bmod p\\\\]\nthen the inverse of an integer is,\n\\\\[ a^{-1} \\equiv a^{p-2} \\bmod p\\\\]\nperforming the above formulate (involving exponentiation) to find inverse is usually slower than using extended Euclidean algorithm. However, there are situations where it is advantageous to use Fermat's Little Theorem, e.g., on smart cards or other devices which have a hardware accelerator for fast exponentiation anyway.\n\na generatlization of Fermat's little Theorem to any integer moduli, i.e., moduli that are not necessarily primes, is Euler's theorem.\n***\n**Euler's Theorem**\nlet \\\\(a\\\\) and \\\\(m\\\\) be integers with \\\\(gcd(a,m) = 1\\\\), then\n\\\\[ a^{\\Phi(m)} \\equiv 1 \\bmod m\\\\]\n***\nsince it works modulo m, it is applicable to integer rings \\\\(Z_{m}\\\\)\n\n## key generation\n***\n**Output**: public key: \\\\( k_{pub} = (n,e) and private key: k_{pr} = (d) \\\\)\n1. choose two large primes p and q.\n2. compute \\\\(n = p\\cdot q\\\\)\n3. compute \\\\( \\Phi(n) = (p-1)(q-1)\\\\)\n4. select the public exponent \\\\( e \\in {1,2,...,\\Phi(n)-1} \\\\) such that \n\\\\[ gcd(e,\\Phi(n)) = 1\\\\]\n5. compute the private key d such that\n\\\\[ d \\cdot e \\equiv 1 \\bmod \\Phi(n)\\\\]\n***\nthe condition that \\\\( gcd(e,\\Phi(n)) = 1\\\\) ensures that the inverse of \\\\(e\\\\) exists modulo \\\\(\\Phi(n)\\\\), so that there is always a private key \\\\(d\\\\).\nthe computation of key keys \\\\(d\\\\) and \\\\(e\\\\) canb e doen at once using the extended Euclidean algorith. \n\n\n## Encryption and Decryption\n\n***\n**RSA Encryption** Given the privaate key \\\\( k_{pub} = (n,e) \\\\) and the plaintext \\\\(x\\\\), the encryption is:\n\\\\[ y = e_{k_{pub}}(x) \\equiv x^{e} \\bmod n\\\\]\nwhere \\\\(x,y \\in Z_{n}\\\\)\n***\n***\n**RSA Decryption** Given the public key \\\\d = k_{pr} \\\\) and the plaintext \\\\(y\\\\), the decryption is:\n\\\\[ x = d_{k_{pr}}(y) \\equiv y^{d} \\bmod n\\\\]\nwhere \\\\(x,y \\in Z_{n}\\\\)\n***\n\n## Digital signature\nthe message \\\\(x\\\\) that is being signed is in the range \\\\(1,2,...,n-1\\\\)\n![rsa digital signature](/images/cryptography/rsa/rsa_signature.png)\n## references\n- [1] [A Method for Obtaining Digital\nSignatures and Public-Key Cryptosystems](https://web.williams.edu/Mathematics/lg5/302/RSA.pdf) ","source":"_posts/cryptography/cryptography-02-rsa.md","raw":"---\ntitle: cryptography (2) RSA cryptosystem\ndate: 2023-06-10 14:29:26\ntags: [cryptography]\n---\n\n\n\n\n## introduction\nthe security of RSA relies on the difficulty of factoring a product of two large primes(the integer factorization problem). it is firstly presented in 1978 in [1].\n\n## Euclidean Algorithm\nthe gcd of two positive integers \\\\(r_0\\\\) and \\\\(r_1\\\\) is denoted by \n\\\\[gcd(r_0, r_1)\\\\]\nthe Eucliedean algorithm is used to calculate gcd, based on as simple observation that\n\\\\[gcd(r_0, r_1) = gcd(r_0-r_1, r_1)\\\\]\nwhere we assume \\\\(r_0 > r_1\\\\)\nThis property can easily be proven: Let \\\\(gcd(r_0, r_1) = g\\\\), since \\\\(g\\\\) divides both \\\\(r_0\\\\) and \\\\(r_1\\\\), we can write \\\\(r_0 = g \\cdot x\\\\) and \\\\(r_1 = g \\cdot y\\\\), where \\\\(x>y\\\\) and \\\\(x\\\\) and \\\\(y\\\\) are coprime integers, i.e., they do not have common factors. Moreover, it is easy to show that \\\\((x-y)\\\\) and \\\\(y\\\\) are also coprime. it follows from here that\n\\\\[gcd(r_0 - r_1, r_1) = gcd(g \\cdot (x-y), g\\cdot y) = g\\\\]\n\nit also follow immediately that we can apply the process iteratively:\n\\\\[gcd(r_0 - r_1, r_1) = gcd(r_0 - mr_1, r_1) \\\\]\nas long as \\\\((r_0 - mr_1) >0\\\\). the algorithm use the fewest number of steps if we choose maximum value of \\\\(m\\\\). this is the case if we compute:\n\\\\[gcd(r_0, r_1) = gcd( r_1, r_0 \\bmod r_1) \\\\]\nthis process can be applied recursively until we obtain finally \\\\[gcd(r_l, 0) = r_l \\\\]\nthe euclidean algorithm is very efficient, even with the very long numbers. the number of iterations is close to the number of digits of the input operands. that means, for instance, that the number of iterations of a gcd involving 1024-bit nubmers is 1024 times a constant.\n## Extended Euclidean Algorithm\nan extension of the Euclidean algorithm allows us to compute ***modular inverse***. in addition to computing the gcd, the ***extended Euclidean algorithm*** computes a linear combination of the form\n\\\\[gcd(r_0, r_1) = s \\cdot r_0 + t\\cdot r_1 \\\\]\nwhere s and t are integer coefficients. this equation is ofthen referred to as ***Diophantine equation***\n\nthe detail of the algorithm can be foud in section 6.3.2 of the book understanding cryptography by Christof Paar. Here presents the general idea by using an example.\nlet \\\\(r_0 =973 , r_1 = 301\\\\). during the steps of Euclidean Algorithm, we obtain \\\\(973 = 3\\cdot301 + 70\\\\)\nwhich is \\\\[r_0 = q_1 \\cdot r_1 + r_2\\\\] \nrearrange:\n\\\\[r_2 = r_0 + (-q_1) \\cdot r_1\\\\] \nreplacing (r_0, r_1) and iteratively by (r_1, r_2), ... (r_{i-1}, r_{i}), util \\\\(r_{i+1} = 0\\\\)\nthen \\\\(r_{i}\\\\) is \\\\(gcd(r_0,r_1)\\\\), and can have a representation of \n\\\\[gcd(r_0, r_1) = s\\cdot r_0 + t\\cdot r_1 \\\\]. \nsince the inverse only exists if \\\\(gcd(r_0, r_1)=1\\\\). we obtain\n\\\\[ s\\cdot r_0 + t\\cdot r_1 = 1\\\\]\ntaking this equation modulo \\\\(r_0\\\\) we obtain\n\\\\[ s\\cdot 0 + t\\cdot r_1 \\equiv 1 \\bmod r_0\\\\]\n\\\\[ t\\cdot r_1 \\equiv 1 \\bmod r_0\\\\]\nt is the definition of the inverse of \\\\(r_1\\\\)\n\nThus, if we need to compute an inverse \\\\(a^{-1} \\bmod m\\\\), we apply EEA with the input parameters \\\\(m\\\\) and \\\\(a\\\\)\n## Euler's Phi Function\nwe consider the ring \\\\( Z_m\\\\) i.e., the set of integers \\\\({0,1,...,m-1}\\\\). we are interested in teh problem of knowing how many numbers in this set are relatively prime to m. this quantity is given by Euler's phi function, which is \\\\(\\Phi(m)\\\\)\n\n***\nlet m have the following canonical factorization\n\\\\[ m = p_{1}^{e_1} \\cdot p_{2}^{e_2} \\cdot ... \\cdot p_{n}^{e_n}\\\\]\nwhere the \\\\(p_i\\\\) are distinct prime numbers and \\\\( e_i\\\\) are positive integers, then\n\n\\\\[ \\Phi(m) = \\prod_{i=1}^{n}(p_{i}^{e_i} - p_{i}^{e_i -1} ) \\\\]\n***\nit is important to stress that we need to know the factoorization of m in order to calculate Euler's phi function.\n\n## Fermat's little theorem\nFermat's little theorem states that if p is a prime number, then for any integer a, the number \n\\\\(a^{p}-a \\\\) is an integer multiple of p. In the notation of modular arithmetic, this is expressed as\n\\\\[ a^{p} \\equiv a \\bmod p\\\\]\nthe theorem can be stated in the form also,\n\\\\[ a^{p-1} \\equiv 1 \\bmod p\\\\]\nthen the inverse of an integer is,\n\\\\[ a^{-1} \\equiv a^{p-2} \\bmod p\\\\]\nperforming the above formulate (involving exponentiation) to find inverse is usually slower than using extended Euclidean algorithm. However, there are situations where it is advantageous to use Fermat's Little Theorem, e.g., on smart cards or other devices which have a hardware accelerator for fast exponentiation anyway.\n\na generatlization of Fermat's little Theorem to any integer moduli, i.e., moduli that are not necessarily primes, is Euler's theorem.\n***\n**Euler's Theorem**\nlet \\\\(a\\\\) and \\\\(m\\\\) be integers with \\\\(gcd(a,m) = 1\\\\), then\n\\\\[ a^{\\Phi(m)} \\equiv 1 \\bmod m\\\\]\n***\nsince it works modulo m, it is applicable to integer rings \\\\(Z_{m}\\\\)\n\n## key generation\n***\n**Output**: public key: \\\\( k_{pub} = (n,e) and private key: k_{pr} = (d) \\\\)\n1. choose two large primes p and q.\n2. compute \\\\(n = p\\cdot q\\\\)\n3. compute \\\\( \\Phi(n) = (p-1)(q-1)\\\\)\n4. select the public exponent \\\\( e \\in {1,2,...,\\Phi(n)-1} \\\\) such that \n\\\\[ gcd(e,\\Phi(n)) = 1\\\\]\n5. compute the private key d such that\n\\\\[ d \\cdot e \\equiv 1 \\bmod \\Phi(n)\\\\]\n***\nthe condition that \\\\( gcd(e,\\Phi(n)) = 1\\\\) ensures that the inverse of \\\\(e\\\\) exists modulo \\\\(\\Phi(n)\\\\), so that there is always a private key \\\\(d\\\\).\nthe computation of key keys \\\\(d\\\\) and \\\\(e\\\\) canb e doen at once using the extended Euclidean algorith. \n\n\n## Encryption and Decryption\n\n***\n**RSA Encryption** Given the privaate key \\\\( k_{pub} = (n,e) \\\\) and the plaintext \\\\(x\\\\), the encryption is:\n\\\\[ y = e_{k_{pub}}(x) \\equiv x^{e} \\bmod n\\\\]\nwhere \\\\(x,y \\in Z_{n}\\\\)\n***\n***\n**RSA Decryption** Given the public key \\\\d = k_{pr} \\\\) and the plaintext \\\\(y\\\\), the decryption is:\n\\\\[ x = d_{k_{pr}}(y) \\equiv y^{d} \\bmod n\\\\]\nwhere \\\\(x,y \\in Z_{n}\\\\)\n***\n\n## Digital signature\nthe message \\\\(x\\\\) that is being signed is in the range \\\\(1,2,...,n-1\\\\)\n![rsa digital signature](/images/cryptography/rsa/rsa_signature.png)\n## references\n- [1] [A Method for Obtaining Digital\nSignatures and Public-Key Cryptosystems](https://web.williams.edu/Mathematics/lg5/302/RSA.pdf) ","slug":"cryptography/cryptography-02-rsa","published":1,"updated":"2023-11-05T04:21:17.033Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8do000gqwsj13yvh382","content":"\n\n\n\n

introduction

the security of RSA relies on the difficulty of factoring a product of two large primes(the integer factorization problem). it is firstly presented in 1978 in [1].

\n

Euclidean Algorithm

the gcd of two positive integers \\(r_0\\) and \\(r_1\\) is denoted by
\\[gcd(r_0, r_1)\\]
the Eucliedean algorithm is used to calculate gcd, based on as simple observation that
\\[gcd(r_0, r_1) = gcd(r_0-r_1, r_1)\\]
where we assume \\(r_0 > r_1\\)
This property can easily be proven: Let \\(gcd(r_0, r_1) = g\\), since \\(g\\) divides both \\(r_0\\) and \\(r_1\\), we can write \\(r_0 = g \\cdot x\\) and \\(r_1 = g \\cdot y\\), where \\(x>y\\) and \\(x\\) and \\(y\\) are coprime integers, i.e., they do not have common factors. Moreover, it is easy to show that \\((x-y)\\) and \\(y\\) are also coprime. it follows from here that
\\[gcd(r_0 - r_1, r_1) = gcd(g \\cdot (x-y), g\\cdot y) = g\\]

\n

it also follow immediately that we can apply the process iteratively:
\\[gcd(r_0 - r_1, r_1) = gcd(r_0 - mr_1, r_1) \\]
as long as \\((r_0 - mr_1) >0\\). the algorithm use the fewest number of steps if we choose maximum value of \\(m\\). this is the case if we compute:
\\[gcd(r_0, r_1) = gcd( r_1, r_0 \\bmod r_1) \\]
this process can be applied recursively until we obtain finally \\[gcd(r_l, 0) = r_l \\]
the euclidean algorithm is very efficient, even with the very long numbers. the number of iterations is close to the number of digits of the input operands. that means, for instance, that the number of iterations of a gcd involving 1024-bit nubmers is 1024 times a constant.

\n

Extended Euclidean Algorithm

an extension of the Euclidean algorithm allows us to compute modular inverse. in addition to computing the gcd, the extended Euclidean algorithm computes a linear combination of the form
\\[gcd(r_0, r_1) = s \\cdot r_0 + t\\cdot r_1 \\]
where s and t are integer coefficients. this equation is ofthen referred to as Diophantine equation

\n

the detail of the algorithm can be foud in section 6.3.2 of the book understanding cryptography by Christof Paar. Here presents the general idea by using an example.
let \\(r_0 =973 , r_1 = 301\\). during the steps of Euclidean Algorithm, we obtain \\(973 = 3\\cdot301 + 70\\)
which is \\[r_0 = q_1 \\cdot r_1 + r_2\\]
rearrange:
\\[r_2 = r_0 + (-q_1) \\cdot r_1\\]
replacing (r_0, r_1) and iteratively by (r_1, r_2), … (r_{i-1}, r_{i}), util \\(r_{i+1} = 0\\)
then \\(r_{i}\\) is \\(gcd(r_0,r_1)\\), and can have a representation of
\\[gcd(r_0, r_1) = s\\cdot r_0 + t\\cdot r_1 \\].
since the inverse only exists if \\(gcd(r_0, r_1)=1\\). we obtain
\\[ s\\cdot r_0 + t\\cdot r_1 = 1\\]
taking this equation modulo \\(r_0\\) we obtain
\\[ s\\cdot 0 + t\\cdot r_1 \\equiv 1 \\bmod r_0\\]
\\[ t\\cdot r_1 \\equiv 1 \\bmod r_0\\]
t is the definition of the inverse of \\(r_1\\)

\n

Thus, if we need to compute an inverse \\(a^{-1} \\bmod m\\), we apply EEA with the input parameters \\(m\\) and \\(a\\)

\n

Euler’s Phi Function

we consider the ring \\( Z_m\\) i.e., the set of integers \\({0,1,…,m-1}\\). we are interested in teh problem of knowing how many numbers in this set are relatively prime to m. this quantity is given by Euler’s phi function, which is \\(\\Phi(m)\\)

\n
\n

let m have the following canonical factorization
\\[ m = p_{1}^{e_1} \\cdot p_{2}^{e_2} \\cdot … \\cdot p_{n}^{e_n}\\]
where the \\(p_i\\) are distinct prime numbers and \\( e_i\\) are positive integers, then

\n

\\[ \\Phi(m) = \\prod_{i=1}^{n}(p_{i}^{e_i} - p_{i}^{e_i -1} ) \\]

\n
\n

it is important to stress that we need to know the factoorization of m in order to calculate Euler’s phi function.

\n

Fermat’s little theorem

Fermat’s little theorem states that if p is a prime number, then for any integer a, the number
\\(a^{p}-a \\) is an integer multiple of p. In the notation of modular arithmetic, this is expressed as
\\[ a^{p} \\equiv a \\bmod p\\]
the theorem can be stated in the form also,
\\[ a^{p-1} \\equiv 1 \\bmod p\\]
then the inverse of an integer is,
\\[ a^{-1} \\equiv a^{p-2} \\bmod p\\]
performing the above formulate (involving exponentiation) to find inverse is usually slower than using extended Euclidean algorithm. However, there are situations where it is advantageous to use Fermat’s Little Theorem, e.g., on smart cards or other devices which have a hardware accelerator for fast exponentiation anyway.

\n

a generatlization of Fermat’s little Theorem to any integer moduli, i.e., moduli that are not necessarily primes, is Euler’s theorem.

\n
\n

Euler’s Theorem
let \\(a\\) and \\(m\\) be integers with \\(gcd(a,m) = 1\\), then
\\[ a^{\\Phi(m)} \\equiv 1 \\bmod m\\]

\n
\n

since it works modulo m, it is applicable to integer rings \\(Z_{m}\\)

\n

key generation

\n

Output: public key: \\( k_{pub} = (n,e) and private key: k_{pr} = (d) \\)

\n
    \n
  1. choose two large primes p and q.
    2. \n
  2. compute \\(n = p\\cdot q\\)
    3. \n
  3. compute \\( \\Phi(n) = (p-1)(q-1)\\)
    4. \n
  4. select the public exponent \\( e \\in {1,2,…,\\Phi(n)-1} \\) such that
    \\[ gcd(e,\\Phi(n)) = 1\\]
    5. \n
  5. compute the private key d such that
    \\[ d \\cdot e \\equiv 1 \\bmod \\Phi(n)\\]
    6. \n
\n
\n

the condition that \\( gcd(e,\\Phi(n)) = 1\\) ensures that the inverse of \\(e\\) exists modulo \\(\\Phi(n)\\), so that there is always a private key \\(d\\).
the computation of key keys \\(d\\) and \\(e\\) canb e doen at once using the extended Euclidean algorith.

\n

Encryption and Decryption

\n

RSA Encryption Given the privaate key \\( k_{pub} = (n,e) \\) and the plaintext \\(x\\), the encryption is:
\\[ y = e_{k_{pub}}(x) \\equiv x^{e} \\bmod n\\]
where \\(x,y \\in Z_{n}\\)

\n
\n
\n

RSA Decryption Given the public key \\d = k_{pr} \\) and the plaintext \\(y\\), the decryption is:
\\[ x = d_{k_{pr}}(y) \\equiv y^{d} \\bmod n\\]
where \\(x,y \\in Z_{n}\\)

\n
\n

Digital signature

the message \\(x\\) that is being signed is in the range \\(1,2,…,n-1\\)
\"rsa

\n

references

\n","site":{"data":{}},"excerpt":"","more":"\n\n\n\n

introduction

the security of RSA relies on the difficulty of factoring a product of two large primes(the integer factorization problem). it is firstly presented in 1978 in [1].

\n

Euclidean Algorithm

the gcd of two positive integers \\(r_0\\) and \\(r_1\\) is denoted by
\\[gcd(r_0, r_1)\\]
the Eucliedean algorithm is used to calculate gcd, based on as simple observation that
\\[gcd(r_0, r_1) = gcd(r_0-r_1, r_1)\\]
where we assume \\(r_0 > r_1\\)
This property can easily be proven: Let \\(gcd(r_0, r_1) = g\\), since \\(g\\) divides both \\(r_0\\) and \\(r_1\\), we can write \\(r_0 = g \\cdot x\\) and \\(r_1 = g \\cdot y\\), where \\(x>y\\) and \\(x\\) and \\(y\\) are coprime integers, i.e., they do not have common factors. Moreover, it is easy to show that \\((x-y)\\) and \\(y\\) are also coprime. it follows from here that
\\[gcd(r_0 - r_1, r_1) = gcd(g \\cdot (x-y), g\\cdot y) = g\\]

\n

it also follow immediately that we can apply the process iteratively:
\\[gcd(r_0 - r_1, r_1) = gcd(r_0 - mr_1, r_1) \\]
as long as \\((r_0 - mr_1) >0\\). the algorithm use the fewest number of steps if we choose maximum value of \\(m\\). this is the case if we compute:
\\[gcd(r_0, r_1) = gcd( r_1, r_0 \\bmod r_1) \\]
this process can be applied recursively until we obtain finally \\[gcd(r_l, 0) = r_l \\]
the euclidean algorithm is very efficient, even with the very long numbers. the number of iterations is close to the number of digits of the input operands. that means, for instance, that the number of iterations of a gcd involving 1024-bit nubmers is 1024 times a constant.

\n

Extended Euclidean Algorithm

an extension of the Euclidean algorithm allows us to compute modular inverse. in addition to computing the gcd, the extended Euclidean algorithm computes a linear combination of the form
\\[gcd(r_0, r_1) = s \\cdot r_0 + t\\cdot r_1 \\]
where s and t are integer coefficients. this equation is ofthen referred to as Diophantine equation

\n

the detail of the algorithm can be foud in section 6.3.2 of the book understanding cryptography by Christof Paar. Here presents the general idea by using an example.
let \\(r_0 =973 , r_1 = 301\\). during the steps of Euclidean Algorithm, we obtain \\(973 = 3\\cdot301 + 70\\)
which is \\[r_0 = q_1 \\cdot r_1 + r_2\\]
rearrange:
\\[r_2 = r_0 + (-q_1) \\cdot r_1\\]
replacing (r_0, r_1) and iteratively by (r_1, r_2), … (r_{i-1}, r_{i}), util \\(r_{i+1} = 0\\)
then \\(r_{i}\\) is \\(gcd(r_0,r_1)\\), and can have a representation of
\\[gcd(r_0, r_1) = s\\cdot r_0 + t\\cdot r_1 \\].
since the inverse only exists if \\(gcd(r_0, r_1)=1\\). we obtain
\\[ s\\cdot r_0 + t\\cdot r_1 = 1\\]
taking this equation modulo \\(r_0\\) we obtain
\\[ s\\cdot 0 + t\\cdot r_1 \\equiv 1 \\bmod r_0\\]
\\[ t\\cdot r_1 \\equiv 1 \\bmod r_0\\]
t is the definition of the inverse of \\(r_1\\)

\n

Thus, if we need to compute an inverse \\(a^{-1} \\bmod m\\), we apply EEA with the input parameters \\(m\\) and \\(a\\)

\n

Euler’s Phi Function

we consider the ring \\( Z_m\\) i.e., the set of integers \\({0,1,…,m-1}\\). we are interested in teh problem of knowing how many numbers in this set are relatively prime to m. this quantity is given by Euler’s phi function, which is \\(\\Phi(m)\\)

\n
\n

let m have the following canonical factorization
\\[ m = p_{1}^{e_1} \\cdot p_{2}^{e_2} \\cdot … \\cdot p_{n}^{e_n}\\]
where the \\(p_i\\) are distinct prime numbers and \\( e_i\\) are positive integers, then

\n

\\[ \\Phi(m) = \\prod_{i=1}^{n}(p_{i}^{e_i} - p_{i}^{e_i -1} ) \\]

\n
\n

it is important to stress that we need to know the factoorization of m in order to calculate Euler’s phi function.

\n

Fermat’s little theorem

Fermat’s little theorem states that if p is a prime number, then for any integer a, the number
\\(a^{p}-a \\) is an integer multiple of p. In the notation of modular arithmetic, this is expressed as
\\[ a^{p} \\equiv a \\bmod p\\]
the theorem can be stated in the form also,
\\[ a^{p-1} \\equiv 1 \\bmod p\\]
then the inverse of an integer is,
\\[ a^{-1} \\equiv a^{p-2} \\bmod p\\]
performing the above formulate (involving exponentiation) to find inverse is usually slower than using extended Euclidean algorithm. However, there are situations where it is advantageous to use Fermat’s Little Theorem, e.g., on smart cards or other devices which have a hardware accelerator for fast exponentiation anyway.

\n

a generatlization of Fermat’s little Theorem to any integer moduli, i.e., moduli that are not necessarily primes, is Euler’s theorem.

\n
\n

Euler’s Theorem
let \\(a\\) and \\(m\\) be integers with \\(gcd(a,m) = 1\\), then
\\[ a^{\\Phi(m)} \\equiv 1 \\bmod m\\]

\n
\n

since it works modulo m, it is applicable to integer rings \\(Z_{m}\\)

\n

key generation

\n

Output: public key: \\( k_{pub} = (n,e) and private key: k_{pr} = (d) \\)

\n
    \n
  1. choose two large primes p and q.
    2. \n
  2. compute \\(n = p\\cdot q\\)
    3. \n
  3. compute \\( \\Phi(n) = (p-1)(q-1)\\)
    4. \n
  4. select the public exponent \\( e \\in {1,2,…,\\Phi(n)-1} \\) such that
    \\[ gcd(e,\\Phi(n)) = 1\\]
    5. \n
  5. compute the private key d such that
    \\[ d \\cdot e \\equiv 1 \\bmod \\Phi(n)\\]
    6. \n
\n
\n

the condition that \\( gcd(e,\\Phi(n)) = 1\\) ensures that the inverse of \\(e\\) exists modulo \\(\\Phi(n)\\), so that there is always a private key \\(d\\).
the computation of key keys \\(d\\) and \\(e\\) canb e doen at once using the extended Euclidean algorith.

\n

Encryption and Decryption

\n

RSA Encryption Given the privaate key \\( k_{pub} = (n,e) \\) and the plaintext \\(x\\), the encryption is:
\\[ y = e_{k_{pub}}(x) \\equiv x^{e} \\bmod n\\]
where \\(x,y \\in Z_{n}\\)

\n
\n
\n

RSA Decryption Given the public key \\d = k_{pr} \\) and the plaintext \\(y\\), the decryption is:
\\[ x = d_{k_{pr}}(y) \\equiv y^{d} \\bmod n\\]
where \\(x,y \\in Z_{n}\\)

\n
\n

Digital signature

the message \\(x\\) that is being signed is in the range \\(1,2,…,n-1\\)
\"rsa

\n

references

\n"},{"title":"paillier encryption","date":"2023-02-23T13:25:41.000Z","_content":"\n\n\n\n## fundamentals\n1. fundamental theorem of arighmetic\nthe fundamental theorem of arithmetic, also called the unique factorization theorem and prime factorization theorem, states that every integer greater than 1 can be represented uniquely as a product of prime numbers, up to the order of the factors [wiki](https://en.wikipedia.org/wiki/Fundamental_theorem_of_arithmetic)\n2. Euler's totient function\nIn number theory, Euler's totient function counts the positive integers up to a given integer n that are relatively prime to n. It is written using the Greek letter phi as \\\\( \\phi (n) \\\\), and may also be called Euler's phi function. In other words, it is the number of integers k in the range 1 ≤ k ≤ n for which the greatest common divisor gcd(n, k) is equal to 1. The integers k of this form are sometimes referred to as totatives of n. the collection of k is denoted by \\\\( Z_{n}^{\\ast } \\\\), and \\\\[ \\phi(n) = |Z_n^{\\ast }| \\\\]\n3. if p is prime, then \\\\( Z_p^{\\ast } = Z_p \\\\), \\\\( \\phi(p) = p-1 \\\\)\n4. if p is prime, for any integer r, then \\\\( \\begin{align} \\tag{0.1} \\phi(p^{r}) =p^{r-1}\\phi(p)=p^{r-1}(p-1)\\end{align} \\\\)\n5. Euler's totient function is a multiplicative function, meaning that if two numbers m and n are relatively prime, then \\\\(\\phi(mn) = \\phi(m)\\phi(n)\\\\)\n6. Euler's product formula, it states\n\\\\[ \\phi(n) = n \\prod_{p|n}^{}(1-\\frac{1}{p}) \\\\]\nwhere the product is over the distinct prime numbers dividing n.\n7. Euler's theorem\nif \\\\(a\\\\) and \\\\(n\\\\) are coprime positive integers, and \\\\( \\phi(n)\\\\) is Euler's totient function, then \\\\(a\\\\) raised to the power \\\\(\\phi(n)\\\\) is congruent to 1 modulo n; that is\n\\\\[a^{\\phi(n)} \\equiv 1 \\bmod n\\\\]\n8. according to 7, we have \\\\( a \\cdot a^{\\phi(n)-1} \\equiv 1 \\bmod n \\\\). then\n\\\\[ a^{-1} = a^{\\phi(n)-1} \\\\]\n9. Fermat's little theorem\nFermat's little theorem states that if p is a prime number, then for any integer a, the number \n\\\\(a^{p}-a \\\\) is an integer multiple of p. In the notation of modular arithmetic, this is expressed as\n\\\\[ a^{p} \\equiv a \\bmod p\\\\]\n10. Binomial theorem\nit states\n\\\\[ y = (1+n)^{x} = \\sum_{k=0}^{x}\\tbinom{x}{k}n^{k} = 1 + nx + \\tbinom{x}{2}n^2 + ...\\\\]\nobserve that, the higher degree could be divided by \\\\(n^2\\\\). we have\n\\\\[ \\begin{align} \\tag{0.2} (1+n)^{x} \\equiv 1 + nx \\bmod n^2 \\end{align} \\\\]\ntherefore, \\\\( y - 1 \\equiv nx \\bmod n^2 \\\\). then we have\n\\\\[ x \\equiv \\frac{y-1}{n} \\bmod n \\\\].\nIn paillier, later we define \\\\( \\begin{align} \\tag{0.3} L(y) = \\frac{y-1}{n} \\end{align} \\\\)\ntherefore\n\\\\[ L(y \\bmod n^2) \\equiv x \\bmod n \\\\]\n\n\n## Paillier\n1. key generation\n`KeyGen() -> (pk, sk)`\nrandomly select two big prime numbers \\\\(p, q\\\\). it shoud satisfy \\\\(gcd(pq, (p-1)(q-1)) =1 \\\\), \\\\(p\\\\) and \\\\(q\\\\) should have similar bit length. let \\\\( n = pq \\\\), \\\\(\\lambda = lcm(p-1, q-1)\\\\). randomly sample \\\\( g \\in Z_{n^2}^{\\ast}\\\\). to simplify, let \\\\( g = n+1\\\\). we have\n\\\\[ pk=(n,g) \\\\]\n\\\\[ sk = (\\lambda)\\\\]\n\n2. encryption\n`Enc(pk, m) -> c`\nrandomly sample \\\\( r \\in Z_{n}^{\\ast}\\\\), then also have \\\\( r \\in Z_{n^2}^{\\ast}\\\\), cypher is calculated\n\\\\[ \\begin{align} \\tag{1.1} c = g^mr^n \\bmod n^2 \\end{align} \\\\]\n\n3. Decryption\n`Dec(sk, c) -> m`\nLet \\\\(L(x) = \\frac{x-1}{n} \\\\), we have message\n\\\\[ \\begin{align} \\tag{1.2} m = \\frac{L(c^{\\lambda} \\bmod n^2)}{L(g^{\\lambda} \\bmod n^2)} \\bmod n \\end{align}\\\\]\n\n4. proof of correctness\nbased on Eq(1), we have \\\\[ \\begin{align} \\tag{1.3} c^{\\lambda} \\bmod n^2 = g^{m\\lambda}r^{n\\lambda} \\bmod n^2 \\end{align}\\\\]\nwhere \\\\( r^{n\\lambda} \\bmod n^2 \\equiv 1 \\bmod n^2\\\\), which is proved by Carmichael theorem later on. then Eq(3) becomes\n \\\\[ \\begin{align} \\tag{1.4} c^{\\lambda} \\bmod n^2 = g^{m\\lambda}\\bmod n^2 \\end{align}\\\\]\nsince \\\\( g = n+1\\\\), we have\n\\\\[ \\begin{align} \\tag{1.5} c^{\\lambda} \\bmod n^2 = (1+n)^{m\\lambda}\\bmod n^2 \\end{align}\\\\]\nAccording to Eq(0.2), we have\n\\\\[ \\begin{align} \\tag{1.6} c^{\\lambda} \\bmod n^2 = 1 + nm\\lambda \\bmod n^2 \\end{align}\\\\]\n\\\\[ \\begin{align} \\tag{1.7} g^{\\lambda} \\bmod n^2 \\equiv (1+n)^{\\lambda} \\bmod n^2 = 1 +\\lambda n \\bmod n^2 \\end{align}\\\\]\ntherefore, based on definition given by Eq(0.3) we have\n\\\\[ \\begin{align} \\tag{1.8} L(c^{\\lambda} \\bmod n^2) = \\frac{c^{\\lambda}-1}{n} \\bmod n^2 \\end{align} \\\\]\nSubstitute Eq(1.6) into Eq(1.8), we have\n\\\\[ \\begin{align} \\tag{1.9} L(c^{\\lambda} \\bmod n^2) = m\\lambda \\bmod n^2 \\end{align} \\\\]\nFurther, we have\n\\\\[ \\begin{align} \\tag{1.10} L(g^{\\lambda} \\bmod n^2) = \\frac{g^\\lambda -1}{n} \\end{align} \\\\]\nSub Eq(1.7) into Eq(1.10), we have\n\\\\[ \\begin{align} \\tag{1.11} L(g^{\\lambda} \\bmod n^2) = \\frac{\\lambda n}{n} \\equiv \\lambda \\bmod n^2\\end{align} \\\\]\nAt last, Eq(1.2) becomes （bu sub Eq1.9 and Eq1.11)\n\\\\[ \\begin{align} m = \\frac{L(c^{\\lambda} \\bmod n^2)}{L(g^{\\lambda} \\bmod n^2)} \\bmod n = \\frac{m \\lambda}{\\lambda} \\equiv m \\bmod n \\end{align}\\\\]\nproved!!!\n\n5. Carmichael theorem\nIn number theory, a branch of mathematics, the Carmichael function \\\\(λ(n)\\\\) of a positive integer n is the smallest positive integer m such that \\\\(a^{m}\\equiv 1{\\pmod {n}}\\\\) (similar but different from Euler's totient function). Carmichael's λ function, the reduced totient function, and the least universal exponent function\n![carmichael theorem](/images/paillier/carmichael_thorem.png)\n![](/images/paillier/carmichael_thorem_2.png)\nlet \\\\( n = pq\\\\), where p and q are prime numbers; \\\\( \\phi(n)\\\\) is the Euler's totient function. Let \\\\(\\lambda(n)\\\\) denotes carmichael function. We have \\\\(\\phi(n)=(p-1)(q-1)\\\\) and \\\\( \\lambda(n)=\\phi(n) = (p-1)(q-1)\\\\).\n\nSince \\\\( |Z_{n^2}^{\\ast}| = \\phi(n^2) = n \\phi(n)\\\\) (according to Eq(0.1)). Thereby, for any \\\\( w \\in Z_{n^2}^{\\ast}\\\\)\n\\\\[ \\begin{align} \\tag{1.12} w^{n\\phi(n)} \\equiv w^{n\\\\lambda} \\equiv 1 \\bmod n^2 \\end{align}\\\\]\n\n\\\\[ \\begin{align} \\tag{1.13} w^{\\lambda} \\equiv 1 \\bmod n \\end{align}\\\\]\nEq(1.13) is just Carmichael's function\n\nBased on Carmichael's theorem\n\\\\[ \\lambda(n^2) = lcm(\\lambda(q^2),\\lambda(p^2)) = lcm(\\phi(q^2),\\phi(p^2)) = lcm(q(q-1), p(p-1)) = pq(lcm(p-1, q-1)) = n\\lambda(n) \\\\] \ntherefore, we have\n\n\\\\[w^{\\lambda(n^2)} = w ^{n\\lambda} \\equiv 1 \\bmod n^2\\\\]\n\n6. Addition homomorphic\n![homomorphic addition](/images/paillier/homomorphic_addition.png)\n\n7. Multiplication homomorphic \n![homomorphic multiplication](/images/paillier/homomorphic_mul.png)\n## references\n- [csdn post](https://blog.csdn.net/qq_42328228/article/details/109349590)","source":"_posts/cryptography/paillier-encryption.md","raw":"---\ntitle: paillier encryption\ndate: 2023-02-23 21:25:41\ntags: [cryptography]\n---\n\n\n\n\n## fundamentals\n1. fundamental theorem of arighmetic\nthe fundamental theorem of arithmetic, also called the unique factorization theorem and prime factorization theorem, states that every integer greater than 1 can be represented uniquely as a product of prime numbers, up to the order of the factors [wiki](https://en.wikipedia.org/wiki/Fundamental_theorem_of_arithmetic)\n2. Euler's totient function\nIn number theory, Euler's totient function counts the positive integers up to a given integer n that are relatively prime to n. It is written using the Greek letter phi as \\\\( \\phi (n) \\\\), and may also be called Euler's phi function. In other words, it is the number of integers k in the range 1 ≤ k ≤ n for which the greatest common divisor gcd(n, k) is equal to 1. The integers k of this form are sometimes referred to as totatives of n. the collection of k is denoted by \\\\( Z_{n}^{\\ast } \\\\), and \\\\[ \\phi(n) = |Z_n^{\\ast }| \\\\]\n3. if p is prime, then \\\\( Z_p^{\\ast } = Z_p \\\\), \\\\( \\phi(p) = p-1 \\\\)\n4. if p is prime, for any integer r, then \\\\( \\begin{align} \\tag{0.1} \\phi(p^{r}) =p^{r-1}\\phi(p)=p^{r-1}(p-1)\\end{align} \\\\)\n5. Euler's totient function is a multiplicative function, meaning that if two numbers m and n are relatively prime, then \\\\(\\phi(mn) = \\phi(m)\\phi(n)\\\\)\n6. Euler's product formula, it states\n\\\\[ \\phi(n) = n \\prod_{p|n}^{}(1-\\frac{1}{p}) \\\\]\nwhere the product is over the distinct prime numbers dividing n.\n7. Euler's theorem\nif \\\\(a\\\\) and \\\\(n\\\\) are coprime positive integers, and \\\\( \\phi(n)\\\\) is Euler's totient function, then \\\\(a\\\\) raised to the power \\\\(\\phi(n)\\\\) is congruent to 1 modulo n; that is\n\\\\[a^{\\phi(n)} \\equiv 1 \\bmod n\\\\]\n8. according to 7, we have \\\\( a \\cdot a^{\\phi(n)-1} \\equiv 1 \\bmod n \\\\). then\n\\\\[ a^{-1} = a^{\\phi(n)-1} \\\\]\n9. Fermat's little theorem\nFermat's little theorem states that if p is a prime number, then for any integer a, the number \n\\\\(a^{p}-a \\\\) is an integer multiple of p. In the notation of modular arithmetic, this is expressed as\n\\\\[ a^{p} \\equiv a \\bmod p\\\\]\n10. Binomial theorem\nit states\n\\\\[ y = (1+n)^{x} = \\sum_{k=0}^{x}\\tbinom{x}{k}n^{k} = 1 + nx + \\tbinom{x}{2}n^2 + ...\\\\]\nobserve that, the higher degree could be divided by \\\\(n^2\\\\). we have\n\\\\[ \\begin{align} \\tag{0.2} (1+n)^{x} \\equiv 1 + nx \\bmod n^2 \\end{align} \\\\]\ntherefore, \\\\( y - 1 \\equiv nx \\bmod n^2 \\\\). then we have\n\\\\[ x \\equiv \\frac{y-1}{n} \\bmod n \\\\].\nIn paillier, later we define \\\\( \\begin{align} \\tag{0.3} L(y) = \\frac{y-1}{n} \\end{align} \\\\)\ntherefore\n\\\\[ L(y \\bmod n^2) \\equiv x \\bmod n \\\\]\n\n\n## Paillier\n1. key generation\n`KeyGen() -> (pk, sk)`\nrandomly select two big prime numbers \\\\(p, q\\\\). it shoud satisfy \\\\(gcd(pq, (p-1)(q-1)) =1 \\\\), \\\\(p\\\\) and \\\\(q\\\\) should have similar bit length. let \\\\( n = pq \\\\), \\\\(\\lambda = lcm(p-1, q-1)\\\\). randomly sample \\\\( g \\in Z_{n^2}^{\\ast}\\\\). to simplify, let \\\\( g = n+1\\\\). we have\n\\\\[ pk=(n,g) \\\\]\n\\\\[ sk = (\\lambda)\\\\]\n\n2. encryption\n`Enc(pk, m) -> c`\nrandomly sample \\\\( r \\in Z_{n}^{\\ast}\\\\), then also have \\\\( r \\in Z_{n^2}^{\\ast}\\\\), cypher is calculated\n\\\\[ \\begin{align} \\tag{1.1} c = g^mr^n \\bmod n^2 \\end{align} \\\\]\n\n3. Decryption\n`Dec(sk, c) -> m`\nLet \\\\(L(x) = \\frac{x-1}{n} \\\\), we have message\n\\\\[ \\begin{align} \\tag{1.2} m = \\frac{L(c^{\\lambda} \\bmod n^2)}{L(g^{\\lambda} \\bmod n^2)} \\bmod n \\end{align}\\\\]\n\n4. proof of correctness\nbased on Eq(1), we have \\\\[ \\begin{align} \\tag{1.3} c^{\\lambda} \\bmod n^2 = g^{m\\lambda}r^{n\\lambda} \\bmod n^2 \\end{align}\\\\]\nwhere \\\\( r^{n\\lambda} \\bmod n^2 \\equiv 1 \\bmod n^2\\\\), which is proved by Carmichael theorem later on. then Eq(3) becomes\n \\\\[ \\begin{align} \\tag{1.4} c^{\\lambda} \\bmod n^2 = g^{m\\lambda}\\bmod n^2 \\end{align}\\\\]\nsince \\\\( g = n+1\\\\), we have\n\\\\[ \\begin{align} \\tag{1.5} c^{\\lambda} \\bmod n^2 = (1+n)^{m\\lambda}\\bmod n^2 \\end{align}\\\\]\nAccording to Eq(0.2), we have\n\\\\[ \\begin{align} \\tag{1.6} c^{\\lambda} \\bmod n^2 = 1 + nm\\lambda \\bmod n^2 \\end{align}\\\\]\n\\\\[ \\begin{align} \\tag{1.7} g^{\\lambda} \\bmod n^2 \\equiv (1+n)^{\\lambda} \\bmod n^2 = 1 +\\lambda n \\bmod n^2 \\end{align}\\\\]\ntherefore, based on definition given by Eq(0.3) we have\n\\\\[ \\begin{align} \\tag{1.8} L(c^{\\lambda} \\bmod n^2) = \\frac{c^{\\lambda}-1}{n} \\bmod n^2 \\end{align} \\\\]\nSubstitute Eq(1.6) into Eq(1.8), we have\n\\\\[ \\begin{align} \\tag{1.9} L(c^{\\lambda} \\bmod n^2) = m\\lambda \\bmod n^2 \\end{align} \\\\]\nFurther, we have\n\\\\[ \\begin{align} \\tag{1.10} L(g^{\\lambda} \\bmod n^2) = \\frac{g^\\lambda -1}{n} \\end{align} \\\\]\nSub Eq(1.7) into Eq(1.10), we have\n\\\\[ \\begin{align} \\tag{1.11} L(g^{\\lambda} \\bmod n^2) = \\frac{\\lambda n}{n} \\equiv \\lambda \\bmod n^2\\end{align} \\\\]\nAt last, Eq(1.2) becomes （bu sub Eq1.9 and Eq1.11)\n\\\\[ \\begin{align} m = \\frac{L(c^{\\lambda} \\bmod n^2)}{L(g^{\\lambda} \\bmod n^2)} \\bmod n = \\frac{m \\lambda}{\\lambda} \\equiv m \\bmod n \\end{align}\\\\]\nproved!!!\n\n5. Carmichael theorem\nIn number theory, a branch of mathematics, the Carmichael function \\\\(λ(n)\\\\) of a positive integer n is the smallest positive integer m such that \\\\(a^{m}\\equiv 1{\\pmod {n}}\\\\) (similar but different from Euler's totient function). Carmichael's λ function, the reduced totient function, and the least universal exponent function\n![carmichael theorem](/images/paillier/carmichael_thorem.png)\n![](/images/paillier/carmichael_thorem_2.png)\nlet \\\\( n = pq\\\\), where p and q are prime numbers; \\\\( \\phi(n)\\\\) is the Euler's totient function. Let \\\\(\\lambda(n)\\\\) denotes carmichael function. We have \\\\(\\phi(n)=(p-1)(q-1)\\\\) and \\\\( \\lambda(n)=\\phi(n) = (p-1)(q-1)\\\\).\n\nSince \\\\( |Z_{n^2}^{\\ast}| = \\phi(n^2) = n \\phi(n)\\\\) (according to Eq(0.1)). Thereby, for any \\\\( w \\in Z_{n^2}^{\\ast}\\\\)\n\\\\[ \\begin{align} \\tag{1.12} w^{n\\phi(n)} \\equiv w^{n\\\\lambda} \\equiv 1 \\bmod n^2 \\end{align}\\\\]\n\n\\\\[ \\begin{align} \\tag{1.13} w^{\\lambda} \\equiv 1 \\bmod n \\end{align}\\\\]\nEq(1.13) is just Carmichael's function\n\nBased on Carmichael's theorem\n\\\\[ \\lambda(n^2) = lcm(\\lambda(q^2),\\lambda(p^2)) = lcm(\\phi(q^2),\\phi(p^2)) = lcm(q(q-1), p(p-1)) = pq(lcm(p-1, q-1)) = n\\lambda(n) \\\\] \ntherefore, we have\n\n\\\\[w^{\\lambda(n^2)} = w ^{n\\lambda} \\equiv 1 \\bmod n^2\\\\]\n\n6. Addition homomorphic\n![homomorphic addition](/images/paillier/homomorphic_addition.png)\n\n7. Multiplication homomorphic \n![homomorphic multiplication](/images/paillier/homomorphic_mul.png)\n## references\n- [csdn post](https://blog.csdn.net/qq_42328228/article/details/109349590)","slug":"cryptography/paillier-encryption","published":1,"updated":"2023-11-05T04:21:13.732Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8dp000nqwsjgbxocb3i","content":"\n\n\n

fundamentals

    \n
  1. fundamental theorem of arighmetic
    the fundamental theorem of arithmetic, also called the unique factorization theorem and prime factorization theorem, states that every integer greater than 1 can be represented uniquely as a product of prime numbers, up to the order of the factors wiki
    2. \n
  2. Euler’s totient function
    In number theory, Euler’s totient function counts the positive integers up to a given integer n that are relatively prime to n. It is written using the Greek letter phi as \\( \\phi (n) \\), and may also be called Euler’s phi function. In other words, it is the number of integers k in the range 1 ≤ k ≤ n for which the greatest common divisor gcd(n, k) is equal to 1. The integers k of this form are sometimes referred to as totatives of n. the collection of k is denoted by \\( Z_{n}^{\\ast } \\), and \\[ \\phi(n) = |Z_n^{\\ast }| \\]
    3. \n
  3. if p is prime, then \\( Z_p^{\\ast } = Z_p \\), \\( \\phi(p) = p-1 \\)
    4. \n
  4. if p is prime, for any integer r, then \\( \\begin{align} \\tag{0.1} \\phi(p^{r}) =p^{r-1}\\phi(p)=p^{r-1}(p-1)\\end{align} \\)
    5. \n
  5. Euler’s totient function is a multiplicative function, meaning that if two numbers m and n are relatively prime, then \\(\\phi(mn) = \\phi(m)\\phi(n)\\)
    6. \n
  6. Euler’s product formula, it states
    \\[ \\phi(n) = n \\prod_{p|n}^{}(1-\\frac{1}{p}) \\]
    where the product is over the distinct prime numbers dividing n.
    7. \n
  7. Euler’s theorem
    if \\(a\\) and \\(n\\) are coprime positive integers, and \\( \\phi(n)\\) is Euler’s totient function, then \\(a\\) raised to the power \\(\\phi(n)\\) is congruent to 1 modulo n; that is
    \\[a^{\\phi(n)} \\equiv 1 \\bmod n\\]
    8. \n
  8. according to 7, we have \\( a \\cdot a^{\\phi(n)-1} \\equiv 1 \\bmod n \\). then
    \\[ a^{-1} = a^{\\phi(n)-1} \\]
    9. \n
  9. Fermat’s little theorem
    Fermat’s little theorem states that if p is a prime number, then for any integer a, the number
    \\(a^{p}-a \\) is an integer multiple of p. In the notation of modular arithmetic, this is expressed as
    \\[ a^{p} \\equiv a \\bmod p\\]
    10. \n
  10. Binomial theorem
    it states
    \\[ y = (1+n)^{x} = \\sum_{k=0}^{x}\\tbinom{x}{k}n^{k} = 1 + nx + \\tbinom{x}{2}n^2 + …\\]
    observe that, the higher degree could be divided by \\(n^2\\). we have
    \\[ \\begin{align} \\tag{0.2} (1+n)^{x} \\equiv 1 + nx \\bmod n^2 \\end{align} \\]
    therefore, \\( y - 1 \\equiv nx \\bmod n^2 \\). then we have
    \\[ x \\equiv \\frac{y-1}{n} \\bmod n \\].
    In paillier, later we define \\( \\begin{align} \\tag{0.3} L(y) = \\frac{y-1}{n} \\end{align} \\)
    therefore
    \\[ L(y \\bmod n^2) \\equiv x \\bmod n \\]
    11. \n
\n

Paillier

    \n

  1. key generation
    KeyGen() -> (pk, sk)
    randomly select two big prime numbers \\(p, q\\). it shoud satisfy \\(gcd(pq, (p-1)(q-1)) =1 \\), \\(p\\) and \\(q\\) should have similar bit length. let \\( n = pq \\), \\(\\lambda = lcm(p-1, q-1)\\). randomly sample \\( g \\in Z_{n^2}^{\\ast}\\). to simplify, let \\( g = n+1\\). we have
    \\[ pk=(n,g) \\]
    \\[ sk = (\\lambda)\\]

    \n
    2. \n

  2. encryption
    Enc(pk, m) -> c
    randomly sample \\( r \\in Z_{n}^{\\ast}\\), then also have \\( r \\in Z_{n^2}^{\\ast}\\), cypher is calculated
    \\[ \\begin{align} \\tag{1.1} c = g^mr^n \\bmod n^2 \\end{align} \\]

    \n
    3. \n

  3. Decryption
    Dec(sk, c) -> m
    Let \\(L(x) = \\frac{x-1}{n} \\), we have message
    \\[ \\begin{align} \\tag{1.2} m = \\frac{L(c^{\\lambda} \\bmod n^2)}{L(g^{\\lambda} \\bmod n^2)} \\bmod n \\end{align}\\]

    \n
    4. \n

  4. proof of correctness
    based on Eq(1), we have \\[ \\begin{align} \\tag{1.3} c^{\\lambda} \\bmod n^2 = g^{m\\lambda}r^{n\\lambda} \\bmod n^2 \\end{align}\\]
    where \\( r^{n\\lambda} \\bmod n^2 \\equiv 1 \\bmod n^2\\), which is proved by Carmichael theorem later on. then Eq(3) becomes
    \\[ \\begin{align} \\tag{1.4} c^{\\lambda} \\bmod n^2 = g^{m\\lambda}\\bmod n^2 \\end{align}\\]
    since \\( g = n+1\\), we have
    \\[ \\begin{align} \\tag{1.5} c^{\\lambda} \\bmod n^2 = (1+n)^{m\\lambda}\\bmod n^2 \\end{align}\\]
    According to Eq(0.2), we have
    \\[ \\begin{align} \\tag{1.6} c^{\\lambda} \\bmod n^2 = 1 + nm\\lambda \\bmod n^2 \\end{align}\\]
    \\[ \\begin{align} \\tag{1.7} g^{\\lambda} \\bmod n^2 \\equiv (1+n)^{\\lambda} \\bmod n^2 = 1 +\\lambda n \\bmod n^2 \\end{align}\\]
    therefore, based on definition given by Eq(0.3) we have
    \\[ \\begin{align} \\tag{1.8} L(c^{\\lambda} \\bmod n^2) = \\frac{c^{\\lambda}-1}{n} \\bmod n^2 \\end{align} \\]
    Substitute Eq(1.6) into Eq(1.8), we have
    \\[ \\begin{align} \\tag{1.9} L(c^{\\lambda} \\bmod n^2) = m\\lambda \\bmod n^2 \\end{align} \\]
    Further, we have
    \\[ \\begin{align} \\tag{1.10} L(g^{\\lambda} \\bmod n^2) = \\frac{g^\\lambda -1}{n} \\end{align} \\]
    Sub Eq(1.7) into Eq(1.10), we have
    \\[ \\begin{align} \\tag{1.11} L(g^{\\lambda} \\bmod n^2) = \\frac{\\lambda n}{n} \\equiv \\lambda \\bmod n^2\\end{align} \\]
    At last, Eq(1.2) becomes （bu sub Eq1.9 and Eq1.11)
    \\[ \\begin{align} m = \\frac{L(c^{\\lambda} \\bmod n^2)}{L(g^{\\lambda} \\bmod n^2)} \\bmod n = \\frac{m \\lambda}{\\lambda} \\equiv m \\bmod n \\end{align}\\]
    proved!!!

    \n
    5. \n

  5. Carmichael theorem
    In number theory, a branch of mathematics, the Carmichael function \\(λ(n)\\) of a positive integer n is the smallest positive integer m such that \\(a^{m}\\equiv 1{\\pmod {n}}\\) (similar but different from Euler’s totient function). Carmichael’s λ function, the reduced totient function, and the least universal exponent function
    \"carmichael

    let \\( n = pq\\), where p and q are prime numbers; \\( \\phi(n)\\) is the Euler’s totient function. Let \\(\\lambda(n)\\) denotes carmichael function. We have \\(\\phi(n)=(p-1)(q-1)\\) and \\( \\lambda(n)=\\phi(n) = (p-1)(q-1)\\).

    \n
    6. \n
\n

Since \\( |Z_{n^2}^{\\ast}| = \\phi(n^2) = n \\phi(n)\\) (according to Eq(0.1)). Thereby, for any \\( w \\in Z_{n^2}^{\\ast}\\)
\\[ \\begin{align} \\tag{1.12} w^{n\\phi(n)} \\equiv w^{n\\lambda} \\equiv 1 \\bmod n^2 \\end{align}\\]

\n

\\[ \\begin{align} \\tag{1.13} w^{\\lambda} \\equiv 1 \\bmod n \\end{align}\\]
Eq(1.13) is just Carmichael’s function

\n

Based on Carmichael’s theorem
\\[ \\lambda(n^2) = lcm(\\lambda(q^2),\\lambda(p^2)) = lcm(\\phi(q^2),\\phi(p^2)) = lcm(q(q-1), p(p-1)) = pq(lcm(p-1, q-1)) = n\\lambda(n) \\]
therefore, we have

\n

\\[w^{\\lambda(n^2)} = w ^{n\\lambda} \\equiv 1 \\bmod n^2\\]

\n
    \n

  1. Addition homomorphic
    \"homomorphic

    \n
    2. \n

  2. Multiplication homomorphic
    \"homomorphic

    \n
    3. \n
\n

references

\n","site":{"data":{}},"excerpt":"","more":"\n\n\n

fundamentals

    \n
  1. fundamental theorem of arighmetic
    the fundamental theorem of arithmetic, also called the unique factorization theorem and prime factorization theorem, states that every integer greater than 1 can be represented uniquely as a product of prime numbers, up to the order of the factors wiki
    2. \n
  2. Euler’s totient function
    In number theory, Euler’s totient function counts the positive integers up to a given integer n that are relatively prime to n. It is written using the Greek letter phi as \\( \\phi (n) \\), and may also be called Euler’s phi function. In other words, it is the number of integers k in the range 1 ≤ k ≤ n for which the greatest common divisor gcd(n, k) is equal to 1. The integers k of this form are sometimes referred to as totatives of n. the collection of k is denoted by \\( Z_{n}^{\\ast } \\), and \\[ \\phi(n) = |Z_n^{\\ast }| \\]
    3. \n
  3. if p is prime, then \\( Z_p^{\\ast } = Z_p \\), \\( \\phi(p) = p-1 \\)
    4. \n
  4. if p is prime, for any integer r, then \\( \\begin{align} \\tag{0.1} \\phi(p^{r}) =p^{r-1}\\phi(p)=p^{r-1}(p-1)\\end{align} \\)
    5. \n
  5. Euler’s totient function is a multiplicative function, meaning that if two numbers m and n are relatively prime, then \\(\\phi(mn) = \\phi(m)\\phi(n)\\)
    6. \n
  6. Euler’s product formula, it states
    \\[ \\phi(n) = n \\prod_{p|n}^{}(1-\\frac{1}{p}) \\]
    where the product is over the distinct prime numbers dividing n.
    7. \n
  7. Euler’s theorem
    if \\(a\\) and \\(n\\) are coprime positive integers, and \\( \\phi(n)\\) is Euler’s totient function, then \\(a\\) raised to the power \\(\\phi(n)\\) is congruent to 1 modulo n; that is
    \\[a^{\\phi(n)} \\equiv 1 \\bmod n\\]
    8. \n
  8. according to 7, we have \\( a \\cdot a^{\\phi(n)-1} \\equiv 1 \\bmod n \\). then
    \\[ a^{-1} = a^{\\phi(n)-1} \\]
    9. \n
  9. Fermat’s little theorem
    Fermat’s little theorem states that if p is a prime number, then for any integer a, the number
    \\(a^{p}-a \\) is an integer multiple of p. In the notation of modular arithmetic, this is expressed as
    \\[ a^{p} \\equiv a \\bmod p\\]
    10. \n
  10. Binomial theorem
    it states
    \\[ y = (1+n)^{x} = \\sum_{k=0}^{x}\\tbinom{x}{k}n^{k} = 1 + nx + \\tbinom{x}{2}n^2 + …\\]
    observe that, the higher degree could be divided by \\(n^2\\). we have
    \\[ \\begin{align} \\tag{0.2} (1+n)^{x} \\equiv 1 + nx \\bmod n^2 \\end{align} \\]
    therefore, \\( y - 1 \\equiv nx \\bmod n^2 \\). then we have
    \\[ x \\equiv \\frac{y-1}{n} \\bmod n \\].
    In paillier, later we define \\( \\begin{align} \\tag{0.3} L(y) = \\frac{y-1}{n} \\end{align} \\)
    therefore
    \\[ L(y \\bmod n^2) \\equiv x \\bmod n \\]
    11. \n
\n

Paillier

    \n

  1. key generation
    KeyGen() -> (pk, sk)
    randomly select two big prime numbers \\(p, q\\). it shoud satisfy \\(gcd(pq, (p-1)(q-1)) =1 \\), \\(p\\) and \\(q\\) should have similar bit length. let \\( n = pq \\), \\(\\lambda = lcm(p-1, q-1)\\). randomly sample \\( g \\in Z_{n^2}^{\\ast}\\). to simplify, let \\( g = n+1\\). we have
    \\[ pk=(n,g) \\]
    \\[ sk = (\\lambda)\\]

    \n
    2. \n

  2. encryption
    Enc(pk, m) -> c
    randomly sample \\( r \\in Z_{n}^{\\ast}\\), then also have \\( r \\in Z_{n^2}^{\\ast}\\), cypher is calculated
    \\[ \\begin{align} \\tag{1.1} c = g^mr^n \\bmod n^2 \\end{align} \\]

    \n
    3. \n

  3. Decryption
    Dec(sk, c) -> m
    Let \\(L(x) = \\frac{x-1}{n} \\), we have message
    \\[ \\begin{align} \\tag{1.2} m = \\frac{L(c^{\\lambda} \\bmod n^2)}{L(g^{\\lambda} \\bmod n^2)} \\bmod n \\end{align}\\]

    \n
    4. \n

  4. proof of correctness
    based on Eq(1), we have \\[ \\begin{align} \\tag{1.3} c^{\\lambda} \\bmod n^2 = g^{m\\lambda}r^{n\\lambda} \\bmod n^2 \\end{align}\\]
    where \\( r^{n\\lambda} \\bmod n^2 \\equiv 1 \\bmod n^2\\), which is proved by Carmichael theorem later on. then Eq(3) becomes
    \\[ \\begin{align} \\tag{1.4} c^{\\lambda} \\bmod n^2 = g^{m\\lambda}\\bmod n^2 \\end{align}\\]
    since \\( g = n+1\\), we have
    \\[ \\begin{align} \\tag{1.5} c^{\\lambda} \\bmod n^2 = (1+n)^{m\\lambda}\\bmod n^2 \\end{align}\\]
    According to Eq(0.2), we have
    \\[ \\begin{align} \\tag{1.6} c^{\\lambda} \\bmod n^2 = 1 + nm\\lambda \\bmod n^2 \\end{align}\\]
    \\[ \\begin{align} \\tag{1.7} g^{\\lambda} \\bmod n^2 \\equiv (1+n)^{\\lambda} \\bmod n^2 = 1 +\\lambda n \\bmod n^2 \\end{align}\\]
    therefore, based on definition given by Eq(0.3) we have
    \\[ \\begin{align} \\tag{1.8} L(c^{\\lambda} \\bmod n^2) = \\frac{c^{\\lambda}-1}{n} \\bmod n^2 \\end{align} \\]
    Substitute Eq(1.6) into Eq(1.8), we have
    \\[ \\begin{align} \\tag{1.9} L(c^{\\lambda} \\bmod n^2) = m\\lambda \\bmod n^2 \\end{align} \\]
    Further, we have
    \\[ \\begin{align} \\tag{1.10} L(g^{\\lambda} \\bmod n^2) = \\frac{g^\\lambda -1}{n} \\end{align} \\]
    Sub Eq(1.7) into Eq(1.10), we have
    \\[ \\begin{align} \\tag{1.11} L(g^{\\lambda} \\bmod n^2) = \\frac{\\lambda n}{n} \\equiv \\lambda \\bmod n^2\\end{align} \\]
    At last, Eq(1.2) becomes （bu sub Eq1.9 and Eq1.11)
    \\[ \\begin{align} m = \\frac{L(c^{\\lambda} \\bmod n^2)}{L(g^{\\lambda} \\bmod n^2)} \\bmod n = \\frac{m \\lambda}{\\lambda} \\equiv m \\bmod n \\end{align}\\]
    proved!!!

    \n
    5. \n

  5. Carmichael theorem
    In number theory, a branch of mathematics, the Carmichael function \\(λ(n)\\) of a positive integer n is the smallest positive integer m such that \\(a^{m}\\equiv 1{\\pmod {n}}\\) (similar but different from Euler’s totient function). Carmichael’s λ function, the reduced totient function, and the least universal exponent function
    \"carmichael

    let \\( n = pq\\), where p and q are prime numbers; \\( \\phi(n)\\) is the Euler’s totient function. Let \\(\\lambda(n)\\) denotes carmichael function. We have \\(\\phi(n)=(p-1)(q-1)\\) and \\( \\lambda(n)=\\phi(n) = (p-1)(q-1)\\).

    \n
    6. \n
\n

Since \\( |Z_{n^2}^{\\ast}| = \\phi(n^2) = n \\phi(n)\\) (according to Eq(0.1)). Thereby, for any \\( w \\in Z_{n^2}^{\\ast}\\)
\\[ \\begin{align} \\tag{1.12} w^{n\\phi(n)} \\equiv w^{n\\lambda} \\equiv 1 \\bmod n^2 \\end{align}\\]

\n

\\[ \\begin{align} \\tag{1.13} w^{\\lambda} \\equiv 1 \\bmod n \\end{align}\\]
Eq(1.13) is just Carmichael’s function

\n

Based on Carmichael’s theorem
\\[ \\lambda(n^2) = lcm(\\lambda(q^2),\\lambda(p^2)) = lcm(\\phi(q^2),\\phi(p^2)) = lcm(q(q-1), p(p-1)) = pq(lcm(p-1, q-1)) = n\\lambda(n) \\]
therefore, we have

\n

\\[w^{\\lambda(n^2)} = w ^{n\\lambda} \\equiv 1 \\bmod n^2\\]

\n
    \n

  1. Addition homomorphic
    \"homomorphic

    \n
    2. \n

  2. Multiplication homomorphic
    \"homomorphic

    \n
    3. \n
\n

references

\n"},{"title":"two party ecdsa","date":"2023-02-07T06:29:26.000Z","_content":"\n\n\n## overview\nthis post is my reading summary of paper Yehuda Lindell 2017: Fast secure two-party ecdsa signing. the implementation could be found in tss-lib (golang), zengo's library (rust).\n\nUnlike other schemes like RSA, Schnorr signatures and more, it is particularly hard to construct efficient threshold signature protocols for ECDSA as there is an inverse computaion of \\\\( k \\\\).\n\nIn this paper, we consider the specific case of two parties (and thus no honest majority) and construct a protocol that is approximately two orders of magnitude faster than the previous best.\n\n## Comparing ECDSA signing to EC-Schnorr signing\nIn both cases, the public verification key is an elliptic curve point \\\\( Q \\\\) and the private signing key is \\\\( x \\\\) such that \\\\( Q = x \\cdot G \\\\), where \\\\( G \\\\) is the generator point of an EC group of order \\\\( q \\\\).\n![schnorr ecdsa comparison](/images/two_party_ecdsa/schnorr_ecdsa_comparison.png)\n\nObserve that Schnorr signing can be easily distributed since the private key \\\\( x \\\\) and the value k are both used in a linear equation. In contrast, in ECDSA signing, the equation for computing \\\\( s \\\\) includes \\\\( k^{-1} \\\\). Now, given shares \\\\(k_1\\\\), \\\\(k_2\\\\) such that \\\\(k_1 + k_2 = k \\bmod q\\\\) .It is very difficult to compute \\\\(k_1^{\\prime}\\\\), \\\\(k_2^{\\prime}\\\\) such that \\\\(k_1^{\\prime} + k_2^{\\prime} = k^{-1} \\bmod q\\\\)\n\n\ntwo-party protocols for ECDSA signing use multiplicative sharing of \\\\( x \\\\) and of \\\\( k \\\\). That is, the parties hold \\\\(x_1\\\\), \\\\(x_2\\\\) such that \\\\(x_1 \\cdot x_2 = x \\bmod q\\\\), and in each signing operation they generate \\\\(k_1\\\\), \\\\(k_2\\\\) such that \\\\(k_1 \\cdot k_2 = k \\bmod q\\\\). This enables them to easily compute \\\\(k^{-1}\\\\) since each party can locally compute \\\\(k_i^{\\prime} = k_i^{-1} \\bmod q\\\\), and then \\\\(k_1^{\\prime}\\\\), \\\\(k_2^{\\prime}\\\\) are multiplicative shares of \\\\(k^{-1}\\\\). The parties can then use additively homomorphic encryption – specifically Paillier encryption – in order to combine their equations. For example, \\\\(P_1\\\\) can compute \\\\(c_1 = Enc_{pk}(k_1^{-1} \\cdot H(m))\\\\) and \\\\(c_2 = Enc_{pk}(k_1^{-1} \\cdot x_1 \\cdot r)\\\\) . Then, using scar multiplication (denoted ⊙) and homomorphic addition (denoted ⊕), \\\\( P_2 \\\\) can compute \\\\( (k_2^{-1} ⊙ c_1 ) ⊕ [( k_2^{-1} \\cdot x_2) ⊙ c_2 ]\\\\), which will be an encryption of \n\n![paillier encryption](/images/two_party_ecdsa/paillier_enc.png)\n\nHowever, proving that each party worked correctly is extremely difficult. For example, the first party must prove that the Paillier encryption includes \\\\(k_1^{-1}\\\\) when the second party only has \\\\(R_1 = k_1 \\cdot G\\\\). it must prove that the Paillier encryptions are to values in the expected range, and more. This can be done, but it results in a protocol that is very expensive.\n\n## their results\n[WIP]\n\n## references\n- [original papger](https://eprint.iacr.org/2017/552.pdf)","source":"_posts/cryptography/two-party-ecdsa.md","raw":"---\ntitle: two party ecdsa\ndate: 2023-02-07 14:29:26\ntags: [cryptography,mpc,ecdsa]\n---\n\n\n\n## overview\nthis post is my reading summary of paper Yehuda Lindell 2017: Fast secure two-party ecdsa signing. the implementation could be found in tss-lib (golang), zengo's library (rust).\n\nUnlike other schemes like RSA, Schnorr signatures and more, it is particularly hard to construct efficient threshold signature protocols for ECDSA as there is an inverse computaion of \\\\( k \\\\).\n\nIn this paper, we consider the specific case of two parties (and thus no honest majority) and construct a protocol that is approximately two orders of magnitude faster than the previous best.\n\n## Comparing ECDSA signing to EC-Schnorr signing\nIn both cases, the public verification key is an elliptic curve point \\\\( Q \\\\) and the private signing key is \\\\( x \\\\) such that \\\\( Q = x \\cdot G \\\\), where \\\\( G \\\\) is the generator point of an EC group of order \\\\( q \\\\).\n![schnorr ecdsa comparison](/images/two_party_ecdsa/schnorr_ecdsa_comparison.png)\n\nObserve that Schnorr signing can be easily distributed since the private key \\\\( x \\\\) and the value k are both used in a linear equation. In contrast, in ECDSA signing, the equation for computing \\\\( s \\\\) includes \\\\( k^{-1} \\\\). Now, given shares \\\\(k_1\\\\), \\\\(k_2\\\\) such that \\\\(k_1 + k_2 = k \\bmod q\\\\) .It is very difficult to compute \\\\(k_1^{\\prime}\\\\), \\\\(k_2^{\\prime}\\\\) such that \\\\(k_1^{\\prime} + k_2^{\\prime} = k^{-1} \\bmod q\\\\)\n\n\ntwo-party protocols for ECDSA signing use multiplicative sharing of \\\\( x \\\\) and of \\\\( k \\\\). That is, the parties hold \\\\(x_1\\\\), \\\\(x_2\\\\) such that \\\\(x_1 \\cdot x_2 = x \\bmod q\\\\), and in each signing operation they generate \\\\(k_1\\\\), \\\\(k_2\\\\) such that \\\\(k_1 \\cdot k_2 = k \\bmod q\\\\). This enables them to easily compute \\\\(k^{-1}\\\\) since each party can locally compute \\\\(k_i^{\\prime} = k_i^{-1} \\bmod q\\\\), and then \\\\(k_1^{\\prime}\\\\), \\\\(k_2^{\\prime}\\\\) are multiplicative shares of \\\\(k^{-1}\\\\). The parties can then use additively homomorphic encryption – specifically Paillier encryption – in order to combine their equations. For example, \\\\(P_1\\\\) can compute \\\\(c_1 = Enc_{pk}(k_1^{-1} \\cdot H(m))\\\\) and \\\\(c_2 = Enc_{pk}(k_1^{-1} \\cdot x_1 \\cdot r)\\\\) . Then, using scar multiplication (denoted ⊙) and homomorphic addition (denoted ⊕), \\\\( P_2 \\\\) can compute \\\\( (k_2^{-1} ⊙ c_1 ) ⊕ [( k_2^{-1} \\cdot x_2) ⊙ c_2 ]\\\\), which will be an encryption of \n\n![paillier encryption](/images/two_party_ecdsa/paillier_enc.png)\n\nHowever, proving that each party worked correctly is extremely difficult. For example, the first party must prove that the Paillier encryption includes \\\\(k_1^{-1}\\\\) when the second party only has \\\\(R_1 = k_1 \\cdot G\\\\). it must prove that the Paillier encryptions are to values in the expected range, and more. This can be done, but it results in a protocol that is very expensive.\n\n## their results\n[WIP]\n\n## references\n- [original papger](https://eprint.iacr.org/2017/552.pdf)","slug":"cryptography/two-party-ecdsa","published":1,"updated":"2023-11-05T04:21:17.034Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8dq000pqwsjfar5aa4q","content":"\n\n\n

overview

this post is my reading summary of paper Yehuda Lindell 2017: Fast secure two-party ecdsa signing. the implementation could be found in tss-lib (golang), zengo’s library (rust).

\n

Unlike other schemes like RSA, Schnorr signatures and more, it is particularly hard to construct efficient threshold signature protocols for ECDSA as there is an inverse computaion of \\( k \\).

\n

In this paper, we consider the specific case of two parties (and thus no honest majority) and construct a protocol that is approximately two orders of magnitude faster than the previous best.

\n

Comparing ECDSA signing to EC-Schnorr signing

In both cases, the public verification key is an elliptic curve point \\( Q \\) and the private signing key is \\( x \\) such that \\( Q = x \\cdot G \\), where \\( G \\) is the generator point of an EC group of order \\( q \\).
\"schnorr

\n

Observe that Schnorr signing can be easily distributed since the private key \\( x \\) and the value k are both used in a linear equation. In contrast, in ECDSA signing, the equation for computing \\( s \\) includes \\( k^{-1} \\). Now, given shares \\(k_1\\), \\(k_2\\) such that \\(k_1 + k_2 = k \\bmod q\\) .It is very difficult to compute \\(k_1^{\\prime}\\), \\(k_2^{\\prime}\\) such that \\(k_1^{\\prime} + k_2^{\\prime} = k^{-1} \\bmod q\\)

\n

two-party protocols for ECDSA signing use multiplicative sharing of \\( x \\) and of \\( k \\). That is, the parties hold \\(x_1\\), \\(x_2\\) such that \\(x_1 \\cdot x_2 = x \\bmod q\\), and in each signing operation they generate \\(k_1\\), \\(k_2\\) such that \\(k_1 \\cdot k_2 = k \\bmod q\\). This enables them to easily compute \\(k^{-1}\\) since each party can locally compute \\(k_i^{\\prime} = k_i^{-1} \\bmod q\\), and then \\(k_1^{\\prime}\\), \\(k_2^{\\prime}\\) are multiplicative shares of \\(k^{-1}\\). The parties can then use additively homomorphic encryption – specifically Paillier encryption – in order to combine their equations. For example, \\(P_1\\) can compute \\(c_1 = Enc_{pk}(k_1^{-1} \\cdot H(m))\\) and \\(c_2 = Enc_{pk}(k_1^{-1} \\cdot x_1 \\cdot r)\\) . Then, using scar multiplication (denoted ⊙) and homomorphic addition (denoted ⊕), \\( P_2 \\) can compute \\( (k_2^{-1} ⊙ c_1 ) ⊕ [( k_2^{-1} \\cdot x_2) ⊙ c_2 ]\\), which will be an encryption of

\n

\"paillier

\n

However, proving that each party worked correctly is extremely difficult. For example, the first party must prove that the Paillier encryption includes \\(k_1^{-1}\\) when the second party only has \\(R_1 = k_1 \\cdot G\\). it must prove that the Paillier encryptions are to values in the expected range, and more. This can be done, but it results in a protocol that is very expensive.

\n

their results

[WIP]

\n

references

\n","site":{"data":{}},"excerpt":"","more":"\n\n\n

overview

this post is my reading summary of paper Yehuda Lindell 2017: Fast secure two-party ecdsa signing. the implementation could be found in tss-lib (golang), zengo’s library (rust).

\n

Unlike other schemes like RSA, Schnorr signatures and more, it is particularly hard to construct efficient threshold signature protocols for ECDSA as there is an inverse computaion of \\( k \\).

\n

In this paper, we consider the specific case of two parties (and thus no honest majority) and construct a protocol that is approximately two orders of magnitude faster than the previous best.

\n

Comparing ECDSA signing to EC-Schnorr signing

In both cases, the public verification key is an elliptic curve point \\( Q \\) and the private signing key is \\( x \\) such that \\( Q = x \\cdot G \\), where \\( G \\) is the generator point of an EC group of order \\( q \\).
\"schnorr

\n

Observe that Schnorr signing can be easily distributed since the private key \\( x \\) and the value k are both used in a linear equation. In contrast, in ECDSA signing, the equation for computing \\( s \\) includes \\( k^{-1} \\). Now, given shares \\(k_1\\), \\(k_2\\) such that \\(k_1 + k_2 = k \\bmod q\\) .It is very difficult to compute \\(k_1^{\\prime}\\), \\(k_2^{\\prime}\\) such that \\(k_1^{\\prime} + k_2^{\\prime} = k^{-1} \\bmod q\\)

\n

two-party protocols for ECDSA signing use multiplicative sharing of \\( x \\) and of \\( k \\). That is, the parties hold \\(x_1\\), \\(x_2\\) such that \\(x_1 \\cdot x_2 = x \\bmod q\\), and in each signing operation they generate \\(k_1\\), \\(k_2\\) such that \\(k_1 \\cdot k_2 = k \\bmod q\\). This enables them to easily compute \\(k^{-1}\\) since each party can locally compute \\(k_i^{\\prime} = k_i^{-1} \\bmod q\\), and then \\(k_1^{\\prime}\\), \\(k_2^{\\prime}\\) are multiplicative shares of \\(k^{-1}\\). The parties can then use additively homomorphic encryption – specifically Paillier encryption – in order to combine their equations. For example, \\(P_1\\) can compute \\(c_1 = Enc_{pk}(k_1^{-1} \\cdot H(m))\\) and \\(c_2 = Enc_{pk}(k_1^{-1} \\cdot x_1 \\cdot r)\\) . Then, using scar multiplication (denoted ⊙) and homomorphic addition (denoted ⊕), \\( P_2 \\) can compute \\( (k_2^{-1} ⊙ c_1 ) ⊕ [( k_2^{-1} \\cdot x_2) ⊙ c_2 ]\\), which will be an encryption of

\n

\"paillier

\n

However, proving that each party worked correctly is extremely difficult. For example, the first party must prove that the Paillier encryption includes \\(k_1^{-1}\\) when the second party only has \\(R_1 = k_1 \\cdot G\\). it must prove that the Paillier encryptions are to values in the expected range, and more. This can be done, but it results in a protocol that is very expensive.

\n

their results

[WIP]

\n

references

\n"},{"title":"go reflect","date":"2023-03-02T02:44:50.000Z","_content":"\n## introduction\nReflection is the ability of a program to examine its own structure, particularly through types. \n\n## type and interfaces\nGo is statically typed. Every variable has a static type, Exactly one type known and fixed at compile time. If we declare\n```go\ntype MyInt int\n\nvar i int\nvar j MyInt\n```\n**The variables i and j have distinct static types and, although they have the same underlying type, they cannot be assigned to one another without a conversion.**\n\nOne important category of type is interface types, which represent fixed sets of methods. An interface variable can store any concrete (non-interface) value as long as that value implements the interface’s methods. An extremely important example of an interface type is the empty interface:\n```go\ninterface{}\n```\nor its equivalent alias,\n```go\nany\n```\nIt represents the empty set of methods and is satisfied by any value at all, since every value has zero or more methods.\na variable of interface type always has the same static type, and even though at run time the value stored in the interface variable may change type, that value will always satisfy the interface.\n\n## the representation of an interface\nA variable of interface type stores a pair: the concrete value assigned to the variable, and that value’s type descriptor.\nFor instance, after\n```golang\nvar r io.Reader\ntty, err := os.OpenFile(\"/dev/tty\", os.O_RDWR, 0)\nif err != nil {\n return nil, err\n}\nr = tty\n```\nr contains, schematically, the (value, type) pair, (tty, *os.File). Notice that the type *os.File implements methods other than Read; even though the interface value provides access only to the Read method, the value inside carries all the type information about that value. That’s why we can do things like this:\n```go\nvar w io.Writer\nw = r.(io.Writer)\n```\nThe expression in this assignment is a type assertion; what it asserts is that the item inside r also implements io.Writer, and so we can assign it to w. After the assignment, w will contain the pair (tty, *os.File). That’s the same pair as was held in r. \nOne important detail is that the pair inside an interface variable always has the form (value, concrete type) and cannot have the form (value, interface type). Interfaces do not hold interface values.\n\n## the first law of reflection\n1. Reflection goes from interface value to reflection object\nAt the basic level, reflection is just a mechanism to examine the type and value pair stored inside an interface variable. `reflect.TypeOf` and `reflect.ValueOf`, retrieve `reflect.Type` and `reflect.Value` pieces out of an interface value.\n```go\nvar x float64 = 3.4\nfmt.Println(\"type:\", reflect.TypeOf(x))\n```\n```\ntype: float64\n```\n```go\nvar x float64 = 3.4\nv := reflect.ValueOf(x)\nfmt.Println(\"type:\", v.Type())\nfmt.Println(\"kind is float64:\", v.Kind() == reflect.Float64)\nfmt.Println(\"value:\", v.Float())\n```\n```\ntype: float64\nkind is float64: true\nvalue: 3.4\n```\nThere are also methods like SetInt and SetFloat. **to keep the API simple, the “getter” and “setter” methods of Value operate on the largest type that can hold the value**: int64 for all the signed integers, for instance. \n```go\nvar x uint8 = 'x'\nv := reflect.ValueOf(x)\nfmt.Println(\"type:\", v.Type()) // uint8.\nfmt.Println(\"kind is uint8: \", v.Kind() == reflect.Uint8) // true.\nx = uint8(v.Uint()) // v.Uint returns a uint64.\n```\n\n2. Reflection goes from reflection object to interface value.\nGiven a reflect.Value we can recover an interface value using the Interface method;\n```go\n// Interface returns v's current value as an interface{}.\n// It is equivalent to:\n//\n//\tvar i interface{} = (v's underlying value)\nfunc (v Value) Interface() interface{}\n```\n```go\ny := v.Interface().(float64) // y will have type float64.\nfmt.Println(y)\n```\n\n3. To modify a reflection object, the value must be settable.\nThe CanSet method of Value reports the settability of a Value; in our case,\n```go\nvar x float64 = 3.4\nv := reflect.ValueOf(x)\nfmt.Println(\"settability of v:\", v.CanSet())\n```\n```\nsettability of v: false\n```\nwe pass a copy of x to reflect.ValueOf, so the interface value created as the argument to reflect.ValueOf is a copy of x, not x itself. Thus, if the statement `v.SetFloat(7.1)` were allowed to succeed, it would not update x, even though v looks like it was created from x. Instead, it would update the copy of x stored inside the reflection value and x itself would be unaffected. That would be confusing and useless, so it is illegal, and settability is the property used to avoid this issue. If we want to modify x by reflection, we must give the reflection library a pointer to the value we want to modify.\nLet’s do that. \n```go\nvar x float64 = 3.4\np := reflect.ValueOf(&x) // Note: take the address of x.\nfmt.Println(\"type of p:\", p.Type())\nfmt.Println(\"settability of p:\", p.CanSet())\n```\nThe reflection object p isn’t settable, but it’s not p we want to set, it’s (in effect) *p. To get to what p points to, we call the Elem method of Value, which indirects through the pointer, and save the result in a reflection Value called v:\n```go\nv := p.Elem()\nfmt.Println(\"settability of v:\", v.CanSet())\n```\n```\nsettability of v: true\n```\n\n## structs\n\n```go\ntype T struct {\n A int\n B string\n}\nt := T{23, \"skidoo\"}\ns := reflect.ValueOf(&t).Elem()\ntypeOfT := s.Type()\nfor i := 0; i < s.NumField(); i++ {\n f := s.Field(i)\n fmt.Printf(\"%d: %s %s = %v\\n\", i,\n typeOfT.Field(i).Name, f.Type(), f.Interface())\n}\n```\n```\n0: A int = 23\n1: B string = skidoo\n```\nThere’s one more point about settability introduced in passing here: the field names of T are upper case (exported) because only exported fields of a struct are settable.\nBecause s contains a settable reflection object, we can modify the fields of the structure.\n```\ns.Field(0).SetInt(77)\ns.Field(1).SetString(\"Sunset Strip\")\nfmt.Println(\"t is now\", t)\n```\nIf we modified the program so that s was created from t, not &t, the calls to SetInt and SetString would fail as the fields of t would not be settable.\n\n## references\n- [official blog](https://go.dev/blog/laws-of-reflection)\n- [go data structure: interface](https://research.swtch.com/interfaces)","source":"_posts/golang/go-reflect.md","raw":"---\ntitle: go reflect\ndate: 2023-03-02 10:44:50\ntags: [golang]\n---\n\n## introduction\nReflection is the ability of a program to examine its own structure, particularly through types. \n\n## type and interfaces\nGo is statically typed. Every variable has a static type, Exactly one type known and fixed at compile time. If we declare\n```go\ntype MyInt int\n\nvar i int\nvar j MyInt\n```\n**The variables i and j have distinct static types and, although they have the same underlying type, they cannot be assigned to one another without a conversion.**\n\nOne important category of type is interface types, which represent fixed sets of methods. An interface variable can store any concrete (non-interface) value as long as that value implements the interface’s methods. An extremely important example of an interface type is the empty interface:\n```go\ninterface{}\n```\nor its equivalent alias,\n```go\nany\n```\nIt represents the empty set of methods and is satisfied by any value at all, since every value has zero or more methods.\na variable of interface type always has the same static type, and even though at run time the value stored in the interface variable may change type, that value will always satisfy the interface.\n\n## the representation of an interface\nA variable of interface type stores a pair: the concrete value assigned to the variable, and that value’s type descriptor.\nFor instance, after\n```golang\nvar r io.Reader\ntty, err := os.OpenFile(\"/dev/tty\", os.O_RDWR, 0)\nif err != nil {\n return nil, err\n}\nr = tty\n```\nr contains, schematically, the (value, type) pair, (tty, *os.File). Notice that the type *os.File implements methods other than Read; even though the interface value provides access only to the Read method, the value inside carries all the type information about that value. That’s why we can do things like this:\n```go\nvar w io.Writer\nw = r.(io.Writer)\n```\nThe expression in this assignment is a type assertion; what it asserts is that the item inside r also implements io.Writer, and so we can assign it to w. After the assignment, w will contain the pair (tty, *os.File). That’s the same pair as was held in r. \nOne important detail is that the pair inside an interface variable always has the form (value, concrete type) and cannot have the form (value, interface type). Interfaces do not hold interface values.\n\n## the first law of reflection\n1. Reflection goes from interface value to reflection object\nAt the basic level, reflection is just a mechanism to examine the type and value pair stored inside an interface variable. `reflect.TypeOf` and `reflect.ValueOf`, retrieve `reflect.Type` and `reflect.Value` pieces out of an interface value.\n```go\nvar x float64 = 3.4\nfmt.Println(\"type:\", reflect.TypeOf(x))\n```\n```\ntype: float64\n```\n```go\nvar x float64 = 3.4\nv := reflect.ValueOf(x)\nfmt.Println(\"type:\", v.Type())\nfmt.Println(\"kind is float64:\", v.Kind() == reflect.Float64)\nfmt.Println(\"value:\", v.Float())\n```\n```\ntype: float64\nkind is float64: true\nvalue: 3.4\n```\nThere are also methods like SetInt and SetFloat. **to keep the API simple, the “getter” and “setter” methods of Value operate on the largest type that can hold the value**: int64 for all the signed integers, for instance. \n```go\nvar x uint8 = 'x'\nv := reflect.ValueOf(x)\nfmt.Println(\"type:\", v.Type()) // uint8.\nfmt.Println(\"kind is uint8: \", v.Kind() == reflect.Uint8) // true.\nx = uint8(v.Uint()) // v.Uint returns a uint64.\n```\n\n2. Reflection goes from reflection object to interface value.\nGiven a reflect.Value we can recover an interface value using the Interface method;\n```go\n// Interface returns v's current value as an interface{}.\n// It is equivalent to:\n//\n//\tvar i interface{} = (v's underlying value)\nfunc (v Value) Interface() interface{}\n```\n```go\ny := v.Interface().(float64) // y will have type float64.\nfmt.Println(y)\n```\n\n3. To modify a reflection object, the value must be settable.\nThe CanSet method of Value reports the settability of a Value; in our case,\n```go\nvar x float64 = 3.4\nv := reflect.ValueOf(x)\nfmt.Println(\"settability of v:\", v.CanSet())\n```\n```\nsettability of v: false\n```\nwe pass a copy of x to reflect.ValueOf, so the interface value created as the argument to reflect.ValueOf is a copy of x, not x itself. Thus, if the statement `v.SetFloat(7.1)` were allowed to succeed, it would not update x, even though v looks like it was created from x. Instead, it would update the copy of x stored inside the reflection value and x itself would be unaffected. That would be confusing and useless, so it is illegal, and settability is the property used to avoid this issue. If we want to modify x by reflection, we must give the reflection library a pointer to the value we want to modify.\nLet’s do that. \n```go\nvar x float64 = 3.4\np := reflect.ValueOf(&x) // Note: take the address of x.\nfmt.Println(\"type of p:\", p.Type())\nfmt.Println(\"settability of p:\", p.CanSet())\n```\nThe reflection object p isn’t settable, but it’s not p we want to set, it’s (in effect) *p. To get to what p points to, we call the Elem method of Value, which indirects through the pointer, and save the result in a reflection Value called v:\n```go\nv := p.Elem()\nfmt.Println(\"settability of v:\", v.CanSet())\n```\n```\nsettability of v: true\n```\n\n## structs\n\n```go\ntype T struct {\n A int\n B string\n}\nt := T{23, \"skidoo\"}\ns := reflect.ValueOf(&t).Elem()\ntypeOfT := s.Type()\nfor i := 0; i < s.NumField(); i++ {\n f := s.Field(i)\n fmt.Printf(\"%d: %s %s = %v\\n\", i,\n typeOfT.Field(i).Name, f.Type(), f.Interface())\n}\n```\n```\n0: A int = 23\n1: B string = skidoo\n```\nThere’s one more point about settability introduced in passing here: the field names of T are upper case (exported) because only exported fields of a struct are settable.\nBecause s contains a settable reflection object, we can modify the fields of the structure.\n```\ns.Field(0).SetInt(77)\ns.Field(1).SetString(\"Sunset Strip\")\nfmt.Println(\"t is now\", t)\n```\nIf we modified the program so that s was created from t, not &t, the calls to SetInt and SetString would fail as the fields of t would not be settable.\n\n## references\n- [official blog](https://go.dev/blog/laws-of-reflection)\n- [go data structure: interface](https://research.swtch.com/interfaces)","slug":"golang/go-reflect","published":1,"updated":"2023-11-05T04:21:13.733Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8dq000rqwsj5r3jepey","content":"

introduction

Reflection is the ability of a program to examine its own structure, particularly through types.

\n

type and interfaces

Go is statically typed. Every variable has a static type, Exactly one type known and fixed at compile time. If we declare

\n
1
2
3
4
type MyInt int

var i int
var j MyInt
\n

The variables i and j have distinct static types and, although they have the same underlying type, they cannot be assigned to one another without a conversion.

\n

One important category of type is interface types, which represent fixed sets of methods. An interface variable can store any concrete (non-interface) value as long as that value implements the interface’s methods. An extremely important example of an interface type is the empty interface:

\n
1
interface{}
\n

or its equivalent alias,

\n
1
any
\n

It represents the empty set of methods and is satisfied by any value at all, since every value has zero or more methods.
a variable of interface type always has the same static type, and even though at run time the value stored in the interface variable may change type, that value will always satisfy the interface.

\n

the representation of an interface

A variable of interface type stores a pair: the concrete value assigned to the variable, and that value’s type descriptor.
For instance, after

\n
1
2
3
4
5
6
var r io.Reader
tty, err := os.OpenFile("/dev/tty", os.O_RDWR, 0)
if err != nil {
    return nil, err
}
r = tty
\n

r contains, schematically, the (value, type) pair, (tty, *os.File). Notice that the type *os.File implements methods other than Read; even though the interface value provides access only to the Read method, the value inside carries all the type information about that value. That’s why we can do things like this:

\n
1
2
var w io.Writer
w = r.(io.Writer)
\n

The expression in this assignment is a type assertion; what it asserts is that the item inside r also implements io.Writer, and so we can assign it to w. After the assignment, w will contain the pair (tty, *os.File). That’s the same pair as was held in r.
One important detail is that the pair inside an interface variable always has the form (value, concrete type) and cannot have the form (value, interface type). Interfaces do not hold interface values.

\n

the first law of reflection

    \n

  1. Reflection goes from interface value to reflection object
    At the basic level, reflection is just a mechanism to examine the type and value pair stored inside an interface variable. reflect.TypeOf and reflect.ValueOf, retrieve reflect.Type and reflect.Value pieces out of an interface value.

    \n
    1
    2
    var x float64 = 3.4
    fmt.Println("type:", reflect.TypeOf(x))
    \n
    1
    type: float64
    \n
    1
    2
    3
    4
    5
    var x float64 = 3.4
    v := reflect.ValueOf(x)
    fmt.Println("type:", v.Type())
    fmt.Println("kind is float64:", v.Kind() == reflect.Float64)
    fmt.Println("value:", v.Float())
    \n
    1
    2
    3
    type: float64
    kind is float64: true
    value: 3.4
    \n

    There are also methods like SetInt and SetFloat. to keep the API simple, the “getter” and “setter” methods of Value operate on the largest type that can hold the value: int64 for all the signed integers, for instance.

    \n
    1
    2
    3
    4
    5
    var x uint8 = 'x'
    v := reflect.ValueOf(x)
    fmt.Println("type:", v.Type())                            // uint8.
    fmt.Println("kind is uint8: ", v.Kind() == reflect.Uint8) // true.
    x = uint8(v.Uint())                                       // v.Uint returns a uint64.
    \n
    2. \n

  2. Reflection goes from reflection object to interface value.
    Given a reflect.Value we can recover an interface value using the Interface method;

    \n
    1
    2
    3
    4
    5
    // Interface returns v's current value as an interface{}.
    // It is equivalent to:
    //
    //\tvar i interface{} = (v's underlying value)
    func (v Value) Interface() interface{}
    \n
    1
    2
    y := v.Interface().(float64) // y will have type float64.
    fmt.Println(y)
    \n
    3. \n

  3. To modify a reflection object, the value must be settable.
    The CanSet method of Value reports the settability of a Value; in our case,

    \n
    1
    2
    3
    var x float64 = 3.4
    v := reflect.ValueOf(x)
    fmt.Println("settability of v:", v.CanSet())
    \n
    1
    settability of v: false
    \n

    we pass a copy of x to reflect.ValueOf, so the interface value created as the argument to reflect.ValueOf is a copy of x, not x itself. Thus, if the statement v.SetFloat(7.1) were allowed to succeed, it would not update x, even though v looks like it was created from x. Instead, it would update the copy of x stored inside the reflection value and x itself would be unaffected. That would be confusing and useless, so it is illegal, and settability is the property used to avoid this issue. If we want to modify x by reflection, we must give the reflection library a pointer to the value we want to modify.
    Let’s do that.

    \n
    1
    2
    3
    4
    var x float64 = 3.4
    p := reflect.ValueOf(&x) // Note: take the address of x.
    fmt.Println("type of p:", p.Type())
    fmt.Println("settability of p:", p.CanSet())
    \n

    The reflection object p isn’t settable, but it’s not p we want to set, it’s (in effect) *p. To get to what p points to, we call the Elem method of Value, which indirects through the pointer, and save the result in a reflection Value called v:

    \n
    1
    2
    v := p.Elem()
    fmt.Println("settability of v:", v.CanSet())
    \n
    1
    settability of v: true
    4. \n
\n

structs

1
2
3
4
5
6
7
8
9
10
11
12
type T struct {
    A int
    B string
}
t := T{23, "skidoo"}
s := reflect.ValueOf(&t).Elem()
typeOfT := s.Type()
for i := 0; i < s.NumField(); i++ {
    f := s.Field(i)
    fmt.Printf("%d: %s %s = %v\\n", i,
        typeOfT.Field(i).Name, f.Type(), f.Interface())
}
\n
1
2
0: A int = 23
1: B string = skidoo
\n

There’s one more point about settability introduced in passing here: the field names of T are upper case (exported) because only exported fields of a struct are settable.
Because s contains a settable reflection object, we can modify the fields of the structure.

\n
1
2
3
s.Field(0).SetInt(77)
s.Field(1).SetString("Sunset Strip")
fmt.Println("t is now", t)
\n

If we modified the program so that s was created from t, not &t, the calls to SetInt and SetString would fail as the fields of t would not be settable.

\n

references

\n","site":{"data":{}},"excerpt":"","more":"

introduction

Reflection is the ability of a program to examine its own structure, particularly through types.

\n

type and interfaces

Go is statically typed. Every variable has a static type, Exactly one type known and fixed at compile time. If we declare

\n
1
2
3
4
type MyInt int

var i int
var j MyInt
\n

The variables i and j have distinct static types and, although they have the same underlying type, they cannot be assigned to one another without a conversion.

\n

One important category of type is interface types, which represent fixed sets of methods. An interface variable can store any concrete (non-interface) value as long as that value implements the interface’s methods. An extremely important example of an interface type is the empty interface:

\n
1
interface{}
\n

or its equivalent alias,

\n
1
any
\n

It represents the empty set of methods and is satisfied by any value at all, since every value has zero or more methods.
a variable of interface type always has the same static type, and even though at run time the value stored in the interface variable may change type, that value will always satisfy the interface.

\n

the representation of an interface

A variable of interface type stores a pair: the concrete value assigned to the variable, and that value’s type descriptor.
For instance, after

\n
1
2
3
4
5
6
var r io.Reader
tty, err := os.OpenFile("/dev/tty", os.O_RDWR, 0)
if err != nil {
    return nil, err
}
r = tty
\n

r contains, schematically, the (value, type) pair, (tty, *os.File). Notice that the type *os.File implements methods other than Read; even though the interface value provides access only to the Read method, the value inside carries all the type information about that value. That’s why we can do things like this:

\n
1
2
var w io.Writer
w = r.(io.Writer)
\n

The expression in this assignment is a type assertion; what it asserts is that the item inside r also implements io.Writer, and so we can assign it to w. After the assignment, w will contain the pair (tty, *os.File). That’s the same pair as was held in r.
One important detail is that the pair inside an interface variable always has the form (value, concrete type) and cannot have the form (value, interface type). Interfaces do not hold interface values.

\n

the first law of reflection

    \n

  1. Reflection goes from interface value to reflection object
    At the basic level, reflection is just a mechanism to examine the type and value pair stored inside an interface variable. reflect.TypeOf and reflect.ValueOf, retrieve reflect.Type and reflect.Value pieces out of an interface value.

    \n
    1
    2
    var x float64 = 3.4
    fmt.Println("type:", reflect.TypeOf(x))
    \n
    1
    type: float64
    \n
    1
    2
    3
    4
    5
    var x float64 = 3.4
    v := reflect.ValueOf(x)
    fmt.Println("type:", v.Type())
    fmt.Println("kind is float64:", v.Kind() == reflect.Float64)
    fmt.Println("value:", v.Float())
    \n
    1
    2
    3
    type: float64
    kind is float64: true
    value: 3.4
    \n

    There are also methods like SetInt and SetFloat. to keep the API simple, the “getter” and “setter” methods of Value operate on the largest type that can hold the value: int64 for all the signed integers, for instance.

    \n
    1
    2
    3
    4
    5
    var x uint8 = 'x'
    v := reflect.ValueOf(x)
    fmt.Println("type:", v.Type())                            // uint8.
    fmt.Println("kind is uint8: ", v.Kind() == reflect.Uint8) // true.
    x = uint8(v.Uint())                                       // v.Uint returns a uint64.
    \n
    2. \n

  2. Reflection goes from reflection object to interface value.
    Given a reflect.Value we can recover an interface value using the Interface method;

    \n
    1
    2
    3
    4
    5
    // Interface returns v's current value as an interface{}.
    // It is equivalent to:
    //
    //\tvar i interface{} = (v's underlying value)
    func (v Value) Interface() interface{}
    \n
    1
    2
    y := v.Interface().(float64) // y will have type float64.
    fmt.Println(y)
    \n
    3. \n

  3. To modify a reflection object, the value must be settable.
    The CanSet method of Value reports the settability of a Value; in our case,

    \n
    1
    2
    3
    var x float64 = 3.4
    v := reflect.ValueOf(x)
    fmt.Println("settability of v:", v.CanSet())
    \n
    1
    settability of v: false
    \n

    we pass a copy of x to reflect.ValueOf, so the interface value created as the argument to reflect.ValueOf is a copy of x, not x itself. Thus, if the statement v.SetFloat(7.1) were allowed to succeed, it would not update x, even though v looks like it was created from x. Instead, it would update the copy of x stored inside the reflection value and x itself would be unaffected. That would be confusing and useless, so it is illegal, and settability is the property used to avoid this issue. If we want to modify x by reflection, we must give the reflection library a pointer to the value we want to modify.
    Let’s do that.

    \n
    1
    2
    3
    4
    var x float64 = 3.4
    p := reflect.ValueOf(&x) // Note: take the address of x.
    fmt.Println("type of p:", p.Type())
    fmt.Println("settability of p:", p.CanSet())
    \n

    The reflection object p isn’t settable, but it’s not p we want to set, it’s (in effect) *p. To get to what p points to, we call the Elem method of Value, which indirects through the pointer, and save the result in a reflection Value called v:

    \n
    1
    2
    v := p.Elem()
    fmt.Println("settability of v:", v.CanSet())
    \n
    1
    settability of v: true
    4. \n
\n

structs

1
2
3
4
5
6
7
8
9
10
11
12
type T struct {
    A int
    B string
}
t := T{23, "skidoo"}
s := reflect.ValueOf(&t).Elem()
typeOfT := s.Type()
for i := 0; i < s.NumField(); i++ {
    f := s.Field(i)
    fmt.Printf("%d: %s %s = %v\\n", i,
        typeOfT.Field(i).Name, f.Type(), f.Interface())
}
\n
1
2
0: A int = 23
1: B string = skidoo
\n

There’s one more point about settability introduced in passing here: the field names of T are upper case (exported) because only exported fields of a struct are settable.
Because s contains a settable reflection object, we can modify the fields of the structure.

\n
1
2
3
s.Field(0).SetInt(77)
s.Field(1).SetString("Sunset Strip")
fmt.Println("t is now", t)
\n

If we modified the program so that s was created from t, not &t, the calls to SetInt and SetString would fail as the fields of t would not be settable.

\n

references

\n"},{"title":"go similar concepts comparison","date":"2023-03-05T02:44:50.000Z","_content":"\n## struct{} & struct{}{}\n`struct` is a keyword in Go. It is used to define struct types, which is a sequence of named elements.\n\nFor example:\n```go\ntype Person struct {\n Name string\n Age int\n}\n```\nThe `struct{}` is a struct type with zero elements. It is often used when no information is to be stored. It has the benefit of being 0-sized, so usually no memory is required to store a value of type `struct{}`.\n\n`struct{}{}` on the other hand is a [composite literal](https://go.dev/ref/spec#Composite_literals), it constructs a value of type `struct{}`. A composite literal constructs values for types such as `structs`, `arrays`, `maps` and `slices`. Its syntax is the type followed by the elements in braces. Since the \"empty\" struct (struct{}) has no fields, the elements list is also empty:\n\n struct{} {}\n| ^ | ^\n type empty element list\nAs an example let's create a \"set\" in Go. Go does not have a builtin set data structure, but it has a builtin map. We can use a map as a set, as a map can only have at most one entry with a given key. And since we want to only store keys (elements) in the map, we may choose the map value type to be struct{}.\n\nA map with string elements:\n```go\nvar set map[string]struct{}\n// Initialize the set\nset = make(map[string]struct{})\n\n// Add some values to the set:\nset[\"red\"] = struct{}{}\nset[\"blue\"] = struct{}{}\n\n// Check if a value is in the map:\n_, ok := set[\"red\"]\nfmt.Println(\"Is red in the map?\", ok)\n_, ok = set[\"green\"]\nfmt.Println(\"Is green in the map?\", ok)\n```","source":"_posts/golang/go-similar-concepts-comparison.md","raw":"---\ntitle: go similar concepts comparison\ndate: 2023-03-05 10:44:50\ntags: [golang]\n---\n\n## struct{} & struct{}{}\n`struct` is a keyword in Go. It is used to define struct types, which is a sequence of named elements.\n\nFor example:\n```go\ntype Person struct {\n Name string\n Age int\n}\n```\nThe `struct{}` is a struct type with zero elements. It is often used when no information is to be stored. It has the benefit of being 0-sized, so usually no memory is required to store a value of type `struct{}`.\n\n`struct{}{}` on the other hand is a [composite literal](https://go.dev/ref/spec#Composite_literals), it constructs a value of type `struct{}`. A composite literal constructs values for types such as `structs`, `arrays`, `maps` and `slices`. Its syntax is the type followed by the elements in braces. Since the \"empty\" struct (struct{}) has no fields, the elements list is also empty:\n\n struct{} {}\n| ^ | ^\n type empty element list\nAs an example let's create a \"set\" in Go. Go does not have a builtin set data structure, but it has a builtin map. We can use a map as a set, as a map can only have at most one entry with a given key. And since we want to only store keys (elements) in the map, we may choose the map value type to be struct{}.\n\nA map with string elements:\n```go\nvar set map[string]struct{}\n// Initialize the set\nset = make(map[string]struct{})\n\n// Add some values to the set:\nset[\"red\"] = struct{}{}\nset[\"blue\"] = struct{}{}\n\n// Check if a value is in the map:\n_, ok := set[\"red\"]\nfmt.Println(\"Is red in the map?\", ok)\n_, ok = set[\"green\"]\nfmt.Println(\"Is green in the map?\", ok)\n```","slug":"golang/go-similar-concepts-comparison","published":1,"updated":"2023-11-05T04:21:13.733Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8dq000uqwsj46jo8rlt","content":"

struct{} & struct{}{}

struct is a keyword in Go. It is used to define struct types, which is a sequence of named elements.

\n

For example:

\n
1
2
3
4
type Person struct {
    Name string
    Age  int
}
\n

The struct{} is a struct type with zero elements. It is often used when no information is to be stored. It has the benefit of being 0-sized, so usually no memory is required to store a value of type struct{}.

\n

struct{}{} on the other hand is a composite literal, it constructs a value of type struct{}. A composite literal constructs values for types such as structs, arrays, maps and slices. Its syntax is the type followed by the elements in braces. Since the “empty” struct (struct{}) has no fields, the elements list is also empty:

\n

struct{} {}
| ^ | ^
type empty element list
As an example let’s create a “set” in Go. Go does not have a builtin set data structure, but it has a builtin map. We can use a map as a set, as a map can only have at most one entry with a given key. And since we want to only store keys (elements) in the map, we may choose the map value type to be struct{}.

\n

A map with string elements:

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
var set map[string]struct{}
// Initialize the set
set = make(map[string]struct{})

// Add some values to the set:
set["red"] = struct{}{}
set["blue"] = struct{}{}

// Check if a value is in the map:
_, ok := set["red"]
fmt.Println("Is red in the map?", ok)
_, ok = set["green"]
fmt.Println("Is green in the map?", ok)
","site":{"data":{}},"excerpt":"","more":"

struct{} & struct{}{}

struct is a keyword in Go. It is used to define struct types, which is a sequence of named elements.

\n

For example:

\n
1
2
3
4
type Person struct {
    Name string
    Age  int
}
\n

The struct{} is a struct type with zero elements. It is often used when no information is to be stored. It has the benefit of being 0-sized, so usually no memory is required to store a value of type struct{}.

\n

struct{}{} on the other hand is a composite literal, it constructs a value of type struct{}. A composite literal constructs values for types such as structs, arrays, maps and slices. Its syntax is the type followed by the elements in braces. Since the “empty” struct (struct{}) has no fields, the elements list is also empty:

\n

struct{} {}
| ^ | ^
type empty element list
As an example let’s create a “set” in Go. Go does not have a builtin set data structure, but it has a builtin map. We can use a map as a set, as a map can only have at most one entry with a given key. And since we want to only store keys (elements) in the map, we may choose the map value type to be struct{}.

\n

A map with string elements:

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
var set map[string]struct{}
// Initialize the set
set = make(map[string]struct{})

// Add some values to the set:
set["red"] = struct{}{}
set["blue"] = struct{}{}

// Check if a value is in the map:
_, ok := set["red"]
fmt.Println("Is red in the map?", ok)
_, ok = set["green"]
fmt.Println("Is green in the map?", ok)
"},{"title":"rust resource","date":"2022-09-27T14:04:38.000Z","_content":"\n## learning resources\n- [the book](https://doc.rust-lang.org/book/)\n- [cargo book](https://doc.rust-lang.org/cargo/index.html)\n- [the rust performance book](https://nnethercote.github.io/perf-book/title-page.html)\n- [rust design patterns](https://rust-unofficial.github.io/patterns/intro.html)\n- [the rust RFC book](https://rust-lang.github.io/rfcs/)\n- [the rustdoc book](https://doc.rust-lang.org/rustdoc/what-is-rustdoc.html)\n- [rustnomicon](https://doc.rust-lang.org/nomicon/intro.html)\n- [the rust reference](https://doc.rust-lang.org/reference/introduction.html)\n- [rust by example](https://doc.rust-lang.org/rust-by-example/index.html)\n- [async programming in rust](https://rust-lang.github.io/async-book/01_getting_started/01_chapter.html)\n- [rust compiler development guide](https://rustc-dev-guide.rust-lang.org/about-this-guide.html)\n- [the little book of rust macros](https://veykril.github.io/tlborm/)\n\n## video resources\n- [youtube channel](https://www.youtube.com/@jonhoo)\n\n## books\n- [atomics and locks](https://marabos.nl/atomics/)\n- [data structure & algorithm](https://github.com/QMHTMY/RustBook/blob/main/books/rust-book-zh-cn-shieber.pdf)","source":"_posts/rust/rust-01-resource.md","raw":"---\ntitle: rust resource\ndate: 2022-09-27 22:04:38\ntags: [rust]\n---\n\n## learning resources\n- [the book](https://doc.rust-lang.org/book/)\n- [cargo book](https://doc.rust-lang.org/cargo/index.html)\n- [the rust performance book](https://nnethercote.github.io/perf-book/title-page.html)\n- [rust design patterns](https://rust-unofficial.github.io/patterns/intro.html)\n- [the rust RFC book](https://rust-lang.github.io/rfcs/)\n- [the rustdoc book](https://doc.rust-lang.org/rustdoc/what-is-rustdoc.html)\n- [rustnomicon](https://doc.rust-lang.org/nomicon/intro.html)\n- [the rust reference](https://doc.rust-lang.org/reference/introduction.html)\n- [rust by example](https://doc.rust-lang.org/rust-by-example/index.html)\n- [async programming in rust](https://rust-lang.github.io/async-book/01_getting_started/01_chapter.html)\n- [rust compiler development guide](https://rustc-dev-guide.rust-lang.org/about-this-guide.html)\n- [the little book of rust macros](https://veykril.github.io/tlborm/)\n\n## video resources\n- [youtube channel](https://www.youtube.com/@jonhoo)\n\n## books\n- [atomics and locks](https://marabos.nl/atomics/)\n- [data structure & algorithm](https://github.com/QMHTMY/RustBook/blob/main/books/rust-book-zh-cn-shieber.pdf)","slug":"rust/rust-01-resource","published":1,"updated":"2023-11-05T04:21:13.734Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8dr000wqwsjaarwgrmn","content":"

learning resources

\n

video resources

\n

books

\n","site":{"data":{}},"excerpt":"","more":"

learning resources

\n

video resources

\n

books

\n"},{"title":"rust ownership","date":"2022-10-11T09:09:28.000Z","_content":"\n## ownership rule\n- Each value in Rust has an owner.\n- There can only be one owner at a time.\n- When the owner goes out of scope, the value will be dropped.\n\n## variable scope\n```rust\nfn main() {\n { // s is not valid here, it’s not yet declared\n let s = \"hello\"; // s is valid from this point forward\n // do stuff with s\n } // this scope is now over, and s is no longer valid\n}\n```\n\n## Move\n### stack-only data: Copy trait\nsuch as primitive type\n```rust\nfn main() {\n let x = 5;\n let y = x;\n}\n```\nbind the value 5 to x; then make a copy of the value in x and bind it to y.\n\n### for heap variable\n```rust\nfn main() {\n let s1 = String::from(\"hello\");\n let s2 = s1;\n}\n```\n![move-string](/images/rust/ownership/move-string.png)\nptr, len, capacity is stored in stack, while string value is stored in heap \\\nWhen we assign s1 to s2, the String data is copied, meaning we copy the pointer, the length, and the capacity that are on the stack. We do not copy the data on the heap\n![move-string-2](/images/rust/ownership/move-string-2.png)\nsimilar to shallow copy\n\n## Clone\n```rust\nfn main() {\n let s1 = String::from(\"hello\");\n let s2 = s1.clone();\n\n println!(\"s1 = {}, s2 = {}\", s1, s2);\n}\n```\nIf we do want to deeply copy the heap data of the String, not just the stack data, we can use a common method called clone.\n\n### ownership and function\n- Passing a value to a function will result in a move or copy of ownership\n- difference between \"stack\" and \"heap\" variables: stack variables will be copied, and heap variables will be moved. When a variable containing heap data leaves the scope, its value will be cleared by the drop function, unless the ownership of the data is moved to another variable\n```rust\nfn main() {\n let s = String::from(\"hello\"); // s comes into scope\n takes_ownership(s); // s's value moves into the function...\n // ... and so is no longer valid here\n\n let x = 5; // x comes into scope\n\n makes_copy(x); // x would move into the function,\n // but i32 is Copy, so it's okay to still\n // use x afterward\n\n} // Here, x goes out of scope, then s. But because s's value was moved, nothing\n // special happens.\n\nfn takes_ownership(some_string: String) { // some_string comes into scope\n println!(\"{}\", some_string);\n} // Here, some_string goes out of scope and `drop` is called. The backing\n // memory is freed.\n\nfn makes_copy(some_integer: i32) { // some_integer comes into scope\n println!(\"{}\", some_integer);\n} // Here, some_integer goes out of scope. Nothing special happens.\n\n```\n\n### return values and scope\n```rust\nfn main() {\n let s1 = gives_ownership(); // gives_ownership moves its return\n // value into s1\n\n let s2 = String::from(\"hello\"); // s2 comes into scope\n\n let s3 = takes_and_gives_back(s2); // s2 is moved into\n // takes_and_gives_back, which also\n // moves its return value into s3\n} // Here, s3 goes out of scope and is dropped. s2 was moved, so nothing\n // happens. s1 goes out of scope and is dropped.\n\nfn gives_ownership() -> String { // gives_ownership will move its\n // return value into the function\n // that calls it\n\n let some_string = String::from(\"yours\"); // some_string comes into scope\n\n some_string // some_string is returned and\n // moves out to the calling\n // function\n}\n\n// This function takes a String and returns one\nfn takes_and_gives_back(a_string: String) -> String { // a_string comes into\n // scope\n\n a_string // a_string is returned and moves out to the calling function\n}\n```\n> What if we want to let a function use a value but not take ownership? \n> that's reference\n \n\n## reference & borrow\n- & means reference (borrow but not own)， default immutable\n- &mut a mutable reference， only one mutable reference allowed in same scope (avoid data racing)\n- Multiple mutable references can be created non-simultaneously by creating a new scope\n- **Cannot have mutable and immutable references at the same time**\n```rust\nfn main() {\n let mut s = String::from(\"hello\");\n {\n let s1 = &mut s;\n } // r1 goes out of scope here, so we can make a new reference with no problems.\n let s2 = &mut s;\n}\n\nfn main() {\n let mut s = String::from(\"hello\");\n\n let r1 = &s; // no problem\n let r2 = &s; // no problem\n println!(\"{} and {}\", r1, r2);\n // variables r1 and r2 will not be used after this point\n\n let r3 = &mut s; // no problem\n println!(\"{}\", r3);\n\n println!{\"{}\",r1} // got problem with above mutable borrow\n}\n```\n\n### reference as function arguments\n```rust\nfn main() {\n let s1 = String::from(\"hello\");\n\n let len = calculate_length(&s1);\n\n println!(\"The length of '{}' is {}.\", s1, len);\n}\n\nfn calculate_length(s: &String) -> usize { // s is a reference to a String\n s.len() \n} // Here, s goes out of scope. But because it does not have ownership of what\n // it refers to, nothing happens.\n```\nwe pass &s1 into calculate_length and, in its definition, we take &String rather than String. These ampersands represent references, and they allow you to refer to some value without taking ownership of it. Because it does not own it, the value it points to will not be dropped when the reference stops being used. \\\n When functions have references as parameters instead of the actual values, we won't need to return the values in order to give back ownership, because we never had ownership. \\\n We call the action of creating a reference borrowing. \n\n### mutable references\n```rust\nfn main() {\n let mut s = String::from(\"hello\");\n\n change(&mut s);\n}\n\nfn change(some_string: &mut String) {\n some_string.push_str(\", world\");\n}\n```\n\n### dangling references\n- A pointer refers to an address in memory, but the memory may have been freed and allocated for use by someone else\n- rust，The compiler can guarantee that there will never be dangling references\n```rust\nfn main() {\n let r = dangle();\n}\nfn dangle() -> &string { // dangle returns a reference to a String\n let s = String::from(\"hello\"); // s is a new String\n &s // we return a reference to the String, s\n} // Here, s goes out of scope, and is dropped. Its memory goes away.\n// Danger\n```\nThe solution here is to return the String directly:\n```rust\nfn main() {\n let string = no_dangle();\n}\n\nfn no_dangle() -> String {\n let s = String::from(\"hello\");\n s\n}\n```\nThis works without any problems. Ownership is moved out, and nothing is deallocated.\n\n\n## slice\n- Slices let you reference a contiguous sequence of elements in a collection rather than the whole collection. A slice is a kind of reference, so it does not have ownership.\n\n### string slice\n```rust\nfn main() {\n let mut s = String::from(\"Hello world\");\n\n let hello = &s[0..5]; \n let world = &s[6..11];\n}\n```\nRather than a reference to the entire String, hello is a reference to a portion of the String, \\\nWith Rust's `..` range syntax, if you want to start at index zero, you can drop the value before the two periods \\\nBy the same token, if your slice includes the last byte of the String, you can drop the trailing number. \n> Note: String slice range indices must occur at **valid UTF-8 character boundaries**. If you attempt to create a string slice in the middle of a multibyte character, your program will exit with an error.\n\n```rust\nfn first_word(s :&String) -> &str {\n let bytes = s.as_bytes();\n for(i, &item) in bytes.iter().enumerate() {\n if item == b' ' {\n return &s[..i];\n }\n }\n &s[..]\n}\n```\n\n### String Literals Are Slices\n```rust\nfn main() {\nlet s = \"Hello, world!\";\n}\n```\nThe type of s here is &str: it's a slice pointing to that specific point of the binary. \n\n### String Slices as Parameters\n- Pass &str as a parameter, you can receive parameters of type &String and &str at the same time\n```rust\nfn first_word(s: &String) -> &str\n```\nequivalent to\n```rust\nfn first_word(s: &str) -> &str\n```\n\n### other slices\narray slice\n```rust\nfn main() {\n let a = [1, 2, 3, 4, 5];\n let slice = &a[1..3];\n assert_eq!(slice, &[2, 3]);\n}\n```","source":"_posts/rust/rust-03-ownership.md","raw":"---\ntitle: rust ownership\ndate: 2022-10-11 17:09:28\ntags: [rust]\n---\n\n## ownership rule\n- Each value in Rust has an owner.\n- There can only be one owner at a time.\n- When the owner goes out of scope, the value will be dropped.\n\n## variable scope\n```rust\nfn main() {\n { // s is not valid here, it’s not yet declared\n let s = \"hello\"; // s is valid from this point forward\n // do stuff with s\n } // this scope is now over, and s is no longer valid\n}\n```\n\n## Move\n### stack-only data: Copy trait\nsuch as primitive type\n```rust\nfn main() {\n let x = 5;\n let y = x;\n}\n```\nbind the value 5 to x; then make a copy of the value in x and bind it to y.\n\n### for heap variable\n```rust\nfn main() {\n let s1 = String::from(\"hello\");\n let s2 = s1;\n}\n```\n![move-string](/images/rust/ownership/move-string.png)\nptr, len, capacity is stored in stack, while string value is stored in heap \\\nWhen we assign s1 to s2, the String data is copied, meaning we copy the pointer, the length, and the capacity that are on the stack. We do not copy the data on the heap\n![move-string-2](/images/rust/ownership/move-string-2.png)\nsimilar to shallow copy\n\n## Clone\n```rust\nfn main() {\n let s1 = String::from(\"hello\");\n let s2 = s1.clone();\n\n println!(\"s1 = {}, s2 = {}\", s1, s2);\n}\n```\nIf we do want to deeply copy the heap data of the String, not just the stack data, we can use a common method called clone.\n\n### ownership and function\n- Passing a value to a function will result in a move or copy of ownership\n- difference between \"stack\" and \"heap\" variables: stack variables will be copied, and heap variables will be moved. When a variable containing heap data leaves the scope, its value will be cleared by the drop function, unless the ownership of the data is moved to another variable\n```rust\nfn main() {\n let s = String::from(\"hello\"); // s comes into scope\n takes_ownership(s); // s's value moves into the function...\n // ... and so is no longer valid here\n\n let x = 5; // x comes into scope\n\n makes_copy(x); // x would move into the function,\n // but i32 is Copy, so it's okay to still\n // use x afterward\n\n} // Here, x goes out of scope, then s. But because s's value was moved, nothing\n // special happens.\n\nfn takes_ownership(some_string: String) { // some_string comes into scope\n println!(\"{}\", some_string);\n} // Here, some_string goes out of scope and `drop` is called. The backing\n // memory is freed.\n\nfn makes_copy(some_integer: i32) { // some_integer comes into scope\n println!(\"{}\", some_integer);\n} // Here, some_integer goes out of scope. Nothing special happens.\n\n```\n\n### return values and scope\n```rust\nfn main() {\n let s1 = gives_ownership(); // gives_ownership moves its return\n // value into s1\n\n let s2 = String::from(\"hello\"); // s2 comes into scope\n\n let s3 = takes_and_gives_back(s2); // s2 is moved into\n // takes_and_gives_back, which also\n // moves its return value into s3\n} // Here, s3 goes out of scope and is dropped. s2 was moved, so nothing\n // happens. s1 goes out of scope and is dropped.\n\nfn gives_ownership() -> String { // gives_ownership will move its\n // return value into the function\n // that calls it\n\n let some_string = String::from(\"yours\"); // some_string comes into scope\n\n some_string // some_string is returned and\n // moves out to the calling\n // function\n}\n\n// This function takes a String and returns one\nfn takes_and_gives_back(a_string: String) -> String { // a_string comes into\n // scope\n\n a_string // a_string is returned and moves out to the calling function\n}\n```\n> What if we want to let a function use a value but not take ownership? \n> that's reference\n \n\n## reference & borrow\n- & means reference (borrow but not own)， default immutable\n- &mut a mutable reference， only one mutable reference allowed in same scope (avoid data racing)\n- Multiple mutable references can be created non-simultaneously by creating a new scope\n- **Cannot have mutable and immutable references at the same time**\n```rust\nfn main() {\n let mut s = String::from(\"hello\");\n {\n let s1 = &mut s;\n } // r1 goes out of scope here, so we can make a new reference with no problems.\n let s2 = &mut s;\n}\n\nfn main() {\n let mut s = String::from(\"hello\");\n\n let r1 = &s; // no problem\n let r2 = &s; // no problem\n println!(\"{} and {}\", r1, r2);\n // variables r1 and r2 will not be used after this point\n\n let r3 = &mut s; // no problem\n println!(\"{}\", r3);\n\n println!{\"{}\",r1} // got problem with above mutable borrow\n}\n```\n\n### reference as function arguments\n```rust\nfn main() {\n let s1 = String::from(\"hello\");\n\n let len = calculate_length(&s1);\n\n println!(\"The length of '{}' is {}.\", s1, len);\n}\n\nfn calculate_length(s: &String) -> usize { // s is a reference to a String\n s.len() \n} // Here, s goes out of scope. But because it does not have ownership of what\n // it refers to, nothing happens.\n```\nwe pass &s1 into calculate_length and, in its definition, we take &String rather than String. These ampersands represent references, and they allow you to refer to some value without taking ownership of it. Because it does not own it, the value it points to will not be dropped when the reference stops being used. \\\n When functions have references as parameters instead of the actual values, we won't need to return the values in order to give back ownership, because we never had ownership. \\\n We call the action of creating a reference borrowing. \n\n### mutable references\n```rust\nfn main() {\n let mut s = String::from(\"hello\");\n\n change(&mut s);\n}\n\nfn change(some_string: &mut String) {\n some_string.push_str(\", world\");\n}\n```\n\n### dangling references\n- A pointer refers to an address in memory, but the memory may have been freed and allocated for use by someone else\n- rust，The compiler can guarantee that there will never be dangling references\n```rust\nfn main() {\n let r = dangle();\n}\nfn dangle() -> &string { // dangle returns a reference to a String\n let s = String::from(\"hello\"); // s is a new String\n &s // we return a reference to the String, s\n} // Here, s goes out of scope, and is dropped. Its memory goes away.\n// Danger\n```\nThe solution here is to return the String directly:\n```rust\nfn main() {\n let string = no_dangle();\n}\n\nfn no_dangle() -> String {\n let s = String::from(\"hello\");\n s\n}\n```\nThis works without any problems. Ownership is moved out, and nothing is deallocated.\n\n\n## slice\n- Slices let you reference a contiguous sequence of elements in a collection rather than the whole collection. A slice is a kind of reference, so it does not have ownership.\n\n### string slice\n```rust\nfn main() {\n let mut s = String::from(\"Hello world\");\n\n let hello = &s[0..5]; \n let world = &s[6..11];\n}\n```\nRather than a reference to the entire String, hello is a reference to a portion of the String, \\\nWith Rust's `..` range syntax, if you want to start at index zero, you can drop the value before the two periods \\\nBy the same token, if your slice includes the last byte of the String, you can drop the trailing number. \n> Note: String slice range indices must occur at **valid UTF-8 character boundaries**. If you attempt to create a string slice in the middle of a multibyte character, your program will exit with an error.\n\n```rust\nfn first_word(s :&String) -> &str {\n let bytes = s.as_bytes();\n for(i, &item) in bytes.iter().enumerate() {\n if item == b' ' {\n return &s[..i];\n }\n }\n &s[..]\n}\n```\n\n### String Literals Are Slices\n```rust\nfn main() {\nlet s = \"Hello, world!\";\n}\n```\nThe type of s here is &str: it's a slice pointing to that specific point of the binary. \n\n### String Slices as Parameters\n- Pass &str as a parameter, you can receive parameters of type &String and &str at the same time\n```rust\nfn first_word(s: &String) -> &str\n```\nequivalent to\n```rust\nfn first_word(s: &str) -> &str\n```\n\n### other slices\narray slice\n```rust\nfn main() {\n let a = [1, 2, 3, 4, 5];\n let slice = &a[1..3];\n assert_eq!(slice, &[2, 3]);\n}\n```","slug":"rust/rust-03-ownership","published":1,"updated":"2023-11-05T04:21:13.734Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8dr000yqwsj41fu3ebh","content":"

ownership rule

    \n
  • Each value in Rust has an owner.
    • \n
  • There can only be one owner at a time.
    • \n
  • When the owner goes out of scope, the value will be dropped.
    • \n
\n

variable scope

1
2
3
4
5
6
fn main() {
    {                      // s is not valid here, it’s not yet declared
        let s = "hello";   // s is valid from this point forward
        // do stuff with s
    }                      // this scope is now over, and s is no longer valid
}
\n\n

Move

stack-only data: Copy trait

such as primitive type

\n
1
2
3
4
fn main() {
    let x = 5;
    let y = x;
}
\n

bind the value 5 to x; then make a copy of the value in x and bind it to y.

\n

for heap variable

1
2
3
4
fn main() {
    let s1 = String::from("hello");
    let s2 = s1;
}
\n

\"move-string\"
ptr, len, capacity is stored in stack, while string value is stored in heap
When we assign s1 to s2, the String data is copied, meaning we copy the pointer, the length, and the capacity that are on the stack. We do not copy the data on the heap
\"move-string-2\"
similar to shallow copy

\n

Clone

1
2
3
4
5
6
fn main() {
    let s1 = String::from("hello");
    let s2 = s1.clone();

    println!("s1 = {}, s2 = {}", s1, s2);
}
\n

If we do want to deeply copy the heap data of the String, not just the stack data, we can use a common method called clone.

\n

ownership and function

    \n
  • Passing a value to a function will result in a move or copy of ownership
    • \n
  • difference between “stack” and “heap” variables: stack variables will be copied, and heap variables will be moved. When a variable containing heap data leaves the scope, its value will be cleared by the drop function, unless the ownership of the data is moved to another variable
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    fn main() {
        let s = String::from("hello");  // s comes into scope
        takes_ownership(s);             // s's value moves into the function...
                                        // ... and so is no longer valid here

        let x = 5;                      // x comes into scope

        makes_copy(x);                  // x would move into the function,
                                        // but i32 is Copy, so it's okay to still
                                        // use x afterward

    } // Here, x goes out of scope, then s. But because s's value was moved, nothing
      // special happens.

    fn takes_ownership(some_string: String) { // some_string comes into scope
        println!("{}", some_string);
    } // Here, some_string goes out of scope and `drop` is called. The backing
      // memory is freed.

    fn makes_copy(some_integer: i32) { // some_integer comes into scope
        println!("{}", some_integer);
    } // Here, some_integer goes out of scope. Nothing special happens.

    • \n
\n

return values and scope

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
fn main() {
    let s1 = gives_ownership();         // gives_ownership moves its return
                                        // value into s1

    let s2 = String::from("hello");     // s2 comes into scope

    let s3 = takes_and_gives_back(s2);  // s2 is moved into
                                        // takes_and_gives_back, which also
                                        // moves its return value into s3
} // Here, s3 goes out of scope and is dropped. s2 was moved, so nothing
  // happens. s1 goes out of scope and is dropped.

fn gives_ownership() -> String {             // gives_ownership will move its
                                             // return value into the function
                                             // that calls it

    let some_string = String::from("yours"); // some_string comes into scope

    some_string                              // some_string is returned and
                                             // moves out to the calling
                                             // function
}

// This function takes a String and returns one
fn takes_and_gives_back(a_string: String) -> String { // a_string comes into
                                                      // scope

    a_string  // a_string is returned and moves out to the calling function
}
\n
\n

What if we want to let a function use a value but not take ownership?
that’s reference

\n
\n

reference & borrow

    \n
  • & means reference (borrow but not own)， default immutable
    • \n
  • &mut a mutable reference， only one mutable reference allowed in same scope (avoid data racing)
    • \n
  • Multiple mutable references can be created non-simultaneously by creating a new scope
    • \n
  • Cannot have mutable and immutable references at the same time
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    fn main() {
        let mut s = String::from("hello");
        {
            let s1 = &mut s;
        } // r1 goes out of scope here, so we can make a new reference with no problems.
        let s2 = &mut s;
    }

    fn main() {
        let mut s = String::from("hello");

        let r1 = &s; // no problem
        let r2 = &s; // no problem
        println!("{} and {}", r1, r2);
        // variables r1 and r2 will not be used after this point

        let r3 = &mut s; // no problem
        println!("{}", r3);

        println!{"{}",r1} // got problem with above mutable borrow
    }
    • \n
\n

reference as function arguments

1
2
3
4
5
6
7
8
9
10
11
12
fn main() {
    let s1 = String::from("hello");

    let len = calculate_length(&s1);

    println!("The length of '{}' is {}.", s1, len);
}

fn calculate_length(s: &String) -> usize { // s is a reference to a String
    s.len() 
}  // Here, s goes out of scope. But because it does not have ownership of what
   // it refers to, nothing happens.
\n

we pass &s1 into calculate_length and, in its definition, we take &String rather than String. These ampersands represent references, and they allow you to refer to some value without taking ownership of it. Because it does not own it, the value it points to will not be dropped when the reference stops being used.
When functions have references as parameters instead of the actual values, we won’t need to return the values in order to give back ownership, because we never had ownership.
We call the action of creating a reference borrowing.

\n

mutable references

1
2
3
4
5
6
7
8
9
fn main() {
    let mut s = String::from("hello");

    change(&mut s);
}

fn change(some_string: &mut String) {
    some_string.push_str(", world");
}
\n\n

dangling references

    \n
  • A pointer refers to an address in memory, but the memory may have been freed and allocated for use by someone else
    • \n
  • rust，The compiler can guarantee that there will never be dangling references
    1
    2
    3
    4
    5
    6
    7
    8
    fn main() {
        let r = dangle();
    }
    fn dangle() -> &string {  // dangle returns a reference to a String
        let s = String::from("hello");  // s is a new String
        &s // we return a reference to the String, s
    } // Here, s goes out of scope, and is dropped. Its memory goes away.
    // Danger
    \nThe solution here is to return the String directly:
    1
    2
    3
    4
    5
    6
    7
    8
    fn main() {
        let string = no_dangle();
    }

    fn no_dangle() -> String {
        let s = String::from("hello");
        s
    }
    \nThis works without any problems. Ownership is moved out, and nothing is deallocated.
    • \n
\n

slice

    \n
  • Slices let you reference a contiguous sequence of elements in a collection rather than the whole collection. A slice is a kind of reference, so it does not have ownership.
    • \n
\n

string slice

1
2
3
4
5
6
fn main() {
    let mut s = String::from("Hello world");

    let hello = &s[0..5]; 
    let world = &s[6..11];
}
\n

Rather than a reference to the entire String, hello is a reference to a portion of the String,
With Rust’s .. range syntax, if you want to start at index zero, you can drop the value before the two periods
By the same token, if your slice includes the last byte of the String, you can drop the trailing number.

\n
\n

Note: String slice range indices must occur at valid UTF-8 character boundaries. If you attempt to create a string slice in the middle of a multibyte character, your program will exit with an error.

\n
\n
1
2
3
4
5
6
7
8
9
fn first_word(s :&String) -> &str {
    let bytes = s.as_bytes();
    for(i, &item) in bytes.iter().enumerate() {
        if item == b' ' {
            return &s[..i];
        }
    }
    &s[..]
}
\n\n

String Literals Are Slices

1
2
3
fn main() {
let s = "Hello, world!";
}
\n

The type of s here is &str: it’s a slice pointing to that specific point of the binary.

\n

String Slices as Parameters

    \n
  • Pass &str as a parameter, you can receive parameters of type &String and &str at the same time
    1
    fn first_word(s: &String) -> &str
    \nequivalent to
    1
    fn first_word(s: &str) -> &str
    • \n
\n

other slices

array slice

\n
1
2
3
4
5
fn main() {
  let a = [1, 2, 3, 4, 5];
  let slice = &a[1..3];
  assert_eq!(slice, &[2, 3]);
}
","site":{"data":{}},"excerpt":"","more":"

ownership rule

    \n
  • Each value in Rust has an owner.
    • \n
  • There can only be one owner at a time.
    • \n
  • When the owner goes out of scope, the value will be dropped.
    • \n
\n

variable scope

1
2
3
4
5
6
fn main() {
    {                      // s is not valid here, it’s not yet declared
        let s = "hello";   // s is valid from this point forward
        // do stuff with s
    }                      // this scope is now over, and s is no longer valid
}
\n\n

Move

stack-only data: Copy trait

such as primitive type

\n
1
2
3
4
fn main() {
    let x = 5;
    let y = x;
}
\n

bind the value 5 to x; then make a copy of the value in x and bind it to y.

\n

for heap variable

1
2
3
4
fn main() {
    let s1 = String::from("hello");
    let s2 = s1;
}
\n

\"move-string\"
ptr, len, capacity is stored in stack, while string value is stored in heap
When we assign s1 to s2, the String data is copied, meaning we copy the pointer, the length, and the capacity that are on the stack. We do not copy the data on the heap
\"move-string-2\"
similar to shallow copy

\n

Clone

1
2
3
4
5
6
fn main() {
    let s1 = String::from("hello");
    let s2 = s1.clone();

    println!("s1 = {}, s2 = {}", s1, s2);
}
\n

If we do want to deeply copy the heap data of the String, not just the stack data, we can use a common method called clone.

\n

ownership and function

    \n
  • Passing a value to a function will result in a move or copy of ownership
    • \n
  • difference between “stack” and “heap” variables: stack variables will be copied, and heap variables will be moved. When a variable containing heap data leaves the scope, its value will be cleared by the drop function, unless the ownership of the data is moved to another variable
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    fn main() {
        let s = String::from("hello");  // s comes into scope
        takes_ownership(s);             // s's value moves into the function...
                                        // ... and so is no longer valid here

        let x = 5;                      // x comes into scope

        makes_copy(x);                  // x would move into the function,
                                        // but i32 is Copy, so it's okay to still
                                        // use x afterward

    } // Here, x goes out of scope, then s. But because s's value was moved, nothing
      // special happens.

    fn takes_ownership(some_string: String) { // some_string comes into scope
        println!("{}", some_string);
    } // Here, some_string goes out of scope and `drop` is called. The backing
      // memory is freed.

    fn makes_copy(some_integer: i32) { // some_integer comes into scope
        println!("{}", some_integer);
    } // Here, some_integer goes out of scope. Nothing special happens.

    • \n
\n

return values and scope

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
fn main() {
    let s1 = gives_ownership();         // gives_ownership moves its return
                                        // value into s1

    let s2 = String::from("hello");     // s2 comes into scope

    let s3 = takes_and_gives_back(s2);  // s2 is moved into
                                        // takes_and_gives_back, which also
                                        // moves its return value into s3
} // Here, s3 goes out of scope and is dropped. s2 was moved, so nothing
  // happens. s1 goes out of scope and is dropped.

fn gives_ownership() -> String {             // gives_ownership will move its
                                             // return value into the function
                                             // that calls it

    let some_string = String::from("yours"); // some_string comes into scope

    some_string                              // some_string is returned and
                                             // moves out to the calling
                                             // function
}

// This function takes a String and returns one
fn takes_and_gives_back(a_string: String) -> String { // a_string comes into
                                                      // scope

    a_string  // a_string is returned and moves out to the calling function
}
\n
\n

What if we want to let a function use a value but not take ownership?
that’s reference

\n
\n

reference & borrow

    \n
  • & means reference (borrow but not own)， default immutable
    • \n
  • &mut a mutable reference， only one mutable reference allowed in same scope (avoid data racing)
    • \n
  • Multiple mutable references can be created non-simultaneously by creating a new scope
    • \n
  • Cannot have mutable and immutable references at the same time
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    fn main() {
        let mut s = String::from("hello");
        {
            let s1 = &mut s;
        } // r1 goes out of scope here, so we can make a new reference with no problems.
        let s2 = &mut s;
    }

    fn main() {
        let mut s = String::from("hello");

        let r1 = &s; // no problem
        let r2 = &s; // no problem
        println!("{} and {}", r1, r2);
        // variables r1 and r2 will not be used after this point

        let r3 = &mut s; // no problem
        println!("{}", r3);

        println!{"{}",r1} // got problem with above mutable borrow
    }
    • \n
\n

reference as function arguments

1
2
3
4
5
6
7
8
9
10
11
12
fn main() {
    let s1 = String::from("hello");

    let len = calculate_length(&s1);

    println!("The length of '{}' is {}.", s1, len);
}

fn calculate_length(s: &String) -> usize { // s is a reference to a String
    s.len() 
}  // Here, s goes out of scope. But because it does not have ownership of what
   // it refers to, nothing happens.
\n

we pass &s1 into calculate_length and, in its definition, we take &String rather than String. These ampersands represent references, and they allow you to refer to some value without taking ownership of it. Because it does not own it, the value it points to will not be dropped when the reference stops being used.
When functions have references as parameters instead of the actual values, we won’t need to return the values in order to give back ownership, because we never had ownership.
We call the action of creating a reference borrowing.

\n

mutable references

1
2
3
4
5
6
7
8
9
fn main() {
    let mut s = String::from("hello");

    change(&mut s);
}

fn change(some_string: &mut String) {
    some_string.push_str(", world");
}
\n\n

dangling references

    \n
  • A pointer refers to an address in memory, but the memory may have been freed and allocated for use by someone else
    • \n
  • rust，The compiler can guarantee that there will never be dangling references
    1
    2
    3
    4
    5
    6
    7
    8
    fn main() {
        let r = dangle();
    }
    fn dangle() -> &string {  // dangle returns a reference to a String
        let s = String::from("hello");  // s is a new String
        &s // we return a reference to the String, s
    } // Here, s goes out of scope, and is dropped. Its memory goes away.
    // Danger
    \nThe solution here is to return the String directly:
    1
    2
    3
    4
    5
    6
    7
    8
    fn main() {
        let string = no_dangle();
    }

    fn no_dangle() -> String {
        let s = String::from("hello");
        s
    }
    \nThis works without any problems. Ownership is moved out, and nothing is deallocated.
    • \n
\n

slice

    \n
  • Slices let you reference a contiguous sequence of elements in a collection rather than the whole collection. A slice is a kind of reference, so it does not have ownership.
    • \n
\n

string slice

1
2
3
4
5
6
fn main() {
    let mut s = String::from("Hello world");

    let hello = &s[0..5]; 
    let world = &s[6..11];
}
\n

Rather than a reference to the entire String, hello is a reference to a portion of the String,
With Rust’s .. range syntax, if you want to start at index zero, you can drop the value before the two periods
By the same token, if your slice includes the last byte of the String, you can drop the trailing number.

\n
\n

Note: String slice range indices must occur at valid UTF-8 character boundaries. If you attempt to create a string slice in the middle of a multibyte character, your program will exit with an error.

\n
\n
1
2
3
4
5
6
7
8
9
fn first_word(s :&String) -> &str {
    let bytes = s.as_bytes();
    for(i, &item) in bytes.iter().enumerate() {
        if item == b' ' {
            return &s[..i];
        }
    }
    &s[..]
}
\n\n

String Literals Are Slices

1
2
3
fn main() {
let s = "Hello, world!";
}
\n

The type of s here is &str: it’s a slice pointing to that specific point of the binary.

\n

String Slices as Parameters

    \n
  • Pass &str as a parameter, you can receive parameters of type &String and &str at the same time
    1
    fn first_word(s: &String) -> &str
    \nequivalent to
    1
    fn first_word(s: &str) -> &str
    • \n
\n

other slices

array slice

\n
1
2
3
4
5
fn main() {
  let a = [1, 2, 3, 4, 5];
  let slice = &a[1..3];
  assert_eq!(slice, &[2, 3]);
}
"},{"title":"rust basics","date":"2022-10-04T07:55:04.000Z","_content":"\n## frequently used cmd\n```\nrustc [filename].rs\ncargo new [project_name]\ncargo build [--release]\ncargo run [--release]\ncargo check # check whether compile success, no executible output\n```\n\n## data type\n\n### integer\n- i8,i16,i32,i64,i128,isize,u8,u16,u32,u64,u128,usize，etc\n- isize, usize indicates that the type is determined by the architecture of the computer. For example, on a 32 bit target, this is 4 bytes and on a 64 bit target, this is 8 bytes.\n- 0x: hex，0o Octal，0b binary，starting with b: byte （u8 only）\n\n| Number Literals | Example |\n| ----------- | ----------- |\n| Decimal | 98_222 |\n| Hex | 0xff |\n| Octal | 0o77 |\n| Binary | 0b1111_0000 |\n| Byte(u8 only) | b'A' |\n\n### Tuple\n- The length of Tuple is fixed, and the length cannot be changed once declared\n```rust\nfn main() {\n // tuple could be declared as mut\n let mut tuple_1 = (\"Hello\", 39, \"Years\");\n let tuple_2:(i32, &str ) = (1983, \"since.\");\n tuple_1.0 = \"Hi\";\n println!(\"{} {} {}\", tuple_1.0, tuple_1.1, tuple_1.2);\n // destructure\n let (a,b) = tuple_2;\n println!(\"{} {}\", a, b);\n}\n```\n\n### array\n- arrays in Rust have a fixed length.\n- Vector is similar to an array, it is provided by the standard library, and its length can be changed\n\n```rust\nfn main() {\n\n let arr_test:[u8; 3] = [1,2,3];\n println!(\"Number is {},{},{}\", arr_test[0],arr_test[1],arr_test[2]);\n\n let arr_test = [\"I\",\"love\",\"you\"];\n println!(\"You said : {} {} {}\", arr_test[0],arr_test[1],arr_test[2]);\n\n let arr_test = [1;3]; \n println!(\"Call Num : {}&{}&{}\", arr_test[0],arr_test[1],arr_test[2]);\n}\n```\n\n\n\n### String\n- Basic data types are stored on the stack, but the String type is stored on the heap\n```rust\nlet s = String::from(\"hello\");\n```\n- push_str(): append a str slice a string\n- push(): appends a single character to a String\n```rust\nfn main() { \n let mut data = String::from(\"andy\");\n data.push_str(\" is stronger\");\n data.push('!');\n}\n```\n- + operator, chaining strings. the left side of the + operator is the ownership of the string, and the right side is the string slice\n- String is actually a wrapper for Vec, so the length can be measured by the len() method, but note that Len() is not length of character, but byte len\n- String iteration\n```rust\nfn main() { \n let mut data = String::from(\"andy\");\n data.push_str(\" is stronger\");\n data.push('!');\n\n for i in data.bytes() {\n ///\n }\n\n for i in data.chars() {\n ///\n }\n}\n```\n\n### Vector\n- Vector is like any other struct. When Vector leaves the scope, the variable value is cleaned up, and all its elements are also cleaned up.\n```rust\nfn main() {\n let vec: Vec = Vec::new();\n let vec2: Vec = vec![3,4，5] // create vector by macro\n for i in vec2 {\n println!(\"Vector value is : {}\", i);\n }\n}\n```\n\n### HashMap\n- HashMap is not preloaded, so it needs to be included `use std::collections::HashMap`\n```rust\nuse std::collections::HashMap;\nfn main() {\n let keys = vec![\"andy\".to_string(), \"cliff\".to_string()] ;\n let ages = vec![38, 26];\n let map :HashMap<_,_> = keys.iter().zip(ages.iter()).collect();\n println!(\"{:?}\", map); /// print {\"andy\": 38, \"cliff\": 26}\n}\n```\n#### HashMap ownership\n- For types that implement the Copy trait (such as i32), the value will be copied into the HashMap\n- For values with ownership, such as (String), the value will be moved and ownership will be given to HashMap\n- If a reference to a value is inserted into the HashMap, the value itself does not move\n\n#### HashMap iteration\n```rust\nuse std::collections::HashMap;\n\nfn main() { \n let name = \"andy\".to_string();\n let age = 36;\n let mut map = HashMap::new();\n map.insert(name, age);\n map.insert(String::from(\"cliff\"), 26);\n println!(\"{:?}\", &map);\n for (k, v) in map {\n println!(\"{} age {}\", k, v);\n } /// cliff age 26\n /// andy age 36\n}\n```\n#### update\n```rust\nuse std::collections::HashMap;\n\nfn main() { \n let name = \"andy\".to_string();\n let age = 36;\n let mut map = HashMap::new();\n map.insert(name, age);\n map.insert(String::from(\"cliff\"), 26);\n\n let result = map.entry(\"bob\".to_string());\n println!(\"{:?}\", result); /// Entry(VacantEntry(\"bob\"))\n\n let result = map.entry(\"andy\".to_string());\n println!(\"{:?}\", result); /// Entry(OccupiedEntry { key: \"andy\", value: 36, .. })\n\n map.entry(\"bob\".to_string()).or_insert(28);\n map.entry(\"cliff\".to_string()).or_insert(0);\n}\n```\n\n## control flow\n- if\n```rust\nfn main() {\n let condition = 1;\n let x = if condition == 1 { \"A\" } else { \"B\" };\n println!(\"Result x = {}\" , x) ;\n}\n```\n- loop\n```rust\nfn main() {\n let mut condition = 0;\n\n let result = 'outer: loop { // 'outer is label\n 'inner: loop {\n condition += 1;\n if 3 == condition {\n break 'outer 3 * condition; // break outer loop\n }\n }\n };\n println!(\"Loop result is : {}\", result); /// Loop result is : 9\n}\n\n```\n- rot\n```rust\nfn main() {\n let arr = [3,2,3];\n for num in arr.iter() {\n println!(\"For value is {}\", num);\n }\n}\n```\n\n## Range iterator\n- Range\n```rust\nfn main() {\n for number in (1..=3) {\n println!(\"Number A is {}\", number ); /// 1,2,3\n }\n \n for number in (1..=3).rev() { /// rev means reverse,\n println!(\"Number B is {}\", number ); /// 3,2,1\n }\n}\n\n```\n\n## struct\n- If struct is declared mutable then all fields in the instance are mutable\n### tuple struct\n```rust\nstruct Color(i32,i32,i32);\nlet black = Color(0,0,0);\n```\n### Unit-Like struct\n```rust\nstruct Man {};\n```\n### struct method\n```rust\n\nfn main() {\n let rec = Rectangle {\n width: 30,\n height: 50,\n };\n\n let result = rec.area(); \n println!(\"rectangle：{:?}，area is：{}\", rec, result);\n}\n\n\n#[derive(Debug)]\nstruct Rectangle {\n width: u32,\n height: u32,\n}\n\nimpl Rectangle {\n fn area(&self) -> u32{\n self.width * self.height\n }\n}\n\n```\n### associative func（similar to static method）\n- You can define a function that does not take `self` as the first parameter in the impl block. This form is called an associated function, and the calling method is similar to `String::from()`\n```rust\nimpl Rectangle {\n fn create_square(width: u32) -> Rectangle {\n Rectangle {\n width,\n height: width,\n }\n }\n}\n```\n \n## enum\n```rust\nenum Ip {\n V4,\n V6,\n}\n\nenum IpAddr {\n V4(String),\n V6(String),\n}\n``` \n\n## match\n- match must exhaust all possibilities\n- If there are too many matchings, you can also use \"_\" for wildcarding, but note that \"_\" must be placed at the end\n```rust\nenum Color {\n Red,\n Yellow,\n Blue,\n}\nenum ColorWithVal {\n Red(u8,u8,u8),\n Yellow(u8,u8,u8),\n Blue(u8,u8,u8),\n}\nfn main(){\n let colour = Color::Blue;\n match colour {\n Color::Red => {\n println!(\"Red colour.\");\n },\n _ => {\n println!(\"Other colour.\");\n }\n }\n\n let colour = ColorWithVal::Red(222,111,22);\n match colour {\n ColorWithVal::Red(r,g,b) => {\n println!(\"Red colour. {},{},{}\", r,g,b);\n },\n _ => {\n println!(\"Other colour.\");\n }\n }\n}\n```\n\n## if let\n```rust\nfn main(){\n let colour = Color::Red(Some(222),Some(222),Some(222));\n\n if let Color::Red(r,g,b) = colour {\n println!(\"Red colour. {:?},{:?},{:?}\", r,g,b);\n } else {\n println!(\"Other colour.\");\n }\n}\n```\n\n## Result\n- Recoverable err via Result, non-recoverable via panic!\n- upon panic!, the program will expand an error message, unwind, clean up the call stack (Stack) and finally exit the program\n- You can set panic = 'abort' in Cargo.toml to terminate the cleaning of the call stack\n```rust\n[profile.release]\npanic='abort'\n```\n- RUST_BACKTRACE = 1 prints detailed error messages in the stack\n```rust\nuse std::fs::File;\nfn main() { \n let fp = File::open(\"hello.txt\");\n let file = match fp {\n Ok(file)=> {\n file\n },\n Err(error) => panic!(\"file not found {:?} \", error),\n };\n}\n```\n\n```rust\nuse std::{fs::File, io::ErrorKind};\nfn main() { \n let fp = File::open(\"hello.txt\");\n let file = match fp {\n Ok(file)=> {\n file\n },\n Err(error) => {\n match error.kind() {\n ErrorKind::NotFound => {\n match File::create(\"hello.txt\") {\n Ok(file) => {\n file\n },\n Err(err) => {\n panic!(\"file create error:{:?}\", &err);\n },\n }\n },\n oe => panic!(\"other error {:?}\", oe),\n }\n } ,\n };\n}\n```\n```rust\nuse std::{fs::File, io::ErrorKind};\nfn main() { \n let file = File::open(\"hello.txt\").unwrap_or_else(|err| {\n if err.kind() == ErrorKind::NotFound {\n File::create(\"hello.txt\").unwrap_or_else(|err|{\n panic!(\"error：{:?}\", err);\n })\n }else{\n panic!(\"other error：{:?}\", err);\n }\n });\n}\n```\n### unwrap && expect\n- If do not want to deal with Err, can use unwarp() method. If result is Ok(val), return val. If Err, then call the panic! macro.\n- expect can specify what the error message is, which is easier to debug\n\n### The question mark operator, ?\nWhen writing code that calls many functions that return the Result type, the error handling can be tedious. The question mark operator, ?, hides some of the boilerplate of propagating errors up the call stack.\n```rust\nlet mut file = File::create(\"my_best_friends.txt\")?;\n```\n\n## generic\n```rust\n#[derive(Debug)]\nstruct Point {\n x : T,\n y : U,\n}\nimpl Point {\n fn mixup(self, other: Point) -> Point {\n Point{x: self.x , y: other.y, }\n }\n}\n```\n\n## trait\n### definition\n```rust\npub trait Summary {\n fn summarize(&self) -> String {\n \"... more\".to_string() /// default \n }\n}\npub struct Tweet {\n user_name :String,\n replay_count :u32,\n like_count :u32,\n}\nimpl Tweet {\n fn like(&mut self) {\n self.like_count += 1;\n }\n}\n\nimpl Summary for Tweet {\n fn summarize(&self) -> String {\n format!(\"{} like count :{} , replay count :{}\", &self.user_name, &self.replay_count, &self.like_count)\n }\n}\n```\n\n### trait as arguments\n```rust\nfn notify_msg (info: T) {\n println!(\"summary : {}\", info.summarize() );\n}\nfn notify_msg (info: impl Summary + Display) {\n println!(\"summary : {}\", info.summarize() );\n}\nfn notify_msg (info: T) \nwhere \n T: Summary + Display,\n{\n println!(\"summary : {}\", info.summarize() );\n println!(\"display implement info : {}\", info);\n}\n```\n### trait as return\n- impl Trait can only return the same type, if it returns a different type, even if the Trait is implemented, an error will be reported\n\n## references\n- [the rust programming language](https://doc.rust-lang.org/book/)\n- [mooc course](https://time.geekbang.org/course/intro/100060601?tab=catalog)\n- [bilibili tutorial](https://www.bilibili.com/video/BV1hp4y1k7SV?p=50&spm_id_from=pageDriver)\n- [jianshu notes](https://www.jianshu.com/p/30d917790298)","source":"_posts/rust/rust-02-basics.md","raw":"---\ntitle: rust basics\ndate: 2022-10-04 15:55:04\ntags: [rust]\n---\n\n## frequently used cmd\n```\nrustc [filename].rs\ncargo new [project_name]\ncargo build [--release]\ncargo run [--release]\ncargo check # check whether compile success, no executible output\n```\n\n## data type\n\n### integer\n- i8,i16,i32,i64,i128,isize,u8,u16,u32,u64,u128,usize，etc\n- isize, usize indicates that the type is determined by the architecture of the computer. For example, on a 32 bit target, this is 4 bytes and on a 64 bit target, this is 8 bytes.\n- 0x: hex，0o Octal，0b binary，starting with b: byte （u8 only）\n\n| Number Literals | Example |\n| ----------- | ----------- |\n| Decimal | 98_222 |\n| Hex | 0xff |\n| Octal | 0o77 |\n| Binary | 0b1111_0000 |\n| Byte(u8 only) | b'A' |\n\n### Tuple\n- The length of Tuple is fixed, and the length cannot be changed once declared\n```rust\nfn main() {\n // tuple could be declared as mut\n let mut tuple_1 = (\"Hello\", 39, \"Years\");\n let tuple_2:(i32, &str ) = (1983, \"since.\");\n tuple_1.0 = \"Hi\";\n println!(\"{} {} {}\", tuple_1.0, tuple_1.1, tuple_1.2);\n // destructure\n let (a,b) = tuple_2;\n println!(\"{} {}\", a, b);\n}\n```\n\n### array\n- arrays in Rust have a fixed length.\n- Vector is similar to an array, it is provided by the standard library, and its length can be changed\n\n```rust\nfn main() {\n\n let arr_test:[u8; 3] = [1,2,3];\n println!(\"Number is {},{},{}\", arr_test[0],arr_test[1],arr_test[2]);\n\n let arr_test = [\"I\",\"love\",\"you\"];\n println!(\"You said : {} {} {}\", arr_test[0],arr_test[1],arr_test[2]);\n\n let arr_test = [1;3]; \n println!(\"Call Num : {}&{}&{}\", arr_test[0],arr_test[1],arr_test[2]);\n}\n```\n\n\n\n### String\n- Basic data types are stored on the stack, but the String type is stored on the heap\n```rust\nlet s = String::from(\"hello\");\n```\n- push_str(): append a str slice a string\n- push(): appends a single character to a String\n```rust\nfn main() { \n let mut data = String::from(\"andy\");\n data.push_str(\" is stronger\");\n data.push('!');\n}\n```\n- + operator, chaining strings. the left side of the + operator is the ownership of the string, and the right side is the string slice\n- String is actually a wrapper for Vec, so the length can be measured by the len() method, but note that Len() is not length of character, but byte len\n- String iteration\n```rust\nfn main() { \n let mut data = String::from(\"andy\");\n data.push_str(\" is stronger\");\n data.push('!');\n\n for i in data.bytes() {\n ///\n }\n\n for i in data.chars() {\n ///\n }\n}\n```\n\n### Vector\n- Vector is like any other struct. When Vector leaves the scope, the variable value is cleaned up, and all its elements are also cleaned up.\n```rust\nfn main() {\n let vec: Vec = Vec::new();\n let vec2: Vec = vec![3,4，5] // create vector by macro\n for i in vec2 {\n println!(\"Vector value is : {}\", i);\n }\n}\n```\n\n### HashMap\n- HashMap is not preloaded, so it needs to be included `use std::collections::HashMap`\n```rust\nuse std::collections::HashMap;\nfn main() {\n let keys = vec![\"andy\".to_string(), \"cliff\".to_string()] ;\n let ages = vec![38, 26];\n let map :HashMap<_,_> = keys.iter().zip(ages.iter()).collect();\n println!(\"{:?}\", map); /// print {\"andy\": 38, \"cliff\": 26}\n}\n```\n#### HashMap ownership\n- For types that implement the Copy trait (such as i32), the value will be copied into the HashMap\n- For values with ownership, such as (String), the value will be moved and ownership will be given to HashMap\n- If a reference to a value is inserted into the HashMap, the value itself does not move\n\n#### HashMap iteration\n```rust\nuse std::collections::HashMap;\n\nfn main() { \n let name = \"andy\".to_string();\n let age = 36;\n let mut map = HashMap::new();\n map.insert(name, age);\n map.insert(String::from(\"cliff\"), 26);\n println!(\"{:?}\", &map);\n for (k, v) in map {\n println!(\"{} age {}\", k, v);\n } /// cliff age 26\n /// andy age 36\n}\n```\n#### update\n```rust\nuse std::collections::HashMap;\n\nfn main() { \n let name = \"andy\".to_string();\n let age = 36;\n let mut map = HashMap::new();\n map.insert(name, age);\n map.insert(String::from(\"cliff\"), 26);\n\n let result = map.entry(\"bob\".to_string());\n println!(\"{:?}\", result); /// Entry(VacantEntry(\"bob\"))\n\n let result = map.entry(\"andy\".to_string());\n println!(\"{:?}\", result); /// Entry(OccupiedEntry { key: \"andy\", value: 36, .. })\n\n map.entry(\"bob\".to_string()).or_insert(28);\n map.entry(\"cliff\".to_string()).or_insert(0);\n}\n```\n\n## control flow\n- if\n```rust\nfn main() {\n let condition = 1;\n let x = if condition == 1 { \"A\" } else { \"B\" };\n println!(\"Result x = {}\" , x) ;\n}\n```\n- loop\n```rust\nfn main() {\n let mut condition = 0;\n\n let result = 'outer: loop { // 'outer is label\n 'inner: loop {\n condition += 1;\n if 3 == condition {\n break 'outer 3 * condition; // break outer loop\n }\n }\n };\n println!(\"Loop result is : {}\", result); /// Loop result is : 9\n}\n\n```\n- rot\n```rust\nfn main() {\n let arr = [3,2,3];\n for num in arr.iter() {\n println!(\"For value is {}\", num);\n }\n}\n```\n\n## Range iterator\n- Range\n```rust\nfn main() {\n for number in (1..=3) {\n println!(\"Number A is {}\", number ); /// 1,2,3\n }\n \n for number in (1..=3).rev() { /// rev means reverse,\n println!(\"Number B is {}\", number ); /// 3,2,1\n }\n}\n\n```\n\n## struct\n- If struct is declared mutable then all fields in the instance are mutable\n### tuple struct\n```rust\nstruct Color(i32,i32,i32);\nlet black = Color(0,0,0);\n```\n### Unit-Like struct\n```rust\nstruct Man {};\n```\n### struct method\n```rust\n\nfn main() {\n let rec = Rectangle {\n width: 30,\n height: 50,\n };\n\n let result = rec.area(); \n println!(\"rectangle：{:?}，area is：{}\", rec, result);\n}\n\n\n#[derive(Debug)]\nstruct Rectangle {\n width: u32,\n height: u32,\n}\n\nimpl Rectangle {\n fn area(&self) -> u32{\n self.width * self.height\n }\n}\n\n```\n### associative func（similar to static method）\n- You can define a function that does not take `self` as the first parameter in the impl block. This form is called an associated function, and the calling method is similar to `String::from()`\n```rust\nimpl Rectangle {\n fn create_square(width: u32) -> Rectangle {\n Rectangle {\n width,\n height: width,\n }\n }\n}\n```\n \n## enum\n```rust\nenum Ip {\n V4,\n V6,\n}\n\nenum IpAddr {\n V4(String),\n V6(String),\n}\n``` \n\n## match\n- match must exhaust all possibilities\n- If there are too many matchings, you can also use \"_\" for wildcarding, but note that \"_\" must be placed at the end\n```rust\nenum Color {\n Red,\n Yellow,\n Blue,\n}\nenum ColorWithVal {\n Red(u8,u8,u8),\n Yellow(u8,u8,u8),\n Blue(u8,u8,u8),\n}\nfn main(){\n let colour = Color::Blue;\n match colour {\n Color::Red => {\n println!(\"Red colour.\");\n },\n _ => {\n println!(\"Other colour.\");\n }\n }\n\n let colour = ColorWithVal::Red(222,111,22);\n match colour {\n ColorWithVal::Red(r,g,b) => {\n println!(\"Red colour. {},{},{}\", r,g,b);\n },\n _ => {\n println!(\"Other colour.\");\n }\n }\n}\n```\n\n## if let\n```rust\nfn main(){\n let colour = Color::Red(Some(222),Some(222),Some(222));\n\n if let Color::Red(r,g,b) = colour {\n println!(\"Red colour. {:?},{:?},{:?}\", r,g,b);\n } else {\n println!(\"Other colour.\");\n }\n}\n```\n\n## Result\n- Recoverable err via Result, non-recoverable via panic!\n- upon panic!, the program will expand an error message, unwind, clean up the call stack (Stack) and finally exit the program\n- You can set panic = 'abort' in Cargo.toml to terminate the cleaning of the call stack\n```rust\n[profile.release]\npanic='abort'\n```\n- RUST_BACKTRACE = 1 prints detailed error messages in the stack\n```rust\nuse std::fs::File;\nfn main() { \n let fp = File::open(\"hello.txt\");\n let file = match fp {\n Ok(file)=> {\n file\n },\n Err(error) => panic!(\"file not found {:?} \", error),\n };\n}\n```\n\n```rust\nuse std::{fs::File, io::ErrorKind};\nfn main() { \n let fp = File::open(\"hello.txt\");\n let file = match fp {\n Ok(file)=> {\n file\n },\n Err(error) => {\n match error.kind() {\n ErrorKind::NotFound => {\n match File::create(\"hello.txt\") {\n Ok(file) => {\n file\n },\n Err(err) => {\n panic!(\"file create error:{:?}\", &err);\n },\n }\n },\n oe => panic!(\"other error {:?}\", oe),\n }\n } ,\n };\n}\n```\n```rust\nuse std::{fs::File, io::ErrorKind};\nfn main() { \n let file = File::open(\"hello.txt\").unwrap_or_else(|err| {\n if err.kind() == ErrorKind::NotFound {\n File::create(\"hello.txt\").unwrap_or_else(|err|{\n panic!(\"error：{:?}\", err);\n })\n }else{\n panic!(\"other error：{:?}\", err);\n }\n });\n}\n```\n### unwrap && expect\n- If do not want to deal with Err, can use unwarp() method. If result is Ok(val), return val. If Err, then call the panic! macro.\n- expect can specify what the error message is, which is easier to debug\n\n### The question mark operator, ?\nWhen writing code that calls many functions that return the Result type, the error handling can be tedious. The question mark operator, ?, hides some of the boilerplate of propagating errors up the call stack.\n```rust\nlet mut file = File::create(\"my_best_friends.txt\")?;\n```\n\n## generic\n```rust\n#[derive(Debug)]\nstruct Point {\n x : T,\n y : U,\n}\nimpl Point {\n fn mixup(self, other: Point) -> Point {\n Point{x: self.x , y: other.y, }\n }\n}\n```\n\n## trait\n### definition\n```rust\npub trait Summary {\n fn summarize(&self) -> String {\n \"... more\".to_string() /// default \n }\n}\npub struct Tweet {\n user_name :String,\n replay_count :u32,\n like_count :u32,\n}\nimpl Tweet {\n fn like(&mut self) {\n self.like_count += 1;\n }\n}\n\nimpl Summary for Tweet {\n fn summarize(&self) -> String {\n format!(\"{} like count :{} , replay count :{}\", &self.user_name, &self.replay_count, &self.like_count)\n }\n}\n```\n\n### trait as arguments\n```rust\nfn notify_msg (info: T) {\n println!(\"summary : {}\", info.summarize() );\n}\nfn notify_msg (info: impl Summary + Display) {\n println!(\"summary : {}\", info.summarize() );\n}\nfn notify_msg (info: T) \nwhere \n T: Summary + Display,\n{\n println!(\"summary : {}\", info.summarize() );\n println!(\"display implement info : {}\", info);\n}\n```\n### trait as return\n- impl Trait can only return the same type, if it returns a different type, even if the Trait is implemented, an error will be reported\n\n## references\n- [the rust programming language](https://doc.rust-lang.org/book/)\n- [mooc course](https://time.geekbang.org/course/intro/100060601?tab=catalog)\n- [bilibili tutorial](https://www.bilibili.com/video/BV1hp4y1k7SV?p=50&spm_id_from=pageDriver)\n- [jianshu notes](https://www.jianshu.com/p/30d917790298)","slug":"rust/rust-02-basics","published":1,"updated":"2023-11-05T04:21:13.734Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8dr0010qwsj4oml40f0","content":"

frequently used cmd

1
2
3
4
5
rustc [filename].rs
cargo new [project_name]
cargo build [--release]
cargo run [--release]
cargo check # check whether compile success, no executible output
\n\n

data type

integer

    \n
  • i8,i16,i32,i64,i128,isize,u8,u16,u32,u64,u128,usize，etc
    • \n
  • isize, usize indicates that the type is determined by the architecture of the computer. For example, on a 32 bit target, this is 4 bytes and on a 64 bit target, this is 8 bytes.
    • \n
  • 0x: hex，0o Octal，0b binary，starting with b: byte （u8 only）
    • \n
\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
Number LiteralsExample
Decimal98_222
Hex0xff
Octal0o77
Binary0b1111_0000
Byte(u8 only)b’A’
\n

Tuple

    \n
  • The length of Tuple is fixed, and the length cannot be changed once declared
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    fn main() {
        // tuple could be declared as mut
        let mut tuple_1 = ("Hello", 39, "Years");
        let tuple_2:(i32, &str ) = (1983, "since.");
        tuple_1.0 = "Hi";
        println!("{} {} {}", tuple_1.0, tuple_1.1, tuple_1.2);
        // destructure
        let (a,b) = tuple_2;
        println!("{} {}", a, b);
    }
    • \n
\n

array

    \n
  • arrays in Rust have a fixed length.
    • \n
  • Vector is similar to an array, it is provided by the standard library, and its length can be changed
    • \n
\n
1
2
3
4
5
6
7
8
9
10
11
fn main() {

    let arr_test:[u8; 3] = [1,2,3];
    println!("Number is {},{},{}", arr_test[0],arr_test[1],arr_test[2]);

    let arr_test = ["I","love","you"];
    println!("You said : {} {} {}", arr_test[0],arr_test[1],arr_test[2]);

    let arr_test = [1;3]; 
    println!("Call Num : {}&{}&{}", arr_test[0],arr_test[1],arr_test[2]);
}
\n\n\n\n

String

    \n
  • Basic data types are stored on the stack, but the String type is stored on the heap
    1
    let s = String::from("hello");
    • \n
  • push_str(): append a str slice a string
    • \n
  • push(): appends a single character to a String
    1
    2
    3
    4
    5
    fn main() { 
        let mut data = String::from("andy");
        data.push_str(" is stronger");
        data.push('!');
    }
    • \n
  • + operator, chaining strings. the left side of the + operator is the ownership of the string, and the right side is the string slice
    • \n
  • String is actually a wrapper for Vec, so the length can be measured by the len() method, but note that Len() is not length of character, but byte len
    • \n
  • String iteration
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    fn main() { 
        let mut data = String::from("andy");
        data.push_str(" is stronger");
        data.push('!');

        for i in data.bytes() {
            ///
        }

        for i in data.chars() {
            ///
        }
    }
    • \n
\n

Vector

    \n
  • Vector is like any other struct. When Vector leaves the scope, the variable value is cleaned up, and all its elements are also cleaned up.
    1
    2
    3
    4
    5
    6
    7
    fn main() {
        let vec: Vec<u16> = Vec::new();
        let vec2: Vec<i32> = vec![3,4，5] // create vector by macro
        for i in vec2 {
            println!("Vector value is : {}", i);
        }
    }
    • \n
\n

HashMap

    \n
  • HashMap is not preloaded, so it needs to be included use std::collections::HashMap
    1
    2
    3
    4
    5
    6
    7
    use std::collections::HashMap;
    fn main() {
        let keys = vec!["andy".to_string(), "cliff".to_string()] ;
        let ages = vec![38, 26];
        let map :HashMap<_,_> = keys.iter().zip(ages.iter()).collect();
        println!("{:?}", map); /// print {"andy": 38, "cliff": 26}
    }
    • \n
\n

HashMap ownership

    \n
  • For types that implement the Copy trait (such as i32), the value will be copied into the HashMap
    • \n
  • For values with ownership, such as (String), the value will be moved and ownership will be given to HashMap
    • \n
  • If a reference to a value is inserted into the HashMap, the value itself does not move
    • \n
\n

HashMap iteration

1
2
3
4
5
6
7
8
9
10
11
12
13
14
use std::collections::HashMap;

fn main() { 
    let name = "andy".to_string();
    let age = 36;
    let mut map = HashMap::new();
    map.insert(name, age);
    map.insert(String::from("cliff"), 26);
    println!("{:?}", &map);
    for (k, v) in map {
        println!("{} age {}", k, v);
    } /// cliff age 26
      /// andy age 36
}
\n

update

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
use std::collections::HashMap;

fn main() { 
    let name = "andy".to_string();
    let age = 36;
    let mut map = HashMap::new();
    map.insert(name, age);
    map.insert(String::from("cliff"), 26);

    let result = map.entry("bob".to_string());
    println!("{:?}", result); /// Entry(VacantEntry("bob"))

    let result = map.entry("andy".to_string());
    println!("{:?}", result); /// Entry(OccupiedEntry { key: "andy", value: 36, .. })

    map.entry("bob".to_string()).or_insert(28);
    map.entry("cliff".to_string()).or_insert(0);
}
\n\n

control flow

    \n
  • if
    1
    2
    3
    4
    5
    fn main() {
        let condition = 1;
        let x = if condition == 1 { "A" } else { "B" };
        println!("Result x = {}" , x) ;
    }
    • \n
  • loop
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    fn main() {
        let mut condition = 0;

        let result = 'outer: loop {  // 'outer is label
            'inner: loop {
                condition += 1;
                if 3 == condition {
                    break 'outer 3 * condition; // break outer loop
                }
            }
        };
        println!("Loop result is : {}", result); /// Loop result is : 9
    }

    • \n
  • rot
    1
    2
    3
    4
    5
    6
    fn main() {
        let arr = [3,2,3];
        for num in arr.iter() {
            println!("For value is {}", num);
        }
    }
    • \n
\n

Range iterator

    \n
  • Range
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    fn main() {
         for number in (1..=3) {
            println!("Number A is {}", number ); /// 1,2,3
        }
     
        for number in (1..=3).rev() { /// rev means reverse,
            println!("Number B is {}", number ); /// 3,2,1
        }
    }

    • \n
\n

struct

    \n
  • If struct is declared mutable then all fields in the instance are mutable
    • \n
\n

tuple struct

1
2
struct Color(i32,i32,i32);
let black = Color(0,0,0);
\n

Unit-Like struct

1
struct Man {};
\n

struct method

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

fn main() {
    let rec = Rectangle {
        width: 30,
        height: 50,
    };

    let result = rec.area(); 
    println!("rectangle：{:?}，area is：{}", rec, result);
}


#[derive(Debug)]
struct Rectangle {
    width: u32,
    height: u32,
}

impl Rectangle {
    fn area(&self) -> u32{
        self.width * self.height
    }
}

\n

associative func（similar to static method）

    \n
  • You can define a function that does not take self as the first parameter in the impl block. This form is called an associated function, and the calling method is similar to String::from()
    1
    2
    3
    4
    5
    6
    7
    8
    impl Rectangle {
        fn create_square(width: u32) -> Rectangle {
            Rectangle {
                width,
                height: width,
            }
        }
    }
    • \n
\n

enum

1
2
3
4
5
6
7
8
9
enum Ip {
    V4,
    V6,
}

enum IpAddr {
    V4(String),
    V6(String),
}
\n\n

match

    \n
  • match must exhaust all possibilities
    • \n
  • If there are too many matchings, you can also use ““ for wildcarding, but note that ““ must be placed at the end
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28
    29
    30
    31
    enum Color {
        Red,
        Yellow,
        Blue,
    }
    enum ColorWithVal {
        Red(u8,u8,u8),
        Yellow(u8,u8,u8),
        Blue(u8,u8,u8),
    }
    fn main(){
        let colour = Color::Blue;
        match colour {
            Color::Red => {
                println!("Red colour.");
            },
            _ => {
                println!("Other colour.");
            }
        }

        let colour = ColorWithVal::Red(222,111,22);
        match colour {
            ColorWithVal::Red(r,g,b) => {
                println!("Red colour. {},{},{}", r,g,b);
            },
            _ => {
                println!("Other colour.");
            }
        }
    }
    • \n
\n

if let

1
2
3
4
5
6
7
8
9
fn main(){
    let colour = Color::Red(Some(222),Some(222),Some(222));

    if let Color::Red(r,g,b) = colour {
        println!("Red colour. {:?},{:?},{:?}", r,g,b);
    } else {
        println!("Other colour.");
    }
}
\n\n

Result<T,E>

    \n
  • Recoverable err via Result<T,E>, non-recoverable via panic!
    • \n
  • upon panic!, the program will expand an error message, unwind, clean up the call stack (Stack) and finally exit the program
    • \n
  • You can set panic = ‘abort’ in Cargo.toml to terminate the cleaning of the call stack
    1
    2
    [profile.release]
    panic='abort'
    • \n
  • RUST_BACKTRACE = 1 prints detailed error messages in the stack
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    use std::fs::File;
    fn main() { 
        let fp = File::open("hello.txt");
        let file = match fp {
            Ok(file)=> {
                file
            },
            Err(error) => panic!("file not found {:?} ", error),
        };
    }
    • \n
\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
use std::{fs::File, io::ErrorKind};
fn main() { 
    let fp = File::open("hello.txt");
    let file = match fp {
        Ok(file)=> {
            file
        },
        Err(error) => {
            match error.kind() {
                ErrorKind::NotFound => {
                    match File::create("hello.txt") {
                        Ok(file) => {
                            file
                        },
                        Err(err) => {
                            panic!("file create error:{:?}", &err);
                        },
                    }
                },
                oe => panic!("other error {:?}", oe),
            }
        } ,
    };
}
\n
1
2
3
4
5
6
7
8
9
10
11
12
use std::{fs::File, io::ErrorKind};
fn main() { 
    let file = File::open("hello.txt").unwrap_or_else(|err| {
        if err.kind() == ErrorKind::NotFound {
            File::create("hello.txt").unwrap_or_else(|err|{
                panic!("error：{:?}", err);
            })
        }else{
            panic!("other error：{:?}", err);
        }
    });
}
\n

unwrap && expect

    \n
  • If do not want to deal with Err, can use unwarp() method. If result is Ok(val), return val. If Err, then call the panic! macro.
    • \n
  • expect can specify what the error message is, which is easier to debug
    • \n
\n

The question mark operator, ?

When writing code that calls many functions that return the Result type, the error handling can be tedious. The question mark operator, ?, hides some of the boilerplate of propagating errors up the call stack.

\n
1
let mut file = File::create("my_best_friends.txt")?;
\n\n

generic

1
2
3
4
5
6
7
8
9
10
#[derive(Debug)]
struct Point<T, U>  {
   x : T,
   y : U,
}
impl <T, U> Point<T, U> {
    fn mixup<V, W>(self, other: Point<V, W>) -> Point<T, W> {
        Point{x: self.x , y: other.y, }
    }
}
\n\n

trait

definition

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
pub trait Summary {
    fn summarize(&self) -> String {
         "... more".to_string() /// default 
    }
}
pub struct Tweet {
    user_name :String,
    replay_count :u32,
    like_count :u32,
}
impl Tweet {
    fn like(&mut self) {
        self.like_count += 1;
    }
}

impl Summary for Tweet {
    fn summarize(&self) -> String {
        format!("{} like count :{} , replay count :{}", &self.user_name, &self.replay_count, &self.like_count)
    }
}
\n\n

trait as arguments

1
2
3
4
5
6
7
8
9
10
11
12
13
fn notify_msg <T:Summary> (info: T) {
    println!("summary : {}", info.summarize() );
}
fn notify_msg (info: impl Summary + Display) {
    println!("summary : {}", info.summarize() );
}
fn notify_msg <T> (info: T) 
where 
    T: Summary + Display,
{
    println!("summary : {}", info.summarize() );
    println!("display implement info : {}", info);
}
\n

trait as return

    \n
  • impl Trait can only return the same type, if it returns a different type, even if the Trait is implemented, an error will be reported
    • \n
\n

references

\n","site":{"data":{}},"excerpt":"","more":"

frequently used cmd

1
2
3
4
5
rustc [filename].rs
cargo new [project_name]
cargo build [--release]
cargo run [--release]
cargo check # check whether compile success, no executible output
\n\n

data type

integer

    \n
  • i8,i16,i32,i64,i128,isize,u8,u16,u32,u64,u128,usize，etc
    • \n
  • isize, usize indicates that the type is determined by the architecture of the computer. For example, on a 32 bit target, this is 4 bytes and on a 64 bit target, this is 8 bytes.
    • \n
  • 0x: hex，0o Octal，0b binary，starting with b: byte （u8 only）
    • \n
\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
Number LiteralsExample
Decimal98_222
Hex0xff
Octal0o77
Binary0b1111_0000
Byte(u8 only)b’A’
\n

Tuple

    \n
  • The length of Tuple is fixed, and the length cannot be changed once declared
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    fn main() {
        // tuple could be declared as mut
        let mut tuple_1 = ("Hello", 39, "Years");
        let tuple_2:(i32, &str ) = (1983, "since.");
        tuple_1.0 = "Hi";
        println!("{} {} {}", tuple_1.0, tuple_1.1, tuple_1.2);
        // destructure
        let (a,b) = tuple_2;
        println!("{} {}", a, b);
    }
    • \n
\n

array

    \n
  • arrays in Rust have a fixed length.
    • \n
  • Vector is similar to an array, it is provided by the standard library, and its length can be changed
    • \n
\n
1
2
3
4
5
6
7
8
9
10
11
fn main() {

    let arr_test:[u8; 3] = [1,2,3];
    println!("Number is {},{},{}", arr_test[0],arr_test[1],arr_test[2]);

    let arr_test = ["I","love","you"];
    println!("You said : {} {} {}", arr_test[0],arr_test[1],arr_test[2]);

    let arr_test = [1;3]; 
    println!("Call Num : {}&{}&{}", arr_test[0],arr_test[1],arr_test[2]);
}
\n\n\n\n

String

    \n
  • Basic data types are stored on the stack, but the String type is stored on the heap
    1
    let s = String::from("hello");
    • \n
  • push_str(): append a str slice a string
    • \n
  • push(): appends a single character to a String
    1
    2
    3
    4
    5
    fn main() { 
        let mut data = String::from("andy");
        data.push_str(" is stronger");
        data.push('!');
    }
    • \n
  • + operator, chaining strings. the left side of the + operator is the ownership of the string, and the right side is the string slice
    • \n
  • String is actually a wrapper for Vec, so the length can be measured by the len() method, but note that Len() is not length of character, but byte len
    • \n
  • String iteration
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    fn main() { 
        let mut data = String::from("andy");
        data.push_str(" is stronger");
        data.push('!');

        for i in data.bytes() {
            ///
        }

        for i in data.chars() {
            ///
        }
    }
    • \n
\n

Vector

    \n
  • Vector is like any other struct. When Vector leaves the scope, the variable value is cleaned up, and all its elements are also cleaned up.
    1
    2
    3
    4
    5
    6
    7
    fn main() {
        let vec: Vec<u16> = Vec::new();
        let vec2: Vec<i32> = vec![3,4，5] // create vector by macro
        for i in vec2 {
            println!("Vector value is : {}", i);
        }
    }
    • \n
\n

HashMap

    \n
  • HashMap is not preloaded, so it needs to be included use std::collections::HashMap
    1
    2
    3
    4
    5
    6
    7
    use std::collections::HashMap;
    fn main() {
        let keys = vec!["andy".to_string(), "cliff".to_string()] ;
        let ages = vec![38, 26];
        let map :HashMap<_,_> = keys.iter().zip(ages.iter()).collect();
        println!("{:?}", map); /// print {"andy": 38, "cliff": 26}
    }
    • \n
\n

HashMap ownership

    \n
  • For types that implement the Copy trait (such as i32), the value will be copied into the HashMap
    • \n
  • For values with ownership, such as (String), the value will be moved and ownership will be given to HashMap
    • \n
  • If a reference to a value is inserted into the HashMap, the value itself does not move
    • \n
\n

HashMap iteration

1
2
3
4
5
6
7
8
9
10
11
12
13
14
use std::collections::HashMap;

fn main() { 
    let name = "andy".to_string();
    let age = 36;
    let mut map = HashMap::new();
    map.insert(name, age);
    map.insert(String::from("cliff"), 26);
    println!("{:?}", &map);
    for (k, v) in map {
        println!("{} age {}", k, v);
    } /// cliff age 26
      /// andy age 36
}
\n

update

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
use std::collections::HashMap;

fn main() { 
    let name = "andy".to_string();
    let age = 36;
    let mut map = HashMap::new();
    map.insert(name, age);
    map.insert(String::from("cliff"), 26);

    let result = map.entry("bob".to_string());
    println!("{:?}", result); /// Entry(VacantEntry("bob"))

    let result = map.entry("andy".to_string());
    println!("{:?}", result); /// Entry(OccupiedEntry { key: "andy", value: 36, .. })

    map.entry("bob".to_string()).or_insert(28);
    map.entry("cliff".to_string()).or_insert(0);
}
\n\n

control flow

    \n
  • if
    1
    2
    3
    4
    5
    fn main() {
        let condition = 1;
        let x = if condition == 1 { "A" } else { "B" };
        println!("Result x = {}" , x) ;
    }
    • \n
  • loop
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    fn main() {
        let mut condition = 0;

        let result = 'outer: loop {  // 'outer is label
            'inner: loop {
                condition += 1;
                if 3 == condition {
                    break 'outer 3 * condition; // break outer loop
                }
            }
        };
        println!("Loop result is : {}", result); /// Loop result is : 9
    }

    • \n
  • rot
    1
    2
    3
    4
    5
    6
    fn main() {
        let arr = [3,2,3];
        for num in arr.iter() {
            println!("For value is {}", num);
        }
    }
    • \n
\n

Range iterator

    \n
  • Range
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    fn main() {
         for number in (1..=3) {
            println!("Number A is {}", number ); /// 1,2,3
        }
     
        for number in (1..=3).rev() { /// rev means reverse,
            println!("Number B is {}", number ); /// 3,2,1
        }
    }

    • \n
\n

struct

    \n
  • If struct is declared mutable then all fields in the instance are mutable
    • \n
\n

tuple struct

1
2
struct Color(i32,i32,i32);
let black = Color(0,0,0);
\n

Unit-Like struct

1
struct Man {};
\n

struct method

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

fn main() {
    let rec = Rectangle {
        width: 30,
        height: 50,
    };

    let result = rec.area(); 
    println!("rectangle：{:?}，area is：{}", rec, result);
}


#[derive(Debug)]
struct Rectangle {
    width: u32,
    height: u32,
}

impl Rectangle {
    fn area(&self) -> u32{
        self.width * self.height
    }
}

\n

associative func（similar to static method）

    \n
  • You can define a function that does not take self as the first parameter in the impl block. This form is called an associated function, and the calling method is similar to String::from()
    1
    2
    3
    4
    5
    6
    7
    8
    impl Rectangle {
        fn create_square(width: u32) -> Rectangle {
            Rectangle {
                width,
                height: width,
            }
        }
    }
    • \n
\n

enum

1
2
3
4
5
6
7
8
9
enum Ip {
    V4,
    V6,
}

enum IpAddr {
    V4(String),
    V6(String),
}
\n\n

match

    \n
  • match must exhaust all possibilities
    • \n
  • If there are too many matchings, you can also use ““ for wildcarding, but note that ““ must be placed at the end
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28
    29
    30
    31
    enum Color {
        Red,
        Yellow,
        Blue,
    }
    enum ColorWithVal {
        Red(u8,u8,u8),
        Yellow(u8,u8,u8),
        Blue(u8,u8,u8),
    }
    fn main(){
        let colour = Color::Blue;
        match colour {
            Color::Red => {
                println!("Red colour.");
            },
            _ => {
                println!("Other colour.");
            }
        }

        let colour = ColorWithVal::Red(222,111,22);
        match colour {
            ColorWithVal::Red(r,g,b) => {
                println!("Red colour. {},{},{}", r,g,b);
            },
            _ => {
                println!("Other colour.");
            }
        }
    }
    • \n
\n

if let

1
2
3
4
5
6
7
8
9
fn main(){
    let colour = Color::Red(Some(222),Some(222),Some(222));

    if let Color::Red(r,g,b) = colour {
        println!("Red colour. {:?},{:?},{:?}", r,g,b);
    } else {
        println!("Other colour.");
    }
}
\n\n

Result<T,E>

    \n
  • Recoverable err via Result<T,E>, non-recoverable via panic!
    • \n
  • upon panic!, the program will expand an error message, unwind, clean up the call stack (Stack) and finally exit the program
    • \n
  • You can set panic = ‘abort’ in Cargo.toml to terminate the cleaning of the call stack
    1
    2
    [profile.release]
    panic='abort'
    • \n
  • RUST_BACKTRACE = 1 prints detailed error messages in the stack
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    use std::fs::File;
    fn main() { 
        let fp = File::open("hello.txt");
        let file = match fp {
            Ok(file)=> {
                file
            },
            Err(error) => panic!("file not found {:?} ", error),
        };
    }
    • \n
\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
use std::{fs::File, io::ErrorKind};
fn main() { 
    let fp = File::open("hello.txt");
    let file = match fp {
        Ok(file)=> {
            file
        },
        Err(error) => {
            match error.kind() {
                ErrorKind::NotFound => {
                    match File::create("hello.txt") {
                        Ok(file) => {
                            file
                        },
                        Err(err) => {
                            panic!("file create error:{:?}", &err);
                        },
                    }
                },
                oe => panic!("other error {:?}", oe),
            }
        } ,
    };
}
\n
1
2
3
4
5
6
7
8
9
10
11
12
use std::{fs::File, io::ErrorKind};
fn main() { 
    let file = File::open("hello.txt").unwrap_or_else(|err| {
        if err.kind() == ErrorKind::NotFound {
            File::create("hello.txt").unwrap_or_else(|err|{
                panic!("error：{:?}", err);
            })
        }else{
            panic!("other error：{:?}", err);
        }
    });
}
\n

unwrap && expect

    \n
  • If do not want to deal with Err, can use unwarp() method. If result is Ok(val), return val. If Err, then call the panic! macro.
    • \n
  • expect can specify what the error message is, which is easier to debug
    • \n
\n

The question mark operator, ?

When writing code that calls many functions that return the Result type, the error handling can be tedious. The question mark operator, ?, hides some of the boilerplate of propagating errors up the call stack.

\n
1
let mut file = File::create("my_best_friends.txt")?;
\n\n

generic

1
2
3
4
5
6
7
8
9
10
#[derive(Debug)]
struct Point<T, U>  {
   x : T,
   y : U,
}
impl <T, U> Point<T, U> {
    fn mixup<V, W>(self, other: Point<V, W>) -> Point<T, W> {
        Point{x: self.x , y: other.y, }
    }
}
\n\n

trait

definition

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
pub trait Summary {
    fn summarize(&self) -> String {
         "... more".to_string() /// default 
    }
}
pub struct Tweet {
    user_name :String,
    replay_count :u32,
    like_count :u32,
}
impl Tweet {
    fn like(&mut self) {
        self.like_count += 1;
    }
}

impl Summary for Tweet {
    fn summarize(&self) -> String {
        format!("{} like count :{} , replay count :{}", &self.user_name, &self.replay_count, &self.like_count)
    }
}
\n\n

trait as arguments

1
2
3
4
5
6
7
8
9
10
11
12
13
fn notify_msg <T:Summary> (info: T) {
    println!("summary : {}", info.summarize() );
}
fn notify_msg (info: impl Summary + Display) {
    println!("summary : {}", info.summarize() );
}
fn notify_msg <T> (info: T) 
where 
    T: Summary + Display,
{
    println!("summary : {}", info.summarize() );
    println!("display implement info : {}", info);
}
\n

trait as return

    \n
  • impl Trait can only return the same type, if it returns a different type, even if the Trait is implemented, an error will be reported
    • \n
\n

references

\n"},{"title":"rust lifetime","date":"2022-10-18T13:33:26.000Z","_content":"\n## lifetime\n- Every reference in Rust has its own lifecycle\n- most of the time, Rust's lifetime is implicit and can be inferred\n- There are two types of life cycle: input life cycle and output life cycle\n- 'static is a special life cycle annotation\n### Example of lifetime out of scope\n```rust\nfn main() {\n let mut x;\n {\n let y = String::from(\"hello\");\n // x = y; // this is allowed\n x = &y; // not allowed. borrowed value (y) does not live long enough\n }\n println!(\"Str:{}\", x);\n}\n```\n\n### lifetime checker\nRust compiler's borrow checker to determine whether a borrow is legal\n```rust\n// If it returns a reference value, no matter how simple your function is written, it will always report an error `missing lifetime specifier.`\nfn longest(x:&str, y:&str) -> &str { /// this function's return type contains a borrowed value, but the signature does not say whether it is borrowed from `x` or `y`\n if x.len() > y.len() {\n x\n }else{\n y\n }\n}\n```\nlet's find out why it is such case\n```rust\nfn main() {\n // variable to hold the result value\n let long_str; \n\n let x = \"abc\".to_string();\n {\n let y = \"bbccd\".to_string();\n long_str = longest(x.as_str(), y.as_str());\n }\n // if x.len() > y.len() then it is OK，the long_str variable will hole x; if not, long_str supposed tohold y, however, y has a smaller scope than x, long_str will hold to a dropped value\n println!(\"Longest str: {}\", long_str);\n}\n\n```\nHence, we need lifetime annotation `'`\n```rust\nfn longest<'a>(x:&'a str, y:&'a str) -> &'a str {\n if x.len() > y.len() {\n x\n }else{\n y\n }\n}\n```\n\n### deeper understanding\n- When returning a reference value from a function, the lifetime of the return type needs to match the lifetime of one of the parameters\n\n### Struct lifetime annotation\n```rust\nfn main() {\n let info = String::from(\"File not found.\");\n // 存放结果值的变量\n let exc = ImportantExcepiton {\n part: info.as_str()\n };\n\n println!(\"{:?}\", exc);\n}\n#[derive(Debug)]\nstruct ImportantExcepiton<'a> {\n part: &'a str,\n}\n```\n- lifetime of the field `part` must be longer than struct\n\n### Lifetime Elision\nIn order to make common patterns more ergonomic, Rust allows lifetimes to be elided in function signatures.\nElision rules are as follows:\n- Each elided lifetime in input position becomes a distinct lifetime parameter.\n- If there is exactly one input lifetime position (elided or not), that lifetime is assigned to all elided output lifetimes.\n- If there are multiple input lifetime positions, but one of them is &self or &mut self, the lifetime of self is assigned to all elided output lifetimes.\n- Otherwise, it is an error to elide an output lifetime.\n\n### struct lifetime annotation\n```rust\n\nfn main() {\n let info = String::from(\"File not found.\");\n let exc = ImportantExcepiton {\n part: info.as_str()\n };\n\n println!(\"{:?}\", exc);\n}\n#[derive(Debug)]\nstruct ImportantExcepiton <'a>{\n part: &'a str,\n}\n\n// the first 'a is decraration, the second is usage\nimpl<'a> ImportantExcepiton <'a> {\n // in return value, 'a is omitted according to Lifetime Elision rule\n fn callname(&self ) -> &str{\n self.part\n }\n}\n```\n\n### 'static\n'static is a special lifetime that takes up the duration of the entire program, for example all string literals have a 'static lifetime","source":"_posts/rust/rust-04-lifetime.md","raw":"---\ntitle: rust lifetime\ndate: 2022-10-18 21:33:26\ntags: [rust]\n---\n\n## lifetime\n- Every reference in Rust has its own lifecycle\n- most of the time, Rust's lifetime is implicit and can be inferred\n- There are two types of life cycle: input life cycle and output life cycle\n- 'static is a special life cycle annotation\n### Example of lifetime out of scope\n```rust\nfn main() {\n let mut x;\n {\n let y = String::from(\"hello\");\n // x = y; // this is allowed\n x = &y; // not allowed. borrowed value (y) does not live long enough\n }\n println!(\"Str:{}\", x);\n}\n```\n\n### lifetime checker\nRust compiler's borrow checker to determine whether a borrow is legal\n```rust\n// If it returns a reference value, no matter how simple your function is written, it will always report an error `missing lifetime specifier.`\nfn longest(x:&str, y:&str) -> &str { /// this function's return type contains a borrowed value, but the signature does not say whether it is borrowed from `x` or `y`\n if x.len() > y.len() {\n x\n }else{\n y\n }\n}\n```\nlet's find out why it is such case\n```rust\nfn main() {\n // variable to hold the result value\n let long_str; \n\n let x = \"abc\".to_string();\n {\n let y = \"bbccd\".to_string();\n long_str = longest(x.as_str(), y.as_str());\n }\n // if x.len() > y.len() then it is OK，the long_str variable will hole x; if not, long_str supposed tohold y, however, y has a smaller scope than x, long_str will hold to a dropped value\n println!(\"Longest str: {}\", long_str);\n}\n\n```\nHence, we need lifetime annotation `'`\n```rust\nfn longest<'a>(x:&'a str, y:&'a str) -> &'a str {\n if x.len() > y.len() {\n x\n }else{\n y\n }\n}\n```\n\n### deeper understanding\n- When returning a reference value from a function, the lifetime of the return type needs to match the lifetime of one of the parameters\n\n### Struct lifetime annotation\n```rust\nfn main() {\n let info = String::from(\"File not found.\");\n // 存放结果值的变量\n let exc = ImportantExcepiton {\n part: info.as_str()\n };\n\n println!(\"{:?}\", exc);\n}\n#[derive(Debug)]\nstruct ImportantExcepiton<'a> {\n part: &'a str,\n}\n```\n- lifetime of the field `part` must be longer than struct\n\n### Lifetime Elision\nIn order to make common patterns more ergonomic, Rust allows lifetimes to be elided in function signatures.\nElision rules are as follows:\n- Each elided lifetime in input position becomes a distinct lifetime parameter.\n- If there is exactly one input lifetime position (elided or not), that lifetime is assigned to all elided output lifetimes.\n- If there are multiple input lifetime positions, but one of them is &self or &mut self, the lifetime of self is assigned to all elided output lifetimes.\n- Otherwise, it is an error to elide an output lifetime.\n\n### struct lifetime annotation\n```rust\n\nfn main() {\n let info = String::from(\"File not found.\");\n let exc = ImportantExcepiton {\n part: info.as_str()\n };\n\n println!(\"{:?}\", exc);\n}\n#[derive(Debug)]\nstruct ImportantExcepiton <'a>{\n part: &'a str,\n}\n\n// the first 'a is decraration, the second is usage\nimpl<'a> ImportantExcepiton <'a> {\n // in return value, 'a is omitted according to Lifetime Elision rule\n fn callname(&self ) -> &str{\n self.part\n }\n}\n```\n\n### 'static\n'static is a special lifetime that takes up the duration of the entire program, for example all string literals have a 'static lifetime","slug":"rust/rust-04-lifetime","published":1,"updated":"2023-11-05T04:21:13.734Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8ds0012qwsj7v2de3x4","content":"

lifetime

    \n
  • Every reference in Rust has its own lifecycle
    • \n
  • most of the time, Rust’s lifetime is implicit and can be inferred
    • \n
  • There are two types of life cycle: input life cycle and output life cycle
    • \n
  • ‘static is a special life cycle annotation
    • \n
\n

Example of lifetime out of scope

1
2
3
4
5
6
7
8
9
fn main() {
    let mut x;
    {
        let y = String::from("hello");
        // x = y; // this is allowed
        x = &y; // not allowed. borrowed value (y) does not live long enough
    }
    println!("Str:{}", x);
}
\n\n

lifetime checker

Rust compiler’s borrow checker to determine whether a borrow is legal

\n
1
2
3
4
5
6
7
8
// If it returns a reference value, no matter how simple your function is written, it will always report an error `missing lifetime specifier.`
fn longest(x:&str, y:&str) -> &str { /// this function's return type contains a borrowed value, but the signature does not say whether it is borrowed from `x` or `y`
    if x.len() > y.len() {
        x
    }else{
        y
    }
}
\n

let’s find out why it is such case

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
fn main() {
    // variable to hold the result value
    let long_str; 

    let x = "abc".to_string();
    {
        let y = "bbccd".to_string();
        long_str = longest(x.as_str(), y.as_str());
    }
    // if x.len() > y.len() then it is OK，the long_str variable will hole x; if not, long_str supposed tohold y, however, y has a smaller scope than x, long_str will hold to a dropped value
    println!("Longest str: {}", long_str);
}

\n

Hence, we need lifetime annotation '

\n
1
2
3
4
5
6
7
fn longest<'a>(x:&'a str, y:&'a str) -> &'a str {
    if x.len() > y.len() {
        x
    }else{
        y
    }
}
\n\n

deeper understanding

    \n
  • When returning a reference value from a function, the lifetime of the return type needs to match the lifetime of one of the parameters
    • \n
\n

Struct lifetime annotation

1
2
3
4
5
6
7
8
9
10
11
12
13
fn main() {
    let info = String::from("File not found.");
    // 存放结果值的变量
    let exc = ImportantExcepiton {
        part: info.as_str()
    };

    println!("{:?}", exc);
}
#[derive(Debug)]
struct ImportantExcepiton<'a> {
    part: &'a str,
}
\n
    \n
  • lifetime of the field part must be longer than struct
    • \n
\n

Lifetime Elision

In order to make common patterns more ergonomic, Rust allows lifetimes to be elided in function signatures.
Elision rules are as follows:

\n
    \n
  • Each elided lifetime in input position becomes a distinct lifetime parameter.
    • \n
  • If there is exactly one input lifetime position (elided or not), that lifetime is assigned to all elided output lifetimes.
    • \n
  • If there are multiple input lifetime positions, but one of them is &self or &mut self, the lifetime of self is assigned to all elided output lifetimes.
    • \n
  • Otherwise, it is an error to elide an output lifetime.
    • \n
\n

struct lifetime annotation

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21

fn main() {
    let info = String::from("File not found.");
    let exc = ImportantExcepiton {
        part: info.as_str()
    };

    println!("{:?}", exc);
}
#[derive(Debug)]
struct ImportantExcepiton <'a>{
    part: &'a str,
}

// the first 'a is decraration, the second is usage
impl<'a> ImportantExcepiton <'a> {
    // in return value, 'a is omitted according to Lifetime Elision rule
    fn callname(&self ) -> &str{
        self.part
    }
}
\n\n

‘static

‘static is a special lifetime that takes up the duration of the entire program, for example all string literals have a ‘static lifetime

\n","site":{"data":{}},"excerpt":"","more":"

lifetime

    \n
  • Every reference in Rust has its own lifecycle
    • \n
  • most of the time, Rust’s lifetime is implicit and can be inferred
    • \n
  • There are two types of life cycle: input life cycle and output life cycle
    • \n
  • ‘static is a special life cycle annotation
    • \n
\n

Example of lifetime out of scope

1
2
3
4
5
6
7
8
9
fn main() {
    let mut x;
    {
        let y = String::from("hello");
        // x = y; // this is allowed
        x = &y; // not allowed. borrowed value (y) does not live long enough
    }
    println!("Str:{}", x);
}
\n\n

lifetime checker

Rust compiler’s borrow checker to determine whether a borrow is legal

\n
1
2
3
4
5
6
7
8
// If it returns a reference value, no matter how simple your function is written, it will always report an error `missing lifetime specifier.`
fn longest(x:&str, y:&str) -> &str { /// this function's return type contains a borrowed value, but the signature does not say whether it is borrowed from `x` or `y`
    if x.len() > y.len() {
        x
    }else{
        y
    }
}
\n

let’s find out why it is such case

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
fn main() {
    // variable to hold the result value
    let long_str; 

    let x = "abc".to_string();
    {
        let y = "bbccd".to_string();
        long_str = longest(x.as_str(), y.as_str());
    }
    // if x.len() > y.len() then it is OK，the long_str variable will hole x; if not, long_str supposed tohold y, however, y has a smaller scope than x, long_str will hold to a dropped value
    println!("Longest str: {}", long_str);
}

\n

Hence, we need lifetime annotation '

\n
1
2
3
4
5
6
7
fn longest<'a>(x:&'a str, y:&'a str) -> &'a str {
    if x.len() > y.len() {
        x
    }else{
        y
    }
}
\n\n

deeper understanding

    \n
  • When returning a reference value from a function, the lifetime of the return type needs to match the lifetime of one of the parameters
    • \n
\n

Struct lifetime annotation

1
2
3
4
5
6
7
8
9
10
11
12
13
fn main() {
    let info = String::from("File not found.");
    // 存放结果值的变量
    let exc = ImportantExcepiton {
        part: info.as_str()
    };

    println!("{:?}", exc);
}
#[derive(Debug)]
struct ImportantExcepiton<'a> {
    part: &'a str,
}
\n
    \n
  • lifetime of the field part must be longer than struct
    • \n
\n

Lifetime Elision

In order to make common patterns more ergonomic, Rust allows lifetimes to be elided in function signatures.
Elision rules are as follows:

\n
    \n
  • Each elided lifetime in input position becomes a distinct lifetime parameter.
    • \n
  • If there is exactly one input lifetime position (elided or not), that lifetime is assigned to all elided output lifetimes.
    • \n
  • If there are multiple input lifetime positions, but one of them is &self or &mut self, the lifetime of self is assigned to all elided output lifetimes.
    • \n
  • Otherwise, it is an error to elide an output lifetime.
    • \n
\n

struct lifetime annotation

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21

fn main() {
    let info = String::from("File not found.");
    let exc = ImportantExcepiton {
        part: info.as_str()
    };

    println!("{:?}", exc);
}
#[derive(Debug)]
struct ImportantExcepiton <'a>{
    part: &'a str,
}

// the first 'a is decraration, the second is usage
impl<'a> ImportantExcepiton <'a> {
    // in return value, 'a is omitted according to Lifetime Elision rule
    fn callname(&self ) -> &str{
        self.part
    }
}
\n\n

‘static

‘static is a special lifetime that takes up the duration of the entire program, for example all string literals have a ‘static lifetime

\n"},{"title":"rust smart pointer","date":"2022-10-30T03:00:38.000Z","_content":"\n## Overview\nThe most common kind of pointer in Rust is a reference (borrow but not own)\nSmart pointers, on the other hand, are data structures that not only act like a pointer but also have additional metadata and capabilities.\nreferences are pointers that only borrow data; in contrast, in many cases, smart pointers own the data they point to.\n\n### smart pointers example\n- String\n- Vec\nBoth these types count as smart pointers because they own some memory and allow you to manipulate it. They also have metadata (such as their capacity) and extra capabilities or guarantees (such as with String ensuring its data will always be valid UTF-8).\n\n### Deref & Drop\nSmart pointers are usually implemented using structs. The characteristic that distinguishes a smart pointer from an ordinary struct is that smart pointers implement the `Deref` and `Drop` traits.\n- The Deref trait allows an instance of the smart pointer struct to behave like a reference so you can write code that works with either references or smart pointers. \n- The Drop trait allows you to customize the code that is run when an instance of the smart pointer goes out of scope.\n\n### the most common smart pointers in the standard library:\n- `Box` for allocating values on the heap\n- `Rc`, a reference counting type that enables multiple ownership\n- `Ref` and `RefMut`, accessed through `RefCell`, a type that enforces the borrowing rules at runtime instead of compile time\n\n## Box\n### when to use\n- When you have a type whose size can’t be known at compile time and you want to use a value of that type in a context that requires an exact size. (such as cons)\n- When you have a large amount of data and you want to transfer ownership but ensure the data won’t be copied when you do so\n- When you want to own a value and you care only that it’s a type that implements a particular trait rather than being of a specific type\n\n\n### enabling recursive types with Box\nAt compile time, Rust needs to know how much space a type takes up\nOne type whose size can’t be known at compile time is a recursive type (cons list), use Box, which only contains a memory address\n\n```rust\nenum List {\n Cons(i32, List),\n Nil,\n}\n\nuse crate::List::{Cons, Nil};\nfn main() {\n let list = Cons(1, Cons(2, Cons(3, Nil))); // not allowed, infinite size\n}\n```\n\nuse Box\n```rust\nenum List {\n Cons(i32, Box),\n Nil,\n}\n\nuse crate::List::{Cons, Nil};\n\nfn main() {\n let list = Cons(1, Box::new(Cons(2, Box::new(Cons(3, Box::new(Nil))))));\n}\n```\n\n## Treating Smart Pointers Like Regular References with the Deref Trait\nImplementing the Deref trait allows you to customize the behavior of the dereference operator, `*`\nBy implementing Deref in such a way that a smart pointer can be treated like a regular reference, you can write code that operates on references and use that code with smart pointers too.\n\n### Defining Our Own Smart Pointer\n```rust\nuse std::ops::Deref;\n\nstruct MyBox(T); // The MyBox type is a tuple struct with one element of type T\n\nimpl MyBox {\n fn new(x: T) -> MyBox {\n MyBox(x)\n }\n}\n\nimpl Deref for MyBox {\n type Target = T;\n\n fn deref(&self) -> &Self::Target {\n &self.0\n }\n}\n\nfn main() {\n let x = 5;\n let y = MyBox::new(x);\n\n assert_eq!(5, x);\n assert_eq!(5, *y);\n}\n```\n- We fill in the body of the deref method with &self.0 so deref returns a reference to the value we want to access with the * operator. \n- behind the scenes Rust actually ran this code: *(y.deref()). Rust substitutes the * operator with a call to the deref method\n- The reason the deref method returns a reference to a value, and that the plain dereference outside the parentheses in *(y.deref()) is still necessary, is the ownership system. If the deref method returned the value directly instead of a reference to the value, the value would be moved out of self. We don’t want to take ownership of the inner value inside MyBox in this case or in most cases where we use the dereference operator.\n\n### Implicit Deref Coercions with Functions and Methods\nDeref coercion is a convenience that Rust performs on arguments to functions and methods. \nDeref coercion works only on types that implement the Deref trait. Deref coercion converts a reference to such a type into a reference to another type. For example, deref coercion can convert &String to &str because String implements the Deref trait such that it returns &str\nA sequence of calls to the deref method converts the type we provided into the type the parameter needs.\n\n```rust\nfn hello(name: &str) {\n println!(\"Hello, {}!\", name);\n}\n\nfn main() {\n let m = MyBox::new(String::from(\"Rust\"));\n hello(\"rust\"); // ok\n hello(&m); // also ok\n hello(&(*m)[..]); // without deref coercion. The (*m) dereferences the MyBox into a String. Then the & and [..] take a string slice of the String that is equal to the whole string to match the signature of hello\n}\n```\nHere we’re calling the hello function with the argument &m, which is a reference to a MyBox value. Because we implemented the Deref trait on MyBox, Rust can turn &MyBox into &String by calling deref. The standard library provides an implementation of Deref on String that returns a string slice. Rust calls deref again to turn the &String into &str, which matches the hello function’s definition.\n\n### How Deref Coercion Interacts with Mutability\nSimilar to how you use the Deref trait to override the * operator on immutable references, you can use the DerefMut trait to override the * operator on mutable references.\n\nRust does deref coercion when it finds types and trait implementations in three cases:\n- From &T to &U when T: Deref\n- From &mut T to &mut U when T: DerefMut\n- From &mut T to &U when T: Deref\n\n## Running Code on Cleanup with the Drop Trait\nDrop, which lets you customize what happens when a value is about to go out of scope. \n\n```rust\nstruct CustomSmartPointer {\n data: String,\n}\n\nimpl Drop for CustomSmartPointer {\n fn drop(&mut self) {\n println!(\"Dropping CustomSmartPointer with data `{}`!\", self.data);\n }\n}\n\nfn main() {\n let c = CustomSmartPointer {\n data: String::from(\"my stuff\"),\n };\n c.drop(); // not allowed\n let d = CustomSmartPointer {\n data: String::from(\"other stuff\"),\n };\n println!(\"CustomSmartPointers created.\");\n}\n\n```\n### Dropping a Value Early with std::mem::drop\nstd::mem::drop is in prelude, can use directly\n```rust\nstruct CustomSmartPointer {\n data: String,\n}\n\nimpl Drop for CustomSmartPointer {\n fn drop(&mut self) {\n println!(\"Dropping CustomSmartPointer with data `{}`!\", self.data);\n }\n}\n\nfn main() {\n let c = CustomSmartPointer {\n data: String::from(\"some data\"),\n };\n println!(\"CustomSmartPointer created.\");\n drop(c); // ok\n println!(\"CustomSmartPointer dropped before the end of main.\");\n} // c goes out of scope, will occur double drop\n\n```\n\n\n## Rc: the Reference Counted Smart Pointer\nIn the majority of cases, ownership is clear: you know exactly which variable owns a given value. However, there are cases when a single value might have multiple owners. \ntype keeps track of the number of references to a value to determine whether or not the value is still in use. If there are zero references to a value, the value can be cleaned up without any references becoming invalid.\n\nWe use the `Rc` type when we want to allocate some data on the heap for multiple parts of our program to read and we can’t determine at compile time which part will finish using the data last.\n\nRc is only for use in single-threaded scenarios\n\n### using Rc to share data\n\n![rc](/images/rust/pointers/rc.png)\nimplement use box will not work, as below\n```rust\nenum List {\n Cons(i32, Box),\n Nil,\n}\n\nuse crate::List::{Cons, Nil};\n\nfn main() {\n let a = Cons(5, Box::new(Cons(10, Box::new(Nil))));\n let b = Cons(3, Box::new(a)); // value moved here\n let c = Cons(4, Box::new(a)); // value used here after move\n}\n```\nuse Rc\n```rust\nenum List {\n Cons(i32, Rc),\n Nil,\n}\n\nuse crate::List::{Cons, Nil};\nuse std::rc::Rc;\n\nfn main() {\n let a = Rc::new(Cons(5, Rc::new(Cons(10, Rc::new(Nil)))));\n let b = Cons(3, Rc::clone(&a)); // shalow copy only reference, not data\n let c = Cons(4, Rc::clone(&a));\n}\n```\n\n\nWe could have called a.clone() rather than Rc::clone(&a), but Rust’s convention is to use Rc::clone in this case. The implementation of Rc::clone doesn’t make a deep copy of all the data like most types’ implementations of clone do. The call to Rc::clone only increments the reference count, which doesn’t take much time.\n\n### Cloning an Rc Increases the Reference Count\n```rust\nenum List {\n Cons(i32, Rc),\n Nil,\n}\n\nuse crate::List::{Cons, Nil};\nuse std::rc::Rc;\n\nfn main() {\n let a = Rc::new(Cons(5, Rc::new(Cons(10, Rc::new(Nil)))));\n println!(\"count after creating a = {}\", Rc::strong_count(&a)); // 1\n let b = Cons(3, Rc::clone(&a));\n println!(\"count after creating b = {}\", Rc::strong_count(&a)); // 2\n {\n let c = Cons(4, Rc::clone(&a));\n println!(\"count after creating c = {}\", Rc::strong_count(&a)); // 3\n }\n println!(\"count after c goes out of scope = {}\", Rc::strong_count(&a)); // 2\n}\n```\nWhat we can’t see in this example is that when b and then a go out of scope at the end of main, the count is then 0, and the Rc is cleaned up completely at that point.\n\n## RefCell and interior mutability pattern\nInterior mutability is a design pattern in Rust that allows you to mutate data even when there are immutable references to that data; normally, this action is disallowed by the borrowing rules.\nTo mutate data, the pattern uses unsafe code inside a data structure to bend Rust’s usual rules that govern mutation and borrowing.\nWe can use types that use the interior mutability pattern when we can ensure that the borrowing rules will be followed at runtime, even though the compiler can’t guarantee that. \nThe unsafe code involved is then wrapped in a safe API, and the outer type is still immutable.\n\n### Enforcing Borrowing Rules at Runtime with RefCell\nUnlike Rc, the RefCell type represents single ownership over the data it holds.\n\nrecall borrowing rules\n- At any given time, you can have either (but not both of) one mutable reference or any number of immutable references.\n- References must always be valid.\nWith references and Box, the borrowing rules’ invariants are enforced at compile time. With RefCell, these invariants are enforced at runtime. \nBecause RefCell allows mutable borrows checked at runtime, you can mutate the value inside the RefCell even when the RefCell is immutable.\n\n### Interior Mutability: A Mutable Borrow to an Immutable Value\n```rust\nfn main() {\n let x = 5;\n let y = &mut x; // not allowed. cannot borrow `x` as mutable, as it is not declared as mutable\n}\n```\n\n### A Use Case for Interior Mutability: Mock Objects\n```rust\npub trait Messenger {\n fn send(&self, msg: &str);\n}\n\npub struct LimitTracker<'a, T: Messenger> {\n messenger: &'a T,\n value: usize,\n max: usize,\n}\n\nimpl<'a, T> LimitTracker<'a, T>\nwhere\n T: Messenger,\n{\n pub fn new(messenger: &T, max: usize) -> LimitTracker {\n LimitTracker {\n messenger,\n value: 0,\n max,\n }\n }\n\n pub fn set_value(&mut self, value: usize) {\n self.value = value;\n\n let percentage_of_max = self.value as f64 / self.max as f64;\n\n if percentage_of_max >= 1.0 {\n self.messenger.send(\"Error: You are over your quota!\");\n } else if percentage_of_max >= 0.9 {\n self.messenger\n .send(\"Urgent warning: You've used up over 90% of your quota!\");\n } else if percentage_of_max >= 0.75 {\n self.messenger\n .send(\"Warning: You've used up over 75% of your quota!\");\n }\n }\n}\n\n```\na problematic usage\n```rust\npub trait Messenger {\n fn send(&self, msg: &str);\n}\n\npub struct LimitTracker<'a, T: Messenger> {\n messenger: &'a T,\n value: usize,\n max: usize,\n}\n\nimpl<'a, T> LimitTracker<'a, T>\nwhere\n T: Messenger,\n{\n pub fn new(messenger: &T, max: usize) -> LimitTracker {\n LimitTracker {\n messenger,\n value: 0,\n max,\n }\n }\n\n pub fn set_value(&mut self, value: usize) {\n self.value = value;\n\n let percentage_of_max = self.value as f64 / self.max as f64;\n\n if percentage_of_max >= 1.0 {\n self.messenger.send(\"Error: You are over your quota!\");\n } else if percentage_of_max >= 0.9 {\n self.messenger\n .send(\"Urgent warning: You've used up over 90% of your quota!\");\n } else if percentage_of_max >= 0.75 {\n self.messenger\n .send(\"Warning: You've used up over 75% of your quota!\");\n }\n }\n}\n\n#[cfg(test)]\nmod tests {\n use super::*;\n\n struct MockMessenger {\n sent_messages: Vec,\n }\n\n impl MockMessenger {\n fn new() -> MockMessenger {\n MockMessenger {\n sent_messages: vec![],\n }\n }\n }\n\n impl Messenger for MockMessenger {\n fn send(&self, message: &str) {\n self.sent_messages.push(String::from(message)); // not allowed. cannot borrow `self.sent_messages` as mutable, as it is behind a `&` reference\n }\n }\n\n #[test]\n fn it_sends_an_over_75_percent_warning_message() {\n let mock_messenger = MockMessenger::new();\n let mut limit_tracker = LimitTracker::new(&mock_messenger, 100);\n\n limit_tracker.set_value(80);\n\n assert_eq!(mock_messenger.sent_messages.len(), 1);\n }\n}\n\n```\nWe can’t modify the MockMessenger to keep track of the messages, because the send method takes an immutable reference to self. We also can’t take the suggestion from the error text to use &mut self instead, because then the signature of send wouldn’t match the signature in the Messenger trait definition\n\nThis is a situation in which interior mutability can help! \n```rust\npub trait Messenger {\n fn send(&self, msg: &str);\n}\n\npub struct LimitTracker<'a, T: Messenger> {\n messenger: &'a T,\n value: usize,\n max: usize,\n}\n\nimpl<'a, T> LimitTracker<'a, T>\nwhere\n T: Messenger,\n{\n pub fn new(messenger: &T, max: usize) -> LimitTracker {\n LimitTracker {\n messenger,\n value: 0,\n max,\n }\n }\n\n pub fn set_value(&mut self, value: usize) {\n self.value = value;\n\n let percentage_of_max = self.value as f64 / self.max as f64;\n\n if percentage_of_max >= 1.0 {\n self.messenger.send(\"Error: You are over your quota!\");\n } else if percentage_of_max >= 0.9 {\n self.messenger\n .send(\"Urgent warning: You've used up over 90% of your quota!\");\n } else if percentage_of_max >= 0.75 {\n self.messenger\n .send(\"Warning: You've used up over 75% of your quota!\");\n }\n }\n}\n\n#[cfg(test)]\nmod tests {\n use super::*;\n use std::cell::RefCell;\n\n struct MockMessenger {\n sent_messages: RefCell>,\n }\n\n impl MockMessenger {\n fn new() -> MockMessenger {\n MockMessenger {\n sent_messages: RefCell::new(vec![]),\n }\n }\n }\n\n impl Messenger for MockMessenger {\n fn send(&self, message: &str) {\n self.sent_messages.borrow_mut().push(String::from(message)); // call borrow_mut on the RefCell> in self.sent_messages to get a mutable reference to the value inside the RefCell>\n }\n }\n\n #[test]\n fn it_sends_an_over_75_percent_warning_message() {\n // --snip--\n let mock_messenger = MockMessenger::new();\n let mut limit_tracker = LimitTracker::new(&mock_messenger, 100);\n\n limit_tracker.set_value(80);\n\n assert_eq!(mock_messenger.sent_messages.borrow().len(), 1); // call borrow on the RefCell> to get an immutable reference to the vector.\n }\n}\n```\n\n### Keeping Track of Borrows at Runtime with RefCell\nWhen creating immutable and mutable references, we use the & and &mut syntax, respectively. With RefCell, we use the borrow and borrow_mut methods, which are part of the safe API that belongs to RefCell. **The borrow method returns the smart pointer type Ref, and borrow_mut returns the smart pointer type RefMut**. Both types implement Deref, so we can treat them like regular references.\n\nThe RefCell keeps track of how many Ref and RefMut smart pointers are currently active.RefCell lets us have many immutable borrows or one mutable borrow at any point in time. If we try to violate these rules, rather than getting a compiler error as we would with references, the implementation of RefCell will panic at runtime. \n```rust\nimpl Messenger for MockMessenger {\n fn send(&self, message: &str) {\n let mut one_borrow = self.sent_messages.borrow_mut();\n let mut two_borrow = self.sent_messages.borrow_mut();\n\n one_borrow.push(String::from(message));\n two_borrow.push(String::from(message));\n }\n }\n```\nWhen we run the tests for our library, the code in will compile without any errors, but the test will fail\nthread 'main' panicked at 'already borrowed\n\n### Having Multiple Owners of Mutable Data by Combining Rc and RefCell\nA common way to use RefCell is in combination with Rc. If you have an Rc that holds a RefCell, you can get a value that can have multiple owners and that you can mutate!\n```rust\n#[derive(Debug)]\nenum List {\n Cons(Rc>, Rc),\n Nil,\n}\n\nuse crate::List::{Cons, Nil};\nuse std::cell::RefCell;\nuse std::rc::Rc;\n\nfn main() {\n let value = Rc::new(RefCell::new(5)); // We create a value that is an instance of Rc> and store it in a variable named value so we can access it directly later.\n\n let a = Rc::new(Cons(Rc::clone(&value), Rc::new(Nil))); // Then we create a List in a with a Cons variant that holds value. We need to clone value so both a and value have ownership of the inner 5 value rather than transferring ownership from value to a or having a borrow from value. We wrap the list a in an Rc so when we create lists b and c, they can both refer to a\n\n let b = Cons(Rc::new(RefCell::new(3)), Rc::clone(&a));\n let c = Cons(Rc::new(RefCell::new(4)), Rc::clone(&a));\n\n *value.borrow_mut() += 10; // After we’ve created the lists in a, b, and c, we add 10 to the value in value. We do this by calling borrow_mut on value, which uses the automatic dereferencing feature to dereference the Rc to the inner RefCell value. The borrow_mut method returns a RefMut smart pointer, and we use the dereference operator on it and change the inner value.\n\n println!(\"a after = {:?}\", a);\n println!(\"b after = {:?}\", b);\n println!(\"c after = {:?}\", c);\n}\n```\n\n**The standard library has other types that provide interior mutability, such as Cell, which is similar except that instead of giving references to the inner value, the value is copied in and out of the Cell. There’s also Mutex, which offers interior mutability that’s safe to use across threads;** \n\n## Reference Cycles Can Leak Memory\nRust’s memory safety guarantees make it difficult, but not impossible, to accidentally create memory that is never cleaned up (known as a memory leak).\n\n### Creating a Reference Cycle\n```rust\nuse crate::List::{Cons, Nil};\nuse std::cell::RefCell;\nuse std::rc::Rc;\n\n#[derive(Debug)]\nenum List {\n Cons(i32, RefCell>), // The second element in the Cons variant is now RefCell>, meaning that we want to modify which List value a Cons variant is pointing to. \n Nil,\n}\n\nimpl List {\n fn tail(&self) -> Option<&RefCell>> { // We’re also adding a tail method to make it convenient for us to access the second item if we have a Cons variant.\n match self {\n Cons(_, item) => Some(item),\n Nil => None,\n }\n }\n}\n\nfn main() {\n use crate::List::{Cons, Nil};\nuse std::cell::RefCell;\nuse std::rc::Rc;\n\n#[derive(Debug)]\nenum List {\n Cons(i32, RefCell>),\n Nil,\n}\n\nimpl List {\n fn tail(&self) -> Option<&RefCell>> {\n match self {\n Cons(_, item) => Some(item),\n Nil => None,\n }\n }\n}\n\nfn main() {\n let a = Rc::new(Cons(5, RefCell::new(Rc::new(Nil))));\n\n println!(\"a initial rc count = {}\", Rc::strong_count(&a));\n println!(\"a next item = {:?}\", a.tail());\n\n let b = Rc::new(Cons(10, RefCell::new(Rc::clone(&a)))); // his code creates a list in a and a list in b that points to the list in a\n\n println!(\"a rc count after b creation = {}\", Rc::strong_count(&a));\n println!(\"b initial rc count = {}\", Rc::strong_count(&b));\n println!(\"b next item = {:?}\", b.tail());\n\n if let Some(link) = a.tail() {\n *link.borrow_mut() = Rc::clone(&b); // modifies the list in a to point to b, creating a reference cycle\n }\n\n println!(\"b rc count after changing a = {}\", Rc::strong_count(&b));\n println!(\"a rc count after changing a = {}\", Rc::strong_count(&a));\n\n // Uncomment the next line to see that we have a cycle;\n // it will overflow the stack\n // println!(\"a next item = {:?}\", a.tail());\n} // At the end of main, Rust drops the variable b, which decreases the reference count of the Rc instance from 2 to 1. The memory that Rc has on the heap won’t be dropped at this point, because its reference count is 1, not 0. Then Rust drops a, which decreases the reference count of the a Rc instance from 2 to 1 as well. This instance’s memory can’t be dropped either, because the other Rc instance still refers to it. The memory allocated to the list will remain uncollected forever.\n```\n![cycle-ref](/images/rust/pointers/cycle.ref.png)\n\n### Preventing Reference Cycles: Turning an Rc into a Weak\nSo far, we’ve demonstrated that calling Rc::clone increases the strong_count of an Rc instance, and an Rc instance is only cleaned up if its strong_count is 0. You can also create a weak reference to the value within an Rc instance by calling Rc::downgrade and passing a reference to the Rc. When you call Rc::downgrade, you get a smart pointer of type Weak. Instead of increasing the strong_count in the Rc instance by 1, calling Rc::downgrade increases the weak_count by 1. The Rc type uses weak_count to keep track of how many Weak references exist, similar to strong_count. The difference is the weak_count doesn’t need to be 0 for the Rc instance to be cleaned up.\n\nStrong references are how you can share ownership of an Rc instance. Weak references don’t express an ownership relationship. They won’t cause a reference cycle because any cycle involving some weak references will be broken once the strong reference count of values involved is 0.\n\nBecause the value that Weak references might have been dropped, to do anything with the value that a Weak is pointing to, you must make sure the value still exists. Do this by calling the upgrade method on a Weak instance, which will return an Option>. You’ll get a result of Some if the Rc value has not been dropped yet and a result of None if the Rc value has been dropped. \n\n\n### Creating a Tree Data Structure: a Node with Child Nodes\n```rust\nuse std::cell::RefCell;\nuse std::rc::Rc;\n\n#[derive(Debug)]\nstruct Node {\n value: i32,\n parent: RefCell>, // To make the child node aware of its parent, we need to add a parent field to our Node struct definition. The trouble is in deciding what the type of parent should be. We know it can’t contain an Rc, because that would create a reference cycle with leaf.parent pointing to branch and branch.children pointing to leaf, which would cause their strong_count values to never be 0.\n children: RefCell>>,\n}\n\nfn main() {\n\n let leaf = Rc::new(Node {\n value: 3,\n parent: RefCell::new(Weak::new()),\n children: RefCell::new(vec![]),\n });\n\n println!(\"leaf parent = {:?}\", leaf.parent.borrow().upgrade()); // try to get a reference to the parent of leaf by using the upgrade method, we get a None value. \n\n let branch = Rc::new(Node {\n value: 5,\n parent: RefCell::new(Weak::new()),\n children: RefCell::new(vec![Rc::clone(&leaf)]), // We clone the Rc in leaf and store that in branch, meaning the Node in leaf now has two owners: leaf and branch. \n });\n\n *leaf.parent.borrow_mut() = Rc::downgrade(&branch); // use the Rc::downgrade function to create a Weak reference to branch from the Rc in branch.\n\n println!(\"leaf parent = {:?}\", leaf.parent.borrow().upgrade());\n\n \n}\n```\n\n### Visualizing Changes to strong_count and weak_count\n```rust\nuse std::cell::RefCell;\nuse std::rc::{Rc, Weak};\n\n#[derive(Debug)]\nstruct Node {\n value: i32,\n parent: RefCell>,\n children: RefCell>>,\n}\n\nfn main() {\n let leaf = Rc::new(Node {\n value: 3,\n parent: RefCell::new(Weak::new()),\n children: RefCell::new(vec![]),\n });\n\n println!(\n \"leaf strong = {}, weak = {}\",\n Rc::strong_count(&leaf),\n Rc::weak_count(&leaf),\n );\n\n {\n let branch = Rc::new(Node {\n value: 5,\n parent: RefCell::new(Weak::new()),\n children: RefCell::new(vec![Rc::clone(&leaf)]),\n });\n\n *leaf.parent.borrow_mut() = Rc::downgrade(&branch);\n\n println!(\n \"branch strong = {}, weak = {}\",\n Rc::strong_count(&branch),\n Rc::weak_count(&branch),\n );\n\n println!(\n \"leaf strong = {}, weak = {}\",\n Rc::strong_count(&leaf),\n Rc::weak_count(&leaf),\n );\n }\n\n println!(\"leaf parent = {:?}\", leaf.parent.borrow().upgrade());\n println!(\n \"leaf strong = {}, weak = {}\",\n Rc::strong_count(&leaf),\n Rc::weak_count(&leaf),\n );\n}\n```","source":"_posts/rust/rust-06-smart-pointer.md","raw":"---\ntitle: rust smart pointer\ndate: 2022-10-30 11:00:38\ntags: [rust]\n---\n\n## Overview\nThe most common kind of pointer in Rust is a reference (borrow but not own)\nSmart pointers, on the other hand, are data structures that not only act like a pointer but also have additional metadata and capabilities.\nreferences are pointers that only borrow data; in contrast, in many cases, smart pointers own the data they point to.\n\n### smart pointers example\n- String\n- Vec\nBoth these types count as smart pointers because they own some memory and allow you to manipulate it. They also have metadata (such as their capacity) and extra capabilities or guarantees (such as with String ensuring its data will always be valid UTF-8).\n\n### Deref & Drop\nSmart pointers are usually implemented using structs. The characteristic that distinguishes a smart pointer from an ordinary struct is that smart pointers implement the `Deref` and `Drop` traits.\n- The Deref trait allows an instance of the smart pointer struct to behave like a reference so you can write code that works with either references or smart pointers. \n- The Drop trait allows you to customize the code that is run when an instance of the smart pointer goes out of scope.\n\n### the most common smart pointers in the standard library:\n- `Box` for allocating values on the heap\n- `Rc`, a reference counting type that enables multiple ownership\n- `Ref` and `RefMut`, accessed through `RefCell`, a type that enforces the borrowing rules at runtime instead of compile time\n\n## Box\n### when to use\n- When you have a type whose size can’t be known at compile time and you want to use a value of that type in a context that requires an exact size. (such as cons)\n- When you have a large amount of data and you want to transfer ownership but ensure the data won’t be copied when you do so\n- When you want to own a value and you care only that it’s a type that implements a particular trait rather than being of a specific type\n\n\n### enabling recursive types with Box\nAt compile time, Rust needs to know how much space a type takes up\nOne type whose size can’t be known at compile time is a recursive type (cons list), use Box, which only contains a memory address\n\n```rust\nenum List {\n Cons(i32, List),\n Nil,\n}\n\nuse crate::List::{Cons, Nil};\nfn main() {\n let list = Cons(1, Cons(2, Cons(3, Nil))); // not allowed, infinite size\n}\n```\n\nuse Box\n```rust\nenum List {\n Cons(i32, Box),\n Nil,\n}\n\nuse crate::List::{Cons, Nil};\n\nfn main() {\n let list = Cons(1, Box::new(Cons(2, Box::new(Cons(3, Box::new(Nil))))));\n}\n```\n\n## Treating Smart Pointers Like Regular References with the Deref Trait\nImplementing the Deref trait allows you to customize the behavior of the dereference operator, `*`\nBy implementing Deref in such a way that a smart pointer can be treated like a regular reference, you can write code that operates on references and use that code with smart pointers too.\n\n### Defining Our Own Smart Pointer\n```rust\nuse std::ops::Deref;\n\nstruct MyBox(T); // The MyBox type is a tuple struct with one element of type T\n\nimpl MyBox {\n fn new(x: T) -> MyBox {\n MyBox(x)\n }\n}\n\nimpl Deref for MyBox {\n type Target = T;\n\n fn deref(&self) -> &Self::Target {\n &self.0\n }\n}\n\nfn main() {\n let x = 5;\n let y = MyBox::new(x);\n\n assert_eq!(5, x);\n assert_eq!(5, *y);\n}\n```\n- We fill in the body of the deref method with &self.0 so deref returns a reference to the value we want to access with the * operator. \n- behind the scenes Rust actually ran this code: *(y.deref()). Rust substitutes the * operator with a call to the deref method\n- The reason the deref method returns a reference to a value, and that the plain dereference outside the parentheses in *(y.deref()) is still necessary, is the ownership system. If the deref method returned the value directly instead of a reference to the value, the value would be moved out of self. We don’t want to take ownership of the inner value inside MyBox in this case or in most cases where we use the dereference operator.\n\n### Implicit Deref Coercions with Functions and Methods\nDeref coercion is a convenience that Rust performs on arguments to functions and methods. \nDeref coercion works only on types that implement the Deref trait. Deref coercion converts a reference to such a type into a reference to another type. For example, deref coercion can convert &String to &str because String implements the Deref trait such that it returns &str\nA sequence of calls to the deref method converts the type we provided into the type the parameter needs.\n\n```rust\nfn hello(name: &str) {\n println!(\"Hello, {}!\", name);\n}\n\nfn main() {\n let m = MyBox::new(String::from(\"Rust\"));\n hello(\"rust\"); // ok\n hello(&m); // also ok\n hello(&(*m)[..]); // without deref coercion. The (*m) dereferences the MyBox into a String. Then the & and [..] take a string slice of the String that is equal to the whole string to match the signature of hello\n}\n```\nHere we’re calling the hello function with the argument &m, which is a reference to a MyBox value. Because we implemented the Deref trait on MyBox, Rust can turn &MyBox into &String by calling deref. The standard library provides an implementation of Deref on String that returns a string slice. Rust calls deref again to turn the &String into &str, which matches the hello function’s definition.\n\n### How Deref Coercion Interacts with Mutability\nSimilar to how you use the Deref trait to override the * operator on immutable references, you can use the DerefMut trait to override the * operator on mutable references.\n\nRust does deref coercion when it finds types and trait implementations in three cases:\n- From &T to &U when T: Deref\n- From &mut T to &mut U when T: DerefMut\n- From &mut T to &U when T: Deref\n\n## Running Code on Cleanup with the Drop Trait\nDrop, which lets you customize what happens when a value is about to go out of scope. \n\n```rust\nstruct CustomSmartPointer {\n data: String,\n}\n\nimpl Drop for CustomSmartPointer {\n fn drop(&mut self) {\n println!(\"Dropping CustomSmartPointer with data `{}`!\", self.data);\n }\n}\n\nfn main() {\n let c = CustomSmartPointer {\n data: String::from(\"my stuff\"),\n };\n c.drop(); // not allowed\n let d = CustomSmartPointer {\n data: String::from(\"other stuff\"),\n };\n println!(\"CustomSmartPointers created.\");\n}\n\n```\n### Dropping a Value Early with std::mem::drop\nstd::mem::drop is in prelude, can use directly\n```rust\nstruct CustomSmartPointer {\n data: String,\n}\n\nimpl Drop for CustomSmartPointer {\n fn drop(&mut self) {\n println!(\"Dropping CustomSmartPointer with data `{}`!\", self.data);\n }\n}\n\nfn main() {\n let c = CustomSmartPointer {\n data: String::from(\"some data\"),\n };\n println!(\"CustomSmartPointer created.\");\n drop(c); // ok\n println!(\"CustomSmartPointer dropped before the end of main.\");\n} // c goes out of scope, will occur double drop\n\n```\n\n\n## Rc: the Reference Counted Smart Pointer\nIn the majority of cases, ownership is clear: you know exactly which variable owns a given value. However, there are cases when a single value might have multiple owners. \ntype keeps track of the number of references to a value to determine whether or not the value is still in use. If there are zero references to a value, the value can be cleaned up without any references becoming invalid.\n\nWe use the `Rc` type when we want to allocate some data on the heap for multiple parts of our program to read and we can’t determine at compile time which part will finish using the data last.\n\nRc is only for use in single-threaded scenarios\n\n### using Rc to share data\n\n![rc](/images/rust/pointers/rc.png)\nimplement use box will not work, as below\n```rust\nenum List {\n Cons(i32, Box),\n Nil,\n}\n\nuse crate::List::{Cons, Nil};\n\nfn main() {\n let a = Cons(5, Box::new(Cons(10, Box::new(Nil))));\n let b = Cons(3, Box::new(a)); // value moved here\n let c = Cons(4, Box::new(a)); // value used here after move\n}\n```\nuse Rc\n```rust\nenum List {\n Cons(i32, Rc),\n Nil,\n}\n\nuse crate::List::{Cons, Nil};\nuse std::rc::Rc;\n\nfn main() {\n let a = Rc::new(Cons(5, Rc::new(Cons(10, Rc::new(Nil)))));\n let b = Cons(3, Rc::clone(&a)); // shalow copy only reference, not data\n let c = Cons(4, Rc::clone(&a));\n}\n```\n\n\nWe could have called a.clone() rather than Rc::clone(&a), but Rust’s convention is to use Rc::clone in this case. The implementation of Rc::clone doesn’t make a deep copy of all the data like most types’ implementations of clone do. The call to Rc::clone only increments the reference count, which doesn’t take much time.\n\n### Cloning an Rc Increases the Reference Count\n```rust\nenum List {\n Cons(i32, Rc),\n Nil,\n}\n\nuse crate::List::{Cons, Nil};\nuse std::rc::Rc;\n\nfn main() {\n let a = Rc::new(Cons(5, Rc::new(Cons(10, Rc::new(Nil)))));\n println!(\"count after creating a = {}\", Rc::strong_count(&a)); // 1\n let b = Cons(3, Rc::clone(&a));\n println!(\"count after creating b = {}\", Rc::strong_count(&a)); // 2\n {\n let c = Cons(4, Rc::clone(&a));\n println!(\"count after creating c = {}\", Rc::strong_count(&a)); // 3\n }\n println!(\"count after c goes out of scope = {}\", Rc::strong_count(&a)); // 2\n}\n```\nWhat we can’t see in this example is that when b and then a go out of scope at the end of main, the count is then 0, and the Rc is cleaned up completely at that point.\n\n## RefCell and interior mutability pattern\nInterior mutability is a design pattern in Rust that allows you to mutate data even when there are immutable references to that data; normally, this action is disallowed by the borrowing rules.\nTo mutate data, the pattern uses unsafe code inside a data structure to bend Rust’s usual rules that govern mutation and borrowing.\nWe can use types that use the interior mutability pattern when we can ensure that the borrowing rules will be followed at runtime, even though the compiler can’t guarantee that. \nThe unsafe code involved is then wrapped in a safe API, and the outer type is still immutable.\n\n### Enforcing Borrowing Rules at Runtime with RefCell\nUnlike Rc, the RefCell type represents single ownership over the data it holds.\n\nrecall borrowing rules\n- At any given time, you can have either (but not both of) one mutable reference or any number of immutable references.\n- References must always be valid.\nWith references and Box, the borrowing rules’ invariants are enforced at compile time. With RefCell, these invariants are enforced at runtime. \nBecause RefCell allows mutable borrows checked at runtime, you can mutate the value inside the RefCell even when the RefCell is immutable.\n\n### Interior Mutability: A Mutable Borrow to an Immutable Value\n```rust\nfn main() {\n let x = 5;\n let y = &mut x; // not allowed. cannot borrow `x` as mutable, as it is not declared as mutable\n}\n```\n\n### A Use Case for Interior Mutability: Mock Objects\n```rust\npub trait Messenger {\n fn send(&self, msg: &str);\n}\n\npub struct LimitTracker<'a, T: Messenger> {\n messenger: &'a T,\n value: usize,\n max: usize,\n}\n\nimpl<'a, T> LimitTracker<'a, T>\nwhere\n T: Messenger,\n{\n pub fn new(messenger: &T, max: usize) -> LimitTracker {\n LimitTracker {\n messenger,\n value: 0,\n max,\n }\n }\n\n pub fn set_value(&mut self, value: usize) {\n self.value = value;\n\n let percentage_of_max = self.value as f64 / self.max as f64;\n\n if percentage_of_max >= 1.0 {\n self.messenger.send(\"Error: You are over your quota!\");\n } else if percentage_of_max >= 0.9 {\n self.messenger\n .send(\"Urgent warning: You've used up over 90% of your quota!\");\n } else if percentage_of_max >= 0.75 {\n self.messenger\n .send(\"Warning: You've used up over 75% of your quota!\");\n }\n }\n}\n\n```\na problematic usage\n```rust\npub trait Messenger {\n fn send(&self, msg: &str);\n}\n\npub struct LimitTracker<'a, T: Messenger> {\n messenger: &'a T,\n value: usize,\n max: usize,\n}\n\nimpl<'a, T> LimitTracker<'a, T>\nwhere\n T: Messenger,\n{\n pub fn new(messenger: &T, max: usize) -> LimitTracker {\n LimitTracker {\n messenger,\n value: 0,\n max,\n }\n }\n\n pub fn set_value(&mut self, value: usize) {\n self.value = value;\n\n let percentage_of_max = self.value as f64 / self.max as f64;\n\n if percentage_of_max >= 1.0 {\n self.messenger.send(\"Error: You are over your quota!\");\n } else if percentage_of_max >= 0.9 {\n self.messenger\n .send(\"Urgent warning: You've used up over 90% of your quota!\");\n } else if percentage_of_max >= 0.75 {\n self.messenger\n .send(\"Warning: You've used up over 75% of your quota!\");\n }\n }\n}\n\n#[cfg(test)]\nmod tests {\n use super::*;\n\n struct MockMessenger {\n sent_messages: Vec,\n }\n\n impl MockMessenger {\n fn new() -> MockMessenger {\n MockMessenger {\n sent_messages: vec![],\n }\n }\n }\n\n impl Messenger for MockMessenger {\n fn send(&self, message: &str) {\n self.sent_messages.push(String::from(message)); // not allowed. cannot borrow `self.sent_messages` as mutable, as it is behind a `&` reference\n }\n }\n\n #[test]\n fn it_sends_an_over_75_percent_warning_message() {\n let mock_messenger = MockMessenger::new();\n let mut limit_tracker = LimitTracker::new(&mock_messenger, 100);\n\n limit_tracker.set_value(80);\n\n assert_eq!(mock_messenger.sent_messages.len(), 1);\n }\n}\n\n```\nWe can’t modify the MockMessenger to keep track of the messages, because the send method takes an immutable reference to self. We also can’t take the suggestion from the error text to use &mut self instead, because then the signature of send wouldn’t match the signature in the Messenger trait definition\n\nThis is a situation in which interior mutability can help! \n```rust\npub trait Messenger {\n fn send(&self, msg: &str);\n}\n\npub struct LimitTracker<'a, T: Messenger> {\n messenger: &'a T,\n value: usize,\n max: usize,\n}\n\nimpl<'a, T> LimitTracker<'a, T>\nwhere\n T: Messenger,\n{\n pub fn new(messenger: &T, max: usize) -> LimitTracker {\n LimitTracker {\n messenger,\n value: 0,\n max,\n }\n }\n\n pub fn set_value(&mut self, value: usize) {\n self.value = value;\n\n let percentage_of_max = self.value as f64 / self.max as f64;\n\n if percentage_of_max >= 1.0 {\n self.messenger.send(\"Error: You are over your quota!\");\n } else if percentage_of_max >= 0.9 {\n self.messenger\n .send(\"Urgent warning: You've used up over 90% of your quota!\");\n } else if percentage_of_max >= 0.75 {\n self.messenger\n .send(\"Warning: You've used up over 75% of your quota!\");\n }\n }\n}\n\n#[cfg(test)]\nmod tests {\n use super::*;\n use std::cell::RefCell;\n\n struct MockMessenger {\n sent_messages: RefCell>,\n }\n\n impl MockMessenger {\n fn new() -> MockMessenger {\n MockMessenger {\n sent_messages: RefCell::new(vec![]),\n }\n }\n }\n\n impl Messenger for MockMessenger {\n fn send(&self, message: &str) {\n self.sent_messages.borrow_mut().push(String::from(message)); // call borrow_mut on the RefCell> in self.sent_messages to get a mutable reference to the value inside the RefCell>\n }\n }\n\n #[test]\n fn it_sends_an_over_75_percent_warning_message() {\n // --snip--\n let mock_messenger = MockMessenger::new();\n let mut limit_tracker = LimitTracker::new(&mock_messenger, 100);\n\n limit_tracker.set_value(80);\n\n assert_eq!(mock_messenger.sent_messages.borrow().len(), 1); // call borrow on the RefCell> to get an immutable reference to the vector.\n }\n}\n```\n\n### Keeping Track of Borrows at Runtime with RefCell\nWhen creating immutable and mutable references, we use the & and &mut syntax, respectively. With RefCell, we use the borrow and borrow_mut methods, which are part of the safe API that belongs to RefCell. **The borrow method returns the smart pointer type Ref, and borrow_mut returns the smart pointer type RefMut**. Both types implement Deref, so we can treat them like regular references.\n\nThe RefCell keeps track of how many Ref and RefMut smart pointers are currently active.RefCell lets us have many immutable borrows or one mutable borrow at any point in time. If we try to violate these rules, rather than getting a compiler error as we would with references, the implementation of RefCell will panic at runtime. \n```rust\nimpl Messenger for MockMessenger {\n fn send(&self, message: &str) {\n let mut one_borrow = self.sent_messages.borrow_mut();\n let mut two_borrow = self.sent_messages.borrow_mut();\n\n one_borrow.push(String::from(message));\n two_borrow.push(String::from(message));\n }\n }\n```\nWhen we run the tests for our library, the code in will compile without any errors, but the test will fail\nthread 'main' panicked at 'already borrowed\n\n### Having Multiple Owners of Mutable Data by Combining Rc and RefCell\nA common way to use RefCell is in combination with Rc. If you have an Rc that holds a RefCell, you can get a value that can have multiple owners and that you can mutate!\n```rust\n#[derive(Debug)]\nenum List {\n Cons(Rc>, Rc),\n Nil,\n}\n\nuse crate::List::{Cons, Nil};\nuse std::cell::RefCell;\nuse std::rc::Rc;\n\nfn main() {\n let value = Rc::new(RefCell::new(5)); // We create a value that is an instance of Rc> and store it in a variable named value so we can access it directly later.\n\n let a = Rc::new(Cons(Rc::clone(&value), Rc::new(Nil))); // Then we create a List in a with a Cons variant that holds value. We need to clone value so both a and value have ownership of the inner 5 value rather than transferring ownership from value to a or having a borrow from value. We wrap the list a in an Rc so when we create lists b and c, they can both refer to a\n\n let b = Cons(Rc::new(RefCell::new(3)), Rc::clone(&a));\n let c = Cons(Rc::new(RefCell::new(4)), Rc::clone(&a));\n\n *value.borrow_mut() += 10; // After we’ve created the lists in a, b, and c, we add 10 to the value in value. We do this by calling borrow_mut on value, which uses the automatic dereferencing feature to dereference the Rc to the inner RefCell value. The borrow_mut method returns a RefMut smart pointer, and we use the dereference operator on it and change the inner value.\n\n println!(\"a after = {:?}\", a);\n println!(\"b after = {:?}\", b);\n println!(\"c after = {:?}\", c);\n}\n```\n\n**The standard library has other types that provide interior mutability, such as Cell, which is similar except that instead of giving references to the inner value, the value is copied in and out of the Cell. There’s also Mutex, which offers interior mutability that’s safe to use across threads;** \n\n## Reference Cycles Can Leak Memory\nRust’s memory safety guarantees make it difficult, but not impossible, to accidentally create memory that is never cleaned up (known as a memory leak).\n\n### Creating a Reference Cycle\n```rust\nuse crate::List::{Cons, Nil};\nuse std::cell::RefCell;\nuse std::rc::Rc;\n\n#[derive(Debug)]\nenum List {\n Cons(i32, RefCell>), // The second element in the Cons variant is now RefCell>, meaning that we want to modify which List value a Cons variant is pointing to. \n Nil,\n}\n\nimpl List {\n fn tail(&self) -> Option<&RefCell>> { // We’re also adding a tail method to make it convenient for us to access the second item if we have a Cons variant.\n match self {\n Cons(_, item) => Some(item),\n Nil => None,\n }\n }\n}\n\nfn main() {\n use crate::List::{Cons, Nil};\nuse std::cell::RefCell;\nuse std::rc::Rc;\n\n#[derive(Debug)]\nenum List {\n Cons(i32, RefCell>),\n Nil,\n}\n\nimpl List {\n fn tail(&self) -> Option<&RefCell>> {\n match self {\n Cons(_, item) => Some(item),\n Nil => None,\n }\n }\n}\n\nfn main() {\n let a = Rc::new(Cons(5, RefCell::new(Rc::new(Nil))));\n\n println!(\"a initial rc count = {}\", Rc::strong_count(&a));\n println!(\"a next item = {:?}\", a.tail());\n\n let b = Rc::new(Cons(10, RefCell::new(Rc::clone(&a)))); // his code creates a list in a and a list in b that points to the list in a\n\n println!(\"a rc count after b creation = {}\", Rc::strong_count(&a));\n println!(\"b initial rc count = {}\", Rc::strong_count(&b));\n println!(\"b next item = {:?}\", b.tail());\n\n if let Some(link) = a.tail() {\n *link.borrow_mut() = Rc::clone(&b); // modifies the list in a to point to b, creating a reference cycle\n }\n\n println!(\"b rc count after changing a = {}\", Rc::strong_count(&b));\n println!(\"a rc count after changing a = {}\", Rc::strong_count(&a));\n\n // Uncomment the next line to see that we have a cycle;\n // it will overflow the stack\n // println!(\"a next item = {:?}\", a.tail());\n} // At the end of main, Rust drops the variable b, which decreases the reference count of the Rc instance from 2 to 1. The memory that Rc has on the heap won’t be dropped at this point, because its reference count is 1, not 0. Then Rust drops a, which decreases the reference count of the a Rc instance from 2 to 1 as well. This instance’s memory can’t be dropped either, because the other Rc instance still refers to it. The memory allocated to the list will remain uncollected forever.\n```\n![cycle-ref](/images/rust/pointers/cycle.ref.png)\n\n### Preventing Reference Cycles: Turning an Rc into a Weak\nSo far, we’ve demonstrated that calling Rc::clone increases the strong_count of an Rc instance, and an Rc instance is only cleaned up if its strong_count is 0. You can also create a weak reference to the value within an Rc instance by calling Rc::downgrade and passing a reference to the Rc. When you call Rc::downgrade, you get a smart pointer of type Weak. Instead of increasing the strong_count in the Rc instance by 1, calling Rc::downgrade increases the weak_count by 1. The Rc type uses weak_count to keep track of how many Weak references exist, similar to strong_count. The difference is the weak_count doesn’t need to be 0 for the Rc instance to be cleaned up.\n\nStrong references are how you can share ownership of an Rc instance. Weak references don’t express an ownership relationship. They won’t cause a reference cycle because any cycle involving some weak references will be broken once the strong reference count of values involved is 0.\n\nBecause the value that Weak references might have been dropped, to do anything with the value that a Weak is pointing to, you must make sure the value still exists. Do this by calling the upgrade method on a Weak instance, which will return an Option>. You’ll get a result of Some if the Rc value has not been dropped yet and a result of None if the Rc value has been dropped. \n\n\n### Creating a Tree Data Structure: a Node with Child Nodes\n```rust\nuse std::cell::RefCell;\nuse std::rc::Rc;\n\n#[derive(Debug)]\nstruct Node {\n value: i32,\n parent: RefCell>, // To make the child node aware of its parent, we need to add a parent field to our Node struct definition. The trouble is in deciding what the type of parent should be. We know it can’t contain an Rc, because that would create a reference cycle with leaf.parent pointing to branch and branch.children pointing to leaf, which would cause their strong_count values to never be 0.\n children: RefCell>>,\n}\n\nfn main() {\n\n let leaf = Rc::new(Node {\n value: 3,\n parent: RefCell::new(Weak::new()),\n children: RefCell::new(vec![]),\n });\n\n println!(\"leaf parent = {:?}\", leaf.parent.borrow().upgrade()); // try to get a reference to the parent of leaf by using the upgrade method, we get a None value. \n\n let branch = Rc::new(Node {\n value: 5,\n parent: RefCell::new(Weak::new()),\n children: RefCell::new(vec![Rc::clone(&leaf)]), // We clone the Rc in leaf and store that in branch, meaning the Node in leaf now has two owners: leaf and branch. \n });\n\n *leaf.parent.borrow_mut() = Rc::downgrade(&branch); // use the Rc::downgrade function to create a Weak reference to branch from the Rc in branch.\n\n println!(\"leaf parent = {:?}\", leaf.parent.borrow().upgrade());\n\n \n}\n```\n\n### Visualizing Changes to strong_count and weak_count\n```rust\nuse std::cell::RefCell;\nuse std::rc::{Rc, Weak};\n\n#[derive(Debug)]\nstruct Node {\n value: i32,\n parent: RefCell>,\n children: RefCell>>,\n}\n\nfn main() {\n let leaf = Rc::new(Node {\n value: 3,\n parent: RefCell::new(Weak::new()),\n children: RefCell::new(vec![]),\n });\n\n println!(\n \"leaf strong = {}, weak = {}\",\n Rc::strong_count(&leaf),\n Rc::weak_count(&leaf),\n );\n\n {\n let branch = Rc::new(Node {\n value: 5,\n parent: RefCell::new(Weak::new()),\n children: RefCell::new(vec![Rc::clone(&leaf)]),\n });\n\n *leaf.parent.borrow_mut() = Rc::downgrade(&branch);\n\n println!(\n \"branch strong = {}, weak = {}\",\n Rc::strong_count(&branch),\n Rc::weak_count(&branch),\n );\n\n println!(\n \"leaf strong = {}, weak = {}\",\n Rc::strong_count(&leaf),\n Rc::weak_count(&leaf),\n );\n }\n\n println!(\"leaf parent = {:?}\", leaf.parent.borrow().upgrade());\n println!(\n \"leaf strong = {}, weak = {}\",\n Rc::strong_count(&leaf),\n Rc::weak_count(&leaf),\n );\n}\n```","slug":"rust/rust-06-smart-pointer","published":1,"updated":"2023-11-05T04:21:13.734Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8ds0014qwsjeg0uevgt","content":"

Overview

The most common kind of pointer in Rust is a reference (borrow but not own)
Smart pointers, on the other hand, are data structures that not only act like a pointer but also have additional metadata and capabilities.
references are pointers that only borrow data; in contrast, in many cases, smart pointers own the data they point to.

\n

smart pointers example

    \n
  • String
    • \n
  • Vec
    Both these types count as smart pointers because they own some memory and allow you to manipulate it. They also have metadata (such as their capacity) and extra capabilities or guarantees (such as with String ensuring its data will always be valid UTF-8).
    • \n
\n

Deref & Drop

Smart pointers are usually implemented using structs. The characteristic that distinguishes a smart pointer from an ordinary struct is that smart pointers implement the Deref and Drop traits.

\n
    \n
  • The Deref trait allows an instance of the smart pointer struct to behave like a reference so you can write code that works with either references or smart pointers.
    • \n
  • The Drop trait allows you to customize the code that is run when an instance of the smart pointer goes out of scope.
    • \n
\n

the most common smart pointers in the standard library:

    \n
  • Box<T> for allocating values on the heap
    • \n
  • Rc<T>, a reference counting type that enables multiple ownership
    • \n
  • Ref<T> and RefMut<T>, accessed through RefCell<T>, a type that enforces the borrowing rules at runtime instead of compile time
    • \n
\n

Box

when to use

    \n
  • When you have a type whose size can’t be known at compile time and you want to use a value of that type in a context that requires an exact size. (such as cons)
    • \n
  • When you have a large amount of data and you want to transfer ownership but ensure the data won’t be copied when you do so
    • \n
  • When you want to own a value and you care only that it’s a type that implements a particular trait rather than being of a specific type
    • \n
\n

enabling recursive types with Box

At compile time, Rust needs to know how much space a type takes up
One type whose size can’t be known at compile time is a recursive type (cons list), use Box, which only contains a memory address

\n
1
2
3
4
5
6
7
8
9
enum List {
    Cons(i32, List),
    Nil,
}

use crate::List::{Cons, Nil};
fn main() {
    let list = Cons(1, Cons(2, Cons(3, Nil))); // not allowed, infinite size
}
\n\n

use Box

\n
1
2
3
4
5
6
7
8
9
10
enum List {
    Cons(i32, Box<List>),
    Nil,
}

use crate::List::{Cons, Nil};

fn main() {
    let list = Cons(1, Box::new(Cons(2, Box::new(Cons(3, Box::new(Nil))))));
}
\n\n

Treating Smart Pointers Like Regular References with the Deref Trait

Implementing the Deref trait allows you to customize the behavior of the dereference operator, *
By implementing Deref in such a way that a smart pointer can be treated like a regular reference, you can write code that operates on references and use that code with smart pointers too.

\n

Defining Our Own Smart Pointer

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
use std::ops::Deref;

struct MyBox<T>(T);  // The MyBox type is a tuple struct with one element of type T

impl<T> MyBox<T> {
    fn new(x: T) -> MyBox<T> {
        MyBox(x)
    }
}

impl<T> Deref for MyBox<T> {
    type Target = T;

    fn deref(&self) -> &Self::Target {
        &self.0
    }
}

fn main() {
    let x = 5;
    let y = MyBox::new(x);

    assert_eq!(5, x);
    assert_eq!(5, *y);
}
\n
    \n
  • We fill in the body of the deref method with &self.0 so deref returns a reference to the value we want to access with the * operator.
    • \n
  • behind the scenes Rust actually ran this code: *(y.deref()). Rust substitutes the * operator with a call to the deref method
    • \n
  • The reason the deref method returns a reference to a value, and that the plain dereference outside the parentheses in *(y.deref()) is still necessary, is the ownership system. If the deref method returned the value directly instead of a reference to the value, the value would be moved out of self. We don’t want to take ownership of the inner value inside MyBox in this case or in most cases where we use the dereference operator.
    • \n
\n

Implicit Deref Coercions with Functions and Methods

Deref coercion is a convenience that Rust performs on arguments to functions and methods.
Deref coercion works only on types that implement the Deref trait. Deref coercion converts a reference to such a type into a reference to another type. For example, deref coercion can convert &String to &str because String implements the Deref trait such that it returns &str
A sequence of calls to the deref method converts the type we provided into the type the parameter needs.

\n
1
2
3
4
5
6
7
8
9
10
fn hello(name: &str) {
    println!("Hello, {}!", name);
}

fn main() {
    let m = MyBox::new(String::from("Rust"));
    hello("rust");  // ok
    hello(&m);      // also ok
    hello(&(*m)[..]); // without deref coercion. The (*m) dereferences the MyBox<String> into a String. Then the & and [..] take a string slice of the String that is equal to the whole string to match the signature of hello
}
\n

Here we’re calling the hello function with the argument &m, which is a reference to a MyBox value. Because we implemented the Deref trait on MyBox, Rust can turn &MyBox into &String by calling deref. The standard library provides an implementation of Deref on String that returns a string slice. Rust calls deref again to turn the &String into &str, which matches the hello function’s definition.

\n

How Deref Coercion Interacts with Mutability

Similar to how you use the Deref trait to override the * operator on immutable references, you can use the DerefMut trait to override the * operator on mutable references.

\n

Rust does deref coercion when it finds types and trait implementations in three cases:

\n
    \n
  • From &T to &U when T: Deref<Target=U>
    • \n
  • From &mut T to &mut U when T: DerefMut<Target=U>
    • \n
  • From &mut T to &U when T: Deref<Target=U>
    • \n
\n

Running Code on Cleanup with the Drop Trait

Drop, which lets you customize what happens when a value is about to go out of scope.

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
struct CustomSmartPointer {
    data: String,
}

impl Drop for CustomSmartPointer {
    fn drop(&mut self) {
        println!("Dropping CustomSmartPointer with data `{}`!", self.data);
    }
}

fn main() {
    let c = CustomSmartPointer {
        data: String::from("my stuff"),
    };
    c.drop(); // not allowed
    let d = CustomSmartPointer {
        data: String::from("other stuff"),
    };
    println!("CustomSmartPointers created.");
}

\n

Dropping a Value Early with std::mem::drop

std::mem::drop is in prelude, can use directly

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
struct CustomSmartPointer {
    data: String,
}

impl Drop for CustomSmartPointer {
    fn drop(&mut self) {
        println!("Dropping CustomSmartPointer with data `{}`!", self.data);
    }
}

fn main() {
    let c = CustomSmartPointer {
        data: String::from("some data"),
    };
    println!("CustomSmartPointer created.");
    drop(c); // ok
    println!("CustomSmartPointer dropped before the end of main.");
} // c goes out of scope, will occur double drop

\n\n\n

Rc: the Reference Counted Smart Pointer

In the majority of cases, ownership is clear: you know exactly which variable owns a given value. However, there are cases when a single value might have multiple owners.
type keeps track of the number of references to a value to determine whether or not the value is still in use. If there are zero references to a value, the value can be cleaned up without any references becoming invalid.

\n

We use the Rc<T> type when we want to allocate some data on the heap for multiple parts of our program to read and we can’t determine at compile time which part will finish using the data last.

\n

Rc is only for use in single-threaded scenarios

\n

using Rc to share data

\"rc\"
implement use box will not work, as below

\n
1
2
3
4
5
6
7
8
9
10
11
12
enum List {
    Cons(i32, Box<List>),
    Nil,
}

use crate::List::{Cons, Nil};

fn main() {
    let a = Cons(5, Box::new(Cons(10, Box::new(Nil))));
    let b = Cons(3, Box::new(a)); // value moved here
    let c = Cons(4, Box::new(a)); // value used here after move
}
\n

use Rc

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
enum List {
    Cons(i32, Rc<List>),
    Nil,
}

use crate::List::{Cons, Nil};
use std::rc::Rc;

fn main() {
    let a = Rc::new(Cons(5, Rc::new(Cons(10, Rc::new(Nil)))));
    let b = Cons(3, Rc::clone(&a)); // shalow copy only reference, not data
    let c = Cons(4, Rc::clone(&a));
}
\n\n\n

We could have called a.clone() rather than Rc::clone(&a), but Rust’s convention is to use Rc::clone in this case. The implementation of Rc::clone doesn’t make a deep copy of all the data like most types’ implementations of clone do. The call to Rc::clone only increments the reference count, which doesn’t take much time.

\n

Cloning an Rc Increases the Reference Count

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
enum List {
    Cons(i32, Rc<List>),
    Nil,
}

use crate::List::{Cons, Nil};
use std::rc::Rc;

fn main() {
    let a = Rc::new(Cons(5, Rc::new(Cons(10, Rc::new(Nil)))));
    println!("count after creating a = {}", Rc::strong_count(&a));  // 1
    let b = Cons(3, Rc::clone(&a));
    println!("count after creating b = {}", Rc::strong_count(&a));  // 2
    {
        let c = Cons(4, Rc::clone(&a));
        println!("count after creating c = {}", Rc::strong_count(&a)); // 3
    }
    println!("count after c goes out of scope = {}", Rc::strong_count(&a));  // 2
}
\n

What we can’t see in this example is that when b and then a go out of scope at the end of main, the count is then 0, and the Rc is cleaned up completely at that point.

\n

RefCell and interior mutability pattern

Interior mutability is a design pattern in Rust that allows you to mutate data even when there are immutable references to that data; normally, this action is disallowed by the borrowing rules.
To mutate data, the pattern uses unsafe code inside a data structure to bend Rust’s usual rules that govern mutation and borrowing.
We can use types that use the interior mutability pattern when we can ensure that the borrowing rules will be followed at runtime, even though the compiler can’t guarantee that.
The unsafe code involved is then wrapped in a safe API, and the outer type is still immutable.

\n

Enforcing Borrowing Rules at Runtime with RefCell

Unlike Rc, the RefCell type represents single ownership over the data it holds.

\n

recall borrowing rules

\n
    \n
  • At any given time, you can have either (but not both of) one mutable reference or any number of immutable references.
    • \n
  • References must always be valid.
    With references and Box, the borrowing rules’ invariants are enforced at compile time. With RefCell, these invariants are enforced at runtime.
    Because RefCell allows mutable borrows checked at runtime, you can mutate the value inside the RefCell even when the RefCell is immutable.
    • \n
\n

Interior Mutability: A Mutable Borrow to an Immutable Value

1
2
3
4
fn main() {
    let x = 5;
    let y = &mut x; // not allowed. cannot borrow `x` as mutable, as it is not declared as mutable
}
\n\n

A Use Case for Interior Mutability: Mock Objects

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
pub trait Messenger {
    fn send(&self, msg: &str);
}

pub struct LimitTracker<'a, T: Messenger> {
    messenger: &'a T,
    value: usize,
    max: usize,
}

impl<'a, T> LimitTracker<'a, T>
where
    T: Messenger,
{
    pub fn new(messenger: &T, max: usize) -> LimitTracker<T> {
        LimitTracker {
            messenger,
            value: 0,
            max,
        }
    }

    pub fn set_value(&mut self, value: usize) {
        self.value = value;

        let percentage_of_max = self.value as f64 / self.max as f64;

        if percentage_of_max >= 1.0 {
            self.messenger.send("Error: You are over your quota!");
        } else if percentage_of_max >= 0.9 {
            self.messenger
                .send("Urgent warning: You've used up over 90% of your quota!");
        } else if percentage_of_max >= 0.75 {
            self.messenger
                .send("Warning: You've used up over 75% of your quota!");
        }
    }
}

\n

a problematic usage

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
pub trait Messenger {
    fn send(&self, msg: &str);
}

pub struct LimitTracker<'a, T: Messenger> {
    messenger: &'a T,
    value: usize,
    max: usize,
}

impl<'a, T> LimitTracker<'a, T>
where
    T: Messenger,
{
    pub fn new(messenger: &T, max: usize) -> LimitTracker<T> {
        LimitTracker {
            messenger,
            value: 0,
            max,
        }
    }

    pub fn set_value(&mut self, value: usize) {
        self.value = value;

        let percentage_of_max = self.value as f64 / self.max as f64;

        if percentage_of_max >= 1.0 {
            self.messenger.send("Error: You are over your quota!");
        } else if percentage_of_max >= 0.9 {
            self.messenger
                .send("Urgent warning: You've used up over 90% of your quota!");
        } else if percentage_of_max >= 0.75 {
            self.messenger
                .send("Warning: You've used up over 75% of your quota!");
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    struct MockMessenger {
        sent_messages: Vec<String>,
    }

    impl MockMessenger {
        fn new() -> MockMessenger {
            MockMessenger {
                sent_messages: vec![],
            }
        }
    }

    impl Messenger for MockMessenger {
        fn send(&self, message: &str) {
            self.sent_messages.push(String::from(message)); // not allowed. cannot borrow `self.sent_messages` as mutable, as it is behind a `&` reference
        }
    }

    #[test]
    fn it_sends_an_over_75_percent_warning_message() {
        let mock_messenger = MockMessenger::new();
        let mut limit_tracker = LimitTracker::new(&mock_messenger, 100);

        limit_tracker.set_value(80);

        assert_eq!(mock_messenger.sent_messages.len(), 1);
    }
}

\n

We can’t modify the MockMessenger to keep track of the messages, because the send method takes an immutable reference to self. We also can’t take the suggestion from the error text to use &mut self instead, because then the signature of send wouldn’t match the signature in the Messenger trait definition

\n

This is a situation in which interior mutability can help!

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
pub trait Messenger {
    fn send(&self, msg: &str);
}

pub struct LimitTracker<'a, T: Messenger> {
    messenger: &'a T,
    value: usize,
    max: usize,
}

impl<'a, T> LimitTracker<'a, T>
where
    T: Messenger,
{
    pub fn new(messenger: &T, max: usize) -> LimitTracker<T> {
        LimitTracker {
            messenger,
            value: 0,
            max,
        }
    }

    pub fn set_value(&mut self, value: usize) {
        self.value = value;

        let percentage_of_max = self.value as f64 / self.max as f64;

        if percentage_of_max >= 1.0 {
            self.messenger.send("Error: You are over your quota!");
        } else if percentage_of_max >= 0.9 {
            self.messenger
                .send("Urgent warning: You've used up over 90% of your quota!");
        } else if percentage_of_max >= 0.75 {
            self.messenger
                .send("Warning: You've used up over 75% of your quota!");
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::cell::RefCell;

    struct MockMessenger {
        sent_messages: RefCell<Vec<String>>,
    }

    impl MockMessenger {
        fn new() -> MockMessenger {
            MockMessenger {
                sent_messages: RefCell::new(vec![]),
            }
        }
    }

    impl Messenger for MockMessenger {
        fn send(&self, message: &str) {
            self.sent_messages.borrow_mut().push(String::from(message)); // call borrow_mut on the RefCell<Vec<String>> in self.sent_messages to get a mutable reference to the value inside the RefCell<Vec<String>>
        }
    }

    #[test]
    fn it_sends_an_over_75_percent_warning_message() {
        // --snip--
        let mock_messenger = MockMessenger::new();
        let mut limit_tracker = LimitTracker::new(&mock_messenger, 100);

        limit_tracker.set_value(80);

        assert_eq!(mock_messenger.sent_messages.borrow().len(), 1); //  call borrow on the RefCell<Vec<String>> to get an immutable reference to the vector.
    }
}
\n\n

Keeping Track of Borrows at Runtime with RefCell

When creating immutable and mutable references, we use the & and &mut syntax, respectively. With RefCell, we use the borrow and borrow_mut methods, which are part of the safe API that belongs to RefCell. The borrow method returns the smart pointer type Ref, and borrow_mut returns the smart pointer type RefMut. Both types implement Deref, so we can treat them like regular references.

\n

The RefCell keeps track of how many Ref and RefMut smart pointers are currently active.RefCell lets us have many immutable borrows or one mutable borrow at any point in time. If we try to violate these rules, rather than getting a compiler error as we would with references, the implementation of RefCell will panic at runtime.

\n
1
2
3
4
5
6
7
8
9
impl Messenger for MockMessenger {
        fn send(&self, message: &str) {
            let mut one_borrow = self.sent_messages.borrow_mut();
            let mut two_borrow = self.sent_messages.borrow_mut();

            one_borrow.push(String::from(message));
            two_borrow.push(String::from(message));
        }
    }
\n

When we run the tests for our library, the code in will compile without any errors, but the test will fail
thread ‘main’ panicked at ‘already borrowed

\n

Having Multiple Owners of Mutable Data by Combining Rc and RefCell

A common way to use RefCell is in combination with Rc. If you have an Rc that holds a RefCell, you can get a value that can have multiple owners and that you can mutate!

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
#[derive(Debug)]
enum List {
    Cons(Rc<RefCell<i32>>, Rc<List>),
    Nil,
}

use crate::List::{Cons, Nil};
use std::cell::RefCell;
use std::rc::Rc;

fn main() {
    let value = Rc::new(RefCell::new(5)); // We create a value that is an instance of Rc<RefCell<i32>> and store it in a variable named value so we can access it directly later.

    let a = Rc::new(Cons(Rc::clone(&value), Rc::new(Nil))); //  Then we create a List in a with a Cons variant that holds value. We need to clone value so both a and value have ownership of the inner 5 value rather than transferring ownership from value to a or having a borrow from value. We wrap the list a in an Rc<T> so when we create lists b and c, they can both refer to a

    let b = Cons(Rc::new(RefCell::new(3)), Rc::clone(&a));
    let c = Cons(Rc::new(RefCell::new(4)), Rc::clone(&a));

    *value.borrow_mut() += 10; // After we’ve created the lists in a, b, and c, we add 10 to the value in value. We do this by calling borrow_mut on value, which uses the automatic dereferencing feature to dereference the Rc<T> to the inner RefCell<T> value. The borrow_mut method returns a RefMut<T> smart pointer, and we use the dereference operator on it and change the inner value.

    println!("a after = {:?}", a);
    println!("b after = {:?}", b);
    println!("c after = {:?}", c);
}
\n\n

The standard library has other types that provide interior mutability, such as Cell, which is similar except that instead of giving references to the inner value, the value is copied in and out of the Cell. There’s also Mutex, which offers interior mutability that’s safe to use across threads;

\n

Reference Cycles Can Leak Memory

Rust’s memory safety guarantees make it difficult, but not impossible, to accidentally create memory that is never cleaned up (known as a memory leak).

\n

Creating a Reference Cycle

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
use crate::List::{Cons, Nil};
use std::cell::RefCell;
use std::rc::Rc;

#[derive(Debug)]
enum List {
    Cons(i32, RefCell<Rc<List>>), // The second element in the Cons variant is now RefCell<Rc<List>>, meaning that we want to modify which List value a Cons variant is pointing to. 
    Nil,
}

impl List {
    fn tail(&self) -> Option<&RefCell<Rc<List>>> { // We’re also adding a tail method to make it convenient for us to access the second item if we have a Cons variant.
        match self {
            Cons(_, item) => Some(item),
            Nil => None,
        }
    }
}

fn main() {
    use crate::List::{Cons, Nil};
use std::cell::RefCell;
use std::rc::Rc;

#[derive(Debug)]
enum List {
    Cons(i32, RefCell<Rc<List>>),
    Nil,
}

impl List {
    fn tail(&self) -> Option<&RefCell<Rc<List>>> {
        match self {
            Cons(_, item) => Some(item),
            Nil => None,
        }
    }
}

fn main() {
    let a = Rc::new(Cons(5, RefCell::new(Rc::new(Nil))));

    println!("a initial rc count = {}", Rc::strong_count(&a));
    println!("a next item = {:?}", a.tail());

    let b = Rc::new(Cons(10, RefCell::new(Rc::clone(&a)))); // his code creates a list in a and a list in b that points to the list in a

    println!("a rc count after b creation = {}", Rc::strong_count(&a));
    println!("b initial rc count = {}", Rc::strong_count(&b));
    println!("b next item = {:?}", b.tail());

    if let Some(link) = a.tail() {
        *link.borrow_mut() = Rc::clone(&b); // modifies the list in a to point to b, creating a reference cycle
    }

    println!("b rc count after changing a = {}", Rc::strong_count(&b));
    println!("a rc count after changing a = {}", Rc::strong_count(&a));

    // Uncomment the next line to see that we have a cycle;
    // it will overflow the stack
    // println!("a next item = {:?}", a.tail());
} // At the end of main, Rust drops the variable b, which decreases the reference count of the Rc<List> instance from 2 to 1. The memory that Rc<List> has on the heap won’t be dropped at this point, because its reference count is 1, not 0. Then Rust drops a, which decreases the reference count of the a Rc<List> instance from 2 to 1 as well. This instance’s memory can’t be dropped either, because the other Rc<List> instance still refers to it. The memory allocated to the list will remain uncollected forever.
\n

\"cycle-ref\"

\n

Preventing Reference Cycles: Turning an Rc into a Weak

So far, we’ve demonstrated that calling Rc::clone increases the strong_count of an Rc instance, and an Rc instance is only cleaned up if its strong_count is 0. You can also create a weak reference to the value within an Rc instance by calling Rc::downgrade and passing a reference to the Rc. When you call Rc::downgrade, you get a smart pointer of type Weak. Instead of increasing the strong_count in the Rc instance by 1, calling Rc::downgrade increases the weak_count by 1. The Rc type uses weak_count to keep track of how many Weak references exist, similar to strong_count. The difference is the weak_count doesn’t need to be 0 for the Rc instance to be cleaned up.

\n

Strong references are how you can share ownership of an Rc instance. Weak references don’t express an ownership relationship. They won’t cause a reference cycle because any cycle involving some weak references will be broken once the strong reference count of values involved is 0.

\n

Because the value that Weak references might have been dropped, to do anything with the value that a Weak is pointing to, you must make sure the value still exists. Do this by calling the upgrade method on a Weak instance, which will return an Option<Rc>. You’ll get a result of Some if the Rc value has not been dropped yet and a result of None if the Rc value has been dropped.

\n

Creating a Tree Data Structure: a Node with Child Nodes

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
use std::cell::RefCell;
use std::rc::Rc;

#[derive(Debug)]
struct Node {
    value: i32,
    parent: RefCell<Weak<Node>>, // To make the child node aware of its parent, we need to add a parent field to our Node struct definition. The trouble is in deciding what the type of parent should be. We know it can’t contain an Rc<T>, because that would create a reference cycle with leaf.parent pointing to branch and branch.children pointing to leaf, which would cause their strong_count values to never be 0.
    children: RefCell<Vec<Rc<Node>>>,
}

fn main() {

    let leaf = Rc::new(Node {
        value: 3,
        parent: RefCell::new(Weak::new()),
        children: RefCell::new(vec![]),
    });

    println!("leaf parent = {:?}", leaf.parent.borrow().upgrade()); // try to get a reference to the parent of leaf by using the upgrade method, we get a None value. 

    let branch = Rc::new(Node {
        value: 5,
        parent: RefCell::new(Weak::new()),
        children: RefCell::new(vec![Rc::clone(&leaf)]), // We clone the Rc<Node> in leaf and store that in branch, meaning the Node in leaf now has two owners: leaf and branch. 
    });

    *leaf.parent.borrow_mut() = Rc::downgrade(&branch); // use the Rc::downgrade function to create a Weak<Node> reference to branch from the Rc<Node> in branch.

    println!("leaf parent = {:?}", leaf.parent.borrow().upgrade());

   
}
\n\n

Visualizing Changes to strong_count and weak_count

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
use std::cell::RefCell;
use std::rc::{Rc, Weak};

#[derive(Debug)]
struct Node {
    value: i32,
    parent: RefCell<Weak<Node>>,
    children: RefCell<Vec<Rc<Node>>>,
}

fn main() {
    let leaf = Rc::new(Node {
        value: 3,
        parent: RefCell::new(Weak::new()),
        children: RefCell::new(vec![]),
    });

    println!(
        "leaf strong = {}, weak = {}",
        Rc::strong_count(&leaf),
        Rc::weak_count(&leaf),
    );

    {
        let branch = Rc::new(Node {
            value: 5,
            parent: RefCell::new(Weak::new()),
            children: RefCell::new(vec![Rc::clone(&leaf)]),
        });

        *leaf.parent.borrow_mut() = Rc::downgrade(&branch);

        println!(
            "branch strong = {}, weak = {}",
            Rc::strong_count(&branch),
            Rc::weak_count(&branch),
        );

        println!(
            "leaf strong = {}, weak = {}",
            Rc::strong_count(&leaf),
            Rc::weak_count(&leaf),
        );
    }

    println!("leaf parent = {:?}", leaf.parent.borrow().upgrade());
    println!(
        "leaf strong = {}, weak = {}",
        Rc::strong_count(&leaf),
        Rc::weak_count(&leaf),
    );
}
","site":{"data":{}},"excerpt":"","more":"

Overview

The most common kind of pointer in Rust is a reference (borrow but not own)
Smart pointers, on the other hand, are data structures that not only act like a pointer but also have additional metadata and capabilities.
references are pointers that only borrow data; in contrast, in many cases, smart pointers own the data they point to.

\n

smart pointers example

    \n
  • String
    • \n
  • Vec
    Both these types count as smart pointers because they own some memory and allow you to manipulate it. They also have metadata (such as their capacity) and extra capabilities or guarantees (such as with String ensuring its data will always be valid UTF-8).
    • \n
\n

Deref & Drop

Smart pointers are usually implemented using structs. The characteristic that distinguishes a smart pointer from an ordinary struct is that smart pointers implement the Deref and Drop traits.

\n
    \n
  • The Deref trait allows an instance of the smart pointer struct to behave like a reference so you can write code that works with either references or smart pointers.
    • \n
  • The Drop trait allows you to customize the code that is run when an instance of the smart pointer goes out of scope.
    • \n
\n

the most common smart pointers in the standard library:

    \n
  • Box<T> for allocating values on the heap
    • \n
  • Rc<T>, a reference counting type that enables multiple ownership
    • \n
  • Ref<T> and RefMut<T>, accessed through RefCell<T>, a type that enforces the borrowing rules at runtime instead of compile time
    • \n
\n

Box

when to use

    \n
  • When you have a type whose size can’t be known at compile time and you want to use a value of that type in a context that requires an exact size. (such as cons)
    • \n
  • When you have a large amount of data and you want to transfer ownership but ensure the data won’t be copied when you do so
    • \n
  • When you want to own a value and you care only that it’s a type that implements a particular trait rather than being of a specific type
    • \n
\n

enabling recursive types with Box

At compile time, Rust needs to know how much space a type takes up
One type whose size can’t be known at compile time is a recursive type (cons list), use Box, which only contains a memory address

\n
1
2
3
4
5
6
7
8
9
enum List {
    Cons(i32, List),
    Nil,
}

use crate::List::{Cons, Nil};
fn main() {
    let list = Cons(1, Cons(2, Cons(3, Nil))); // not allowed, infinite size
}
\n\n

use Box

\n
1
2
3
4
5
6
7
8
9
10
enum List {
    Cons(i32, Box<List>),
    Nil,
}

use crate::List::{Cons, Nil};

fn main() {
    let list = Cons(1, Box::new(Cons(2, Box::new(Cons(3, Box::new(Nil))))));
}
\n\n

Treating Smart Pointers Like Regular References with the Deref Trait

Implementing the Deref trait allows you to customize the behavior of the dereference operator, *
By implementing Deref in such a way that a smart pointer can be treated like a regular reference, you can write code that operates on references and use that code with smart pointers too.

\n

Defining Our Own Smart Pointer

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
use std::ops::Deref;

struct MyBox<T>(T);  // The MyBox type is a tuple struct with one element of type T

impl<T> MyBox<T> {
    fn new(x: T) -> MyBox<T> {
        MyBox(x)
    }
}

impl<T> Deref for MyBox<T> {
    type Target = T;

    fn deref(&self) -> &Self::Target {
        &self.0
    }
}

fn main() {
    let x = 5;
    let y = MyBox::new(x);

    assert_eq!(5, x);
    assert_eq!(5, *y);
}
\n
    \n
  • We fill in the body of the deref method with &self.0 so deref returns a reference to the value we want to access with the * operator.
    • \n
  • behind the scenes Rust actually ran this code: *(y.deref()). Rust substitutes the * operator with a call to the deref method
    • \n
  • The reason the deref method returns a reference to a value, and that the plain dereference outside the parentheses in *(y.deref()) is still necessary, is the ownership system. If the deref method returned the value directly instead of a reference to the value, the value would be moved out of self. We don’t want to take ownership of the inner value inside MyBox in this case or in most cases where we use the dereference operator.
    • \n
\n

Implicit Deref Coercions with Functions and Methods

Deref coercion is a convenience that Rust performs on arguments to functions and methods.
Deref coercion works only on types that implement the Deref trait. Deref coercion converts a reference to such a type into a reference to another type. For example, deref coercion can convert &String to &str because String implements the Deref trait such that it returns &str
A sequence of calls to the deref method converts the type we provided into the type the parameter needs.

\n
1
2
3
4
5
6
7
8
9
10
fn hello(name: &str) {
    println!("Hello, {}!", name);
}

fn main() {
    let m = MyBox::new(String::from("Rust"));
    hello("rust");  // ok
    hello(&m);      // also ok
    hello(&(*m)[..]); // without deref coercion. The (*m) dereferences the MyBox<String> into a String. Then the & and [..] take a string slice of the String that is equal to the whole string to match the signature of hello
}
\n

Here we’re calling the hello function with the argument &m, which is a reference to a MyBox value. Because we implemented the Deref trait on MyBox, Rust can turn &MyBox into &String by calling deref. The standard library provides an implementation of Deref on String that returns a string slice. Rust calls deref again to turn the &String into &str, which matches the hello function’s definition.

\n

How Deref Coercion Interacts with Mutability

Similar to how you use the Deref trait to override the * operator on immutable references, you can use the DerefMut trait to override the * operator on mutable references.

\n

Rust does deref coercion when it finds types and trait implementations in three cases:

\n
    \n
  • From &T to &U when T: Deref<Target=U>
    • \n
  • From &mut T to &mut U when T: DerefMut<Target=U>
    • \n
  • From &mut T to &U when T: Deref<Target=U>
    • \n
\n

Running Code on Cleanup with the Drop Trait

Drop, which lets you customize what happens when a value is about to go out of scope.

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
struct CustomSmartPointer {
    data: String,
}

impl Drop for CustomSmartPointer {
    fn drop(&mut self) {
        println!("Dropping CustomSmartPointer with data `{}`!", self.data);
    }
}

fn main() {
    let c = CustomSmartPointer {
        data: String::from("my stuff"),
    };
    c.drop(); // not allowed
    let d = CustomSmartPointer {
        data: String::from("other stuff"),
    };
    println!("CustomSmartPointers created.");
}

\n

Dropping a Value Early with std::mem::drop

std::mem::drop is in prelude, can use directly

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
struct CustomSmartPointer {
    data: String,
}

impl Drop for CustomSmartPointer {
    fn drop(&mut self) {
        println!("Dropping CustomSmartPointer with data `{}`!", self.data);
    }
}

fn main() {
    let c = CustomSmartPointer {
        data: String::from("some data"),
    };
    println!("CustomSmartPointer created.");
    drop(c); // ok
    println!("CustomSmartPointer dropped before the end of main.");
} // c goes out of scope, will occur double drop

\n\n\n

Rc: the Reference Counted Smart Pointer

In the majority of cases, ownership is clear: you know exactly which variable owns a given value. However, there are cases when a single value might have multiple owners.
type keeps track of the number of references to a value to determine whether or not the value is still in use. If there are zero references to a value, the value can be cleaned up without any references becoming invalid.

\n

We use the Rc<T> type when we want to allocate some data on the heap for multiple parts of our program to read and we can’t determine at compile time which part will finish using the data last.

\n

Rc is only for use in single-threaded scenarios

\n

using Rc to share data

\"rc\"
implement use box will not work, as below

\n
1
2
3
4
5
6
7
8
9
10
11
12
enum List {
    Cons(i32, Box<List>),
    Nil,
}

use crate::List::{Cons, Nil};

fn main() {
    let a = Cons(5, Box::new(Cons(10, Box::new(Nil))));
    let b = Cons(3, Box::new(a)); // value moved here
    let c = Cons(4, Box::new(a)); // value used here after move
}
\n

use Rc

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
enum List {
    Cons(i32, Rc<List>),
    Nil,
}

use crate::List::{Cons, Nil};
use std::rc::Rc;

fn main() {
    let a = Rc::new(Cons(5, Rc::new(Cons(10, Rc::new(Nil)))));
    let b = Cons(3, Rc::clone(&a)); // shalow copy only reference, not data
    let c = Cons(4, Rc::clone(&a));
}
\n\n\n

We could have called a.clone() rather than Rc::clone(&a), but Rust’s convention is to use Rc::clone in this case. The implementation of Rc::clone doesn’t make a deep copy of all the data like most types’ implementations of clone do. The call to Rc::clone only increments the reference count, which doesn’t take much time.

\n

Cloning an Rc Increases the Reference Count

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
enum List {
    Cons(i32, Rc<List>),
    Nil,
}

use crate::List::{Cons, Nil};
use std::rc::Rc;

fn main() {
    let a = Rc::new(Cons(5, Rc::new(Cons(10, Rc::new(Nil)))));
    println!("count after creating a = {}", Rc::strong_count(&a));  // 1
    let b = Cons(3, Rc::clone(&a));
    println!("count after creating b = {}", Rc::strong_count(&a));  // 2
    {
        let c = Cons(4, Rc::clone(&a));
        println!("count after creating c = {}", Rc::strong_count(&a)); // 3
    }
    println!("count after c goes out of scope = {}", Rc::strong_count(&a));  // 2
}
\n

What we can’t see in this example is that when b and then a go out of scope at the end of main, the count is then 0, and the Rc is cleaned up completely at that point.

\n

RefCell and interior mutability pattern

Interior mutability is a design pattern in Rust that allows you to mutate data even when there are immutable references to that data; normally, this action is disallowed by the borrowing rules.
To mutate data, the pattern uses unsafe code inside a data structure to bend Rust’s usual rules that govern mutation and borrowing.
We can use types that use the interior mutability pattern when we can ensure that the borrowing rules will be followed at runtime, even though the compiler can’t guarantee that.
The unsafe code involved is then wrapped in a safe API, and the outer type is still immutable.

\n

Enforcing Borrowing Rules at Runtime with RefCell

Unlike Rc, the RefCell type represents single ownership over the data it holds.

\n

recall borrowing rules

\n
    \n
  • At any given time, you can have either (but not both of) one mutable reference or any number of immutable references.
    • \n
  • References must always be valid.
    With references and Box, the borrowing rules’ invariants are enforced at compile time. With RefCell, these invariants are enforced at runtime.
    Because RefCell allows mutable borrows checked at runtime, you can mutate the value inside the RefCell even when the RefCell is immutable.
    • \n
\n

Interior Mutability: A Mutable Borrow to an Immutable Value

1
2
3
4
fn main() {
    let x = 5;
    let y = &mut x; // not allowed. cannot borrow `x` as mutable, as it is not declared as mutable
}
\n\n

A Use Case for Interior Mutability: Mock Objects

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
pub trait Messenger {
    fn send(&self, msg: &str);
}

pub struct LimitTracker<'a, T: Messenger> {
    messenger: &'a T,
    value: usize,
    max: usize,
}

impl<'a, T> LimitTracker<'a, T>
where
    T: Messenger,
{
    pub fn new(messenger: &T, max: usize) -> LimitTracker<T> {
        LimitTracker {
            messenger,
            value: 0,
            max,
        }
    }

    pub fn set_value(&mut self, value: usize) {
        self.value = value;

        let percentage_of_max = self.value as f64 / self.max as f64;

        if percentage_of_max >= 1.0 {
            self.messenger.send("Error: You are over your quota!");
        } else if percentage_of_max >= 0.9 {
            self.messenger
                .send("Urgent warning: You've used up over 90% of your quota!");
        } else if percentage_of_max >= 0.75 {
            self.messenger
                .send("Warning: You've used up over 75% of your quota!");
        }
    }
}

\n

a problematic usage

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
pub trait Messenger {
    fn send(&self, msg: &str);
}

pub struct LimitTracker<'a, T: Messenger> {
    messenger: &'a T,
    value: usize,
    max: usize,
}

impl<'a, T> LimitTracker<'a, T>
where
    T: Messenger,
{
    pub fn new(messenger: &T, max: usize) -> LimitTracker<T> {
        LimitTracker {
            messenger,
            value: 0,
            max,
        }
    }

    pub fn set_value(&mut self, value: usize) {
        self.value = value;

        let percentage_of_max = self.value as f64 / self.max as f64;

        if percentage_of_max >= 1.0 {
            self.messenger.send("Error: You are over your quota!");
        } else if percentage_of_max >= 0.9 {
            self.messenger
                .send("Urgent warning: You've used up over 90% of your quota!");
        } else if percentage_of_max >= 0.75 {
            self.messenger
                .send("Warning: You've used up over 75% of your quota!");
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    struct MockMessenger {
        sent_messages: Vec<String>,
    }

    impl MockMessenger {
        fn new() -> MockMessenger {
            MockMessenger {
                sent_messages: vec![],
            }
        }
    }

    impl Messenger for MockMessenger {
        fn send(&self, message: &str) {
            self.sent_messages.push(String::from(message)); // not allowed. cannot borrow `self.sent_messages` as mutable, as it is behind a `&` reference
        }
    }

    #[test]
    fn it_sends_an_over_75_percent_warning_message() {
        let mock_messenger = MockMessenger::new();
        let mut limit_tracker = LimitTracker::new(&mock_messenger, 100);

        limit_tracker.set_value(80);

        assert_eq!(mock_messenger.sent_messages.len(), 1);
    }
}

\n

We can’t modify the MockMessenger to keep track of the messages, because the send method takes an immutable reference to self. We also can’t take the suggestion from the error text to use &mut self instead, because then the signature of send wouldn’t match the signature in the Messenger trait definition

\n

This is a situation in which interior mutability can help!

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
pub trait Messenger {
    fn send(&self, msg: &str);
}

pub struct LimitTracker<'a, T: Messenger> {
    messenger: &'a T,
    value: usize,
    max: usize,
}

impl<'a, T> LimitTracker<'a, T>
where
    T: Messenger,
{
    pub fn new(messenger: &T, max: usize) -> LimitTracker<T> {
        LimitTracker {
            messenger,
            value: 0,
            max,
        }
    }

    pub fn set_value(&mut self, value: usize) {
        self.value = value;

        let percentage_of_max = self.value as f64 / self.max as f64;

        if percentage_of_max >= 1.0 {
            self.messenger.send("Error: You are over your quota!");
        } else if percentage_of_max >= 0.9 {
            self.messenger
                .send("Urgent warning: You've used up over 90% of your quota!");
        } else if percentage_of_max >= 0.75 {
            self.messenger
                .send("Warning: You've used up over 75% of your quota!");
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::cell::RefCell;

    struct MockMessenger {
        sent_messages: RefCell<Vec<String>>,
    }

    impl MockMessenger {
        fn new() -> MockMessenger {
            MockMessenger {
                sent_messages: RefCell::new(vec![]),
            }
        }
    }

    impl Messenger for MockMessenger {
        fn send(&self, message: &str) {
            self.sent_messages.borrow_mut().push(String::from(message)); // call borrow_mut on the RefCell<Vec<String>> in self.sent_messages to get a mutable reference to the value inside the RefCell<Vec<String>>
        }
    }

    #[test]
    fn it_sends_an_over_75_percent_warning_message() {
        // --snip--
        let mock_messenger = MockMessenger::new();
        let mut limit_tracker = LimitTracker::new(&mock_messenger, 100);

        limit_tracker.set_value(80);

        assert_eq!(mock_messenger.sent_messages.borrow().len(), 1); //  call borrow on the RefCell<Vec<String>> to get an immutable reference to the vector.
    }
}
\n\n

Keeping Track of Borrows at Runtime with RefCell

When creating immutable and mutable references, we use the & and &mut syntax, respectively. With RefCell, we use the borrow and borrow_mut methods, which are part of the safe API that belongs to RefCell. The borrow method returns the smart pointer type Ref, and borrow_mut returns the smart pointer type RefMut. Both types implement Deref, so we can treat them like regular references.

\n

The RefCell keeps track of how many Ref and RefMut smart pointers are currently active.RefCell lets us have many immutable borrows or one mutable borrow at any point in time. If we try to violate these rules, rather than getting a compiler error as we would with references, the implementation of RefCell will panic at runtime.

\n
1
2
3
4
5
6
7
8
9
impl Messenger for MockMessenger {
        fn send(&self, message: &str) {
            let mut one_borrow = self.sent_messages.borrow_mut();
            let mut two_borrow = self.sent_messages.borrow_mut();

            one_borrow.push(String::from(message));
            two_borrow.push(String::from(message));
        }
    }
\n

When we run the tests for our library, the code in will compile without any errors, but the test will fail
thread ‘main’ panicked at ‘already borrowed

\n

Having Multiple Owners of Mutable Data by Combining Rc and RefCell

A common way to use RefCell is in combination with Rc. If you have an Rc that holds a RefCell, you can get a value that can have multiple owners and that you can mutate!

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
#[derive(Debug)]
enum List {
    Cons(Rc<RefCell<i32>>, Rc<List>),
    Nil,
}

use crate::List::{Cons, Nil};
use std::cell::RefCell;
use std::rc::Rc;

fn main() {
    let value = Rc::new(RefCell::new(5)); // We create a value that is an instance of Rc<RefCell<i32>> and store it in a variable named value so we can access it directly later.

    let a = Rc::new(Cons(Rc::clone(&value), Rc::new(Nil))); //  Then we create a List in a with a Cons variant that holds value. We need to clone value so both a and value have ownership of the inner 5 value rather than transferring ownership from value to a or having a borrow from value. We wrap the list a in an Rc<T> so when we create lists b and c, they can both refer to a

    let b = Cons(Rc::new(RefCell::new(3)), Rc::clone(&a));
    let c = Cons(Rc::new(RefCell::new(4)), Rc::clone(&a));

    *value.borrow_mut() += 10; // After we’ve created the lists in a, b, and c, we add 10 to the value in value. We do this by calling borrow_mut on value, which uses the automatic dereferencing feature to dereference the Rc<T> to the inner RefCell<T> value. The borrow_mut method returns a RefMut<T> smart pointer, and we use the dereference operator on it and change the inner value.

    println!("a after = {:?}", a);
    println!("b after = {:?}", b);
    println!("c after = {:?}", c);
}
\n\n

The standard library has other types that provide interior mutability, such as Cell, which is similar except that instead of giving references to the inner value, the value is copied in and out of the Cell. There’s also Mutex, which offers interior mutability that’s safe to use across threads;

\n

Reference Cycles Can Leak Memory

Rust’s memory safety guarantees make it difficult, but not impossible, to accidentally create memory that is never cleaned up (known as a memory leak).

\n

Creating a Reference Cycle

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
use crate::List::{Cons, Nil};
use std::cell::RefCell;
use std::rc::Rc;

#[derive(Debug)]
enum List {
    Cons(i32, RefCell<Rc<List>>), // The second element in the Cons variant is now RefCell<Rc<List>>, meaning that we want to modify which List value a Cons variant is pointing to. 
    Nil,
}

impl List {
    fn tail(&self) -> Option<&RefCell<Rc<List>>> { // We’re also adding a tail method to make it convenient for us to access the second item if we have a Cons variant.
        match self {
            Cons(_, item) => Some(item),
            Nil => None,
        }
    }
}

fn main() {
    use crate::List::{Cons, Nil};
use std::cell::RefCell;
use std::rc::Rc;

#[derive(Debug)]
enum List {
    Cons(i32, RefCell<Rc<List>>),
    Nil,
}

impl List {
    fn tail(&self) -> Option<&RefCell<Rc<List>>> {
        match self {
            Cons(_, item) => Some(item),
            Nil => None,
        }
    }
}

fn main() {
    let a = Rc::new(Cons(5, RefCell::new(Rc::new(Nil))));

    println!("a initial rc count = {}", Rc::strong_count(&a));
    println!("a next item = {:?}", a.tail());

    let b = Rc::new(Cons(10, RefCell::new(Rc::clone(&a)))); // his code creates a list in a and a list in b that points to the list in a

    println!("a rc count after b creation = {}", Rc::strong_count(&a));
    println!("b initial rc count = {}", Rc::strong_count(&b));
    println!("b next item = {:?}", b.tail());

    if let Some(link) = a.tail() {
        *link.borrow_mut() = Rc::clone(&b); // modifies the list in a to point to b, creating a reference cycle
    }

    println!("b rc count after changing a = {}", Rc::strong_count(&b));
    println!("a rc count after changing a = {}", Rc::strong_count(&a));

    // Uncomment the next line to see that we have a cycle;
    // it will overflow the stack
    // println!("a next item = {:?}", a.tail());
} // At the end of main, Rust drops the variable b, which decreases the reference count of the Rc<List> instance from 2 to 1. The memory that Rc<List> has on the heap won’t be dropped at this point, because its reference count is 1, not 0. Then Rust drops a, which decreases the reference count of the a Rc<List> instance from 2 to 1 as well. This instance’s memory can’t be dropped either, because the other Rc<List> instance still refers to it. The memory allocated to the list will remain uncollected forever.
\n

\"cycle-ref\"

\n

Preventing Reference Cycles: Turning an Rc into a Weak

So far, we’ve demonstrated that calling Rc::clone increases the strong_count of an Rc instance, and an Rc instance is only cleaned up if its strong_count is 0. You can also create a weak reference to the value within an Rc instance by calling Rc::downgrade and passing a reference to the Rc. When you call Rc::downgrade, you get a smart pointer of type Weak. Instead of increasing the strong_count in the Rc instance by 1, calling Rc::downgrade increases the weak_count by 1. The Rc type uses weak_count to keep track of how many Weak references exist, similar to strong_count. The difference is the weak_count doesn’t need to be 0 for the Rc instance to be cleaned up.

\n

Strong references are how you can share ownership of an Rc instance. Weak references don’t express an ownership relationship. They won’t cause a reference cycle because any cycle involving some weak references will be broken once the strong reference count of values involved is 0.

\n

Because the value that Weak references might have been dropped, to do anything with the value that a Weak is pointing to, you must make sure the value still exists. Do this by calling the upgrade method on a Weak instance, which will return an Option<Rc>. You’ll get a result of Some if the Rc value has not been dropped yet and a result of None if the Rc value has been dropped.

\n

Creating a Tree Data Structure: a Node with Child Nodes

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
use std::cell::RefCell;
use std::rc::Rc;

#[derive(Debug)]
struct Node {
    value: i32,
    parent: RefCell<Weak<Node>>, // To make the child node aware of its parent, we need to add a parent field to our Node struct definition. The trouble is in deciding what the type of parent should be. We know it can’t contain an Rc<T>, because that would create a reference cycle with leaf.parent pointing to branch and branch.children pointing to leaf, which would cause their strong_count values to never be 0.
    children: RefCell<Vec<Rc<Node>>>,
}

fn main() {

    let leaf = Rc::new(Node {
        value: 3,
        parent: RefCell::new(Weak::new()),
        children: RefCell::new(vec![]),
    });

    println!("leaf parent = {:?}", leaf.parent.borrow().upgrade()); // try to get a reference to the parent of leaf by using the upgrade method, we get a None value. 

    let branch = Rc::new(Node {
        value: 5,
        parent: RefCell::new(Weak::new()),
        children: RefCell::new(vec![Rc::clone(&leaf)]), // We clone the Rc<Node> in leaf and store that in branch, meaning the Node in leaf now has two owners: leaf and branch. 
    });

    *leaf.parent.borrow_mut() = Rc::downgrade(&branch); // use the Rc::downgrade function to create a Weak<Node> reference to branch from the Rc<Node> in branch.

    println!("leaf parent = {:?}", leaf.parent.borrow().upgrade());

   
}
\n\n

Visualizing Changes to strong_count and weak_count

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
use std::cell::RefCell;
use std::rc::{Rc, Weak};

#[derive(Debug)]
struct Node {
    value: i32,
    parent: RefCell<Weak<Node>>,
    children: RefCell<Vec<Rc<Node>>>,
}

fn main() {
    let leaf = Rc::new(Node {
        value: 3,
        parent: RefCell::new(Weak::new()),
        children: RefCell::new(vec![]),
    });

    println!(
        "leaf strong = {}, weak = {}",
        Rc::strong_count(&leaf),
        Rc::weak_count(&leaf),
    );

    {
        let branch = Rc::new(Node {
            value: 5,
            parent: RefCell::new(Weak::new()),
            children: RefCell::new(vec![Rc::clone(&leaf)]),
        });

        *leaf.parent.borrow_mut() = Rc::downgrade(&branch);

        println!(
            "branch strong = {}, weak = {}",
            Rc::strong_count(&branch),
            Rc::weak_count(&branch),
        );

        println!(
            "leaf strong = {}, weak = {}",
            Rc::strong_count(&leaf),
            Rc::weak_count(&leaf),
        );
    }

    println!("leaf parent = {:?}", leaf.parent.borrow().upgrade());
    println!(
        "leaf strong = {}, weak = {}",
        Rc::strong_count(&leaf),
        Rc::weak_count(&leaf),
    );
}
"},{"title":"rust memory layout","date":"2022-10-25T02:54:13.000Z","_content":"\n\n## trait object\na reference to a trait type (or Box) is called trait object\n\n### two ways convert concrete type to trait object\n1. assigning to variable\n```rust\nuse std::io::Write;\n\nlet mut buffer: Vec = vec![];\nlet w: &mut dyn Write = &mut buffer;\n```\n2. pass a concrete type as a argument to a function\n```rust\nfn main() {\n let mut buffer: Vec = vec![];\n writer(&mut buffer);\n}\n\nfn writer(w: &mut dyn Write) {\n // ...\n}\n```\nin both ways, buffer is converted to a trait object implements Write\n\nin memory, a trait object (in the example, w) is a fat point consists two pointers\n![trait_object_memory](/images/rust/memory/trait_object_memory.png)\nthe vtable is generated once at compile time and shared by all objects of the same type. the vtable contains pointers to the machine code of the functions that must be present for a type to be a \"Writer\"\n\n\n## references\n- https://www.youtube.com/watch?v=rDoqT-a6UFg","source":"_posts/rust/rust-05-memory-layout.md","raw":"---\ntitle: rust memory layout\ndate: 2022-10-25 10:54:13\ntags: [rust]\n---\n\n\n## trait object\na reference to a trait type (or Box) is called trait object\n\n### two ways convert concrete type to trait object\n1. assigning to variable\n```rust\nuse std::io::Write;\n\nlet mut buffer: Vec = vec![];\nlet w: &mut dyn Write = &mut buffer;\n```\n2. pass a concrete type as a argument to a function\n```rust\nfn main() {\n let mut buffer: Vec = vec![];\n writer(&mut buffer);\n}\n\nfn writer(w: &mut dyn Write) {\n // ...\n}\n```\nin both ways, buffer is converted to a trait object implements Write\n\nin memory, a trait object (in the example, w) is a fat point consists two pointers\n![trait_object_memory](/images/rust/memory/trait_object_memory.png)\nthe vtable is generated once at compile time and shared by all objects of the same type. the vtable contains pointers to the machine code of the functions that must be present for a type to be a \"Writer\"\n\n\n## references\n- https://www.youtube.com/watch?v=rDoqT-a6UFg","slug":"rust/rust-05-memory-layout","published":1,"updated":"2023-11-05T04:21:13.734Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8ds0015qwsjh6j437e0","content":"

trait object

a reference to a trait type (or Box) is called trait object

\n

two ways convert concrete type to trait object

    \n
  1. assigning to variable
    1
    2
    3
    4
    use std::io::Write;

    let mut buffer: Vec<u8> = vec![];
    let w: &mut dyn Write = &mut buffer;
    2. \n
  2. pass a concrete type as a argument to a function
    1
    2
    3
    4
    5
    6
    7
    8
    fn main() {
       let mut buffer: Vec<u8> = vec![];
       writer(&mut buffer);
    }

    fn writer(w: &mut dyn Write) {
        // ...
    }
    \nin both ways, buffer is converted to a trait object implements Write
    3. \n
\n

in memory, a trait object (in the example, w) is a fat point consists two pointers
\"trait_object_memory\"
the vtable is generated once at compile time and shared by all objects of the same type. the vtable contains pointers to the machine code of the functions that must be present for a type to be a “Writer”

\n

references

\n","site":{"data":{}},"excerpt":"","more":"

trait object

a reference to a trait type (or Box) is called trait object

\n

two ways convert concrete type to trait object

    \n
  1. assigning to variable
    1
    2
    3
    4
    use std::io::Write;

    let mut buffer: Vec<u8> = vec![];
    let w: &mut dyn Write = &mut buffer;
    2. \n
  2. pass a concrete type as a argument to a function
    1
    2
    3
    4
    5
    6
    7
    8
    fn main() {
       let mut buffer: Vec<u8> = vec![];
       writer(&mut buffer);
    }

    fn writer(w: &mut dyn Write) {
        // ...
    }
    \nin both ways, buffer is converted to a trait object implements Write
    3. \n
\n

in memory, a trait object (in the example, w) is a fat point consists two pointers
\"trait_object_memory\"
the vtable is generated once at compile time and shared by all objects of the same type. the vtable contains pointers to the machine code of the functions that must be present for a type to be a “Writer”

\n

references

\n"},{"title":"rust reflect and macro","date":"2022-12-28T02:24:21.000Z","_content":"\n## reflect\n### trait object\nRust provides dynamic dispatch through a feature called `trait objects`. Trait objects, like &Foo or Box, are normal values that store a value of any type that implements the given trait, where the precise type can only be known at **runtime**. more details can be found on [references 1]\n\n### any\nThis module (std::any) contains the `Any` trait, which enables dynamic typing of any 'static type through runtime reflection. It also contains the `Provider` trait and accompanying API, which enable trait objects to provide data based on typed requests, an alternate form of runtime reflection.\nAny itself can be used to get a TypeId\n\n```rust\nuse std::fmt::Debug;\nuse std::any::Any;\n\nfn log(value: &Any) {\n let value_any = value as &dyn Any; // &dyn Any (a borrowed trait object), Note that &dyn Any is limited to testing whether a value is of a specified concrete type, and cannot be used to test whether a type implements a trait.\n\n match value_any.downcast_ref::() {\n Some(val_str) -> {\n // do with string\n },\n None => {\n // \n }\n }\n}\n```\n\n### porpular crates using Any\n- [oso](https://docs.osohq.com/)\nThe Oso Library is a batteries-included framework for building authorization in your application\n- [bevy](https://bevyengine.org/)\nA refreshingly simple data-driven game engine built in Rust\n\n## macro\n### rust compile process\n![rust compile process](/images/rust/macros/16.compile_process.png)\n\n### front end: rustc\n1. lexical analysis: Code Text -> TokenStream\n2. syntax analysis: TokenStream -> AST (abstract syntax tree)\n3. semantic analyzer: \n AST -> HIR (High-Level Intermediate Representation) -> Type HIR (static type analysis, syntactic desugar, e.g `for` changed to `loop`) -> MIR: (Mid-Level Intermediate Representation, scope, reference & borrow check)\n\n### back end: LLVM\nLLVM IR -> machine code\n\n\n### macros in compiling\n- declarative macros: TokenStream - expand -> TokenStream\n- procedule macros: self defined AST with the help or third party crate such as syn, quote\n\n## declarative macro: macro_rules!\nDeclarative macros allow us to write match-like code. The match expression is a control structure that receives an expression and then matches the result of the expression with multiple patterns. Once a pattern is matched, the code associated with the pattern will be executed\n```rust\n#![allow(unused)]\nfn main() {\n match target {\n match_pattern_1 => expr_1,\n match_pattern_2 => {\n statement1;\n statement2;\n expr_2\n },\n _ => expr_3\n }\n}\n```\n\n### example 1, simplified `vec!`\nbelow example use macro_rules to implement a simplified version of vec!\n```rust\n#[macro_export]\nmacro_rules! vec {\n ( $( $x:expr ),* ) => {\n {\n let mut temp_vec = Vec::new();\n $(\n temp_vec.push($x);\n )*\n temp_vec\n }\n };\n}\n\n#![allow(unused)]\nfn main() {\n let v: Vec = vec![1, 2, 3];\n}\n```\n\n### example 2, unless\n```rust\nmacro_rules! unless {\n ( ($arg:expr) => $branch:tt ) => ( if !$arg {$branch};);\n}\n\nfn cmp(a:i32, b:i32) {\n unless!{\n (a>b) => {println!(\"{} < {}\", a,b);}\n }\n}\nfn main() {\n cmp(1,2); /// print \"1<2\" as the condition is true !(a>b)\n cmp(3,2); /// print nothing\n}\n```\n### example 3, HashMap\n```rust\nmacro_rules! hashmap {\n // match for \"a\" => 1, \"b\" => 2,\n ( $($key:expr => $value:expr,)* ) =>\n { hashmap!($($key => $value),*) }; // recuisive\n // match for \"a\" => 1, \"b\" => 2\n ( $($key:expr => $value:expr),* ) => { \n {\n let mut _map = ::std::collections::HashMap::new();\n $(\n _map.insert($key, $value);\n )*\n _map\n }\n \n };\n}\n\nmacro_rules! hashmap_equivalent {\n ( $($key:expr => $value:expr),* $(,)*) => { \n {\n let mut _map = ::std::collections::HashMap::new();\n $(\n _map.insert($key, $value);\n )*\n _map\n }\n \n };\n}\nfn main() {\n let map = hashmap!{\n \"a\" => 1,\n \"b\" => 2, // with or without ,\n };\n let map_2 = hashmap_equivalent!{\n \"a\" => 1, \n \"b\" => 2, // with or without ,\n };\n}\n```\n\n### metavariables\n- item: an Item\n- stmt: a Statement without the trailing semicolon (except for item statements that require semicolons)\n- expr: an Expression\n- ty: a Type\n- ident: an IDENTIFIER_OR_KEYWORD or RAW_IDENTIFIER\n- path: a TypePath style path\n- tt: a TokenTree (a single token or tokens in matching delimiters (), [], or {})\n- meta: an Attr, the contents of an attribute\n- lifetime: a LIFETIME_TOKEN\n- vis: a possibly empty Visibility qualifier\n- literal: matches LiteralExpression\ndetails to be found [here](https://doc.rust-lang.org/reference/macros-by-example.html)\n\n## procedures macro\n\n## references\n1. [rust trait object](https://web.mit.edu/rust-lang_v1.25/arch/amd64_ubuntu1404/share/doc/rust/html/book/first-edition/trait-objects.html)","source":"_posts/rust/rust-07-macro.md","raw":"---\ntitle: rust reflect and macro\ndate: 2022-12-28 10:24:21\ntags: [rust]\n---\n\n## reflect\n### trait object\nRust provides dynamic dispatch through a feature called `trait objects`. Trait objects, like &Foo or Box, are normal values that store a value of any type that implements the given trait, where the precise type can only be known at **runtime**. more details can be found on [references 1]\n\n### any\nThis module (std::any) contains the `Any` trait, which enables dynamic typing of any 'static type through runtime reflection. It also contains the `Provider` trait and accompanying API, which enable trait objects to provide data based on typed requests, an alternate form of runtime reflection.\nAny itself can be used to get a TypeId\n\n```rust\nuse std::fmt::Debug;\nuse std::any::Any;\n\nfn log(value: &Any) {\n let value_any = value as &dyn Any; // &dyn Any (a borrowed trait object), Note that &dyn Any is limited to testing whether a value is of a specified concrete type, and cannot be used to test whether a type implements a trait.\n\n match value_any.downcast_ref::() {\n Some(val_str) -> {\n // do with string\n },\n None => {\n // \n }\n }\n}\n```\n\n### porpular crates using Any\n- [oso](https://docs.osohq.com/)\nThe Oso Library is a batteries-included framework for building authorization in your application\n- [bevy](https://bevyengine.org/)\nA refreshingly simple data-driven game engine built in Rust\n\n## macro\n### rust compile process\n![rust compile process](/images/rust/macros/16.compile_process.png)\n\n### front end: rustc\n1. lexical analysis: Code Text -> TokenStream\n2. syntax analysis: TokenStream -> AST (abstract syntax tree)\n3. semantic analyzer: \n AST -> HIR (High-Level Intermediate Representation) -> Type HIR (static type analysis, syntactic desugar, e.g `for` changed to `loop`) -> MIR: (Mid-Level Intermediate Representation, scope, reference & borrow check)\n\n### back end: LLVM\nLLVM IR -> machine code\n\n\n### macros in compiling\n- declarative macros: TokenStream - expand -> TokenStream\n- procedule macros: self defined AST with the help or third party crate such as syn, quote\n\n## declarative macro: macro_rules!\nDeclarative macros allow us to write match-like code. The match expression is a control structure that receives an expression and then matches the result of the expression with multiple patterns. Once a pattern is matched, the code associated with the pattern will be executed\n```rust\n#![allow(unused)]\nfn main() {\n match target {\n match_pattern_1 => expr_1,\n match_pattern_2 => {\n statement1;\n statement2;\n expr_2\n },\n _ => expr_3\n }\n}\n```\n\n### example 1, simplified `vec!`\nbelow example use macro_rules to implement a simplified version of vec!\n```rust\n#[macro_export]\nmacro_rules! vec {\n ( $( $x:expr ),* ) => {\n {\n let mut temp_vec = Vec::new();\n $(\n temp_vec.push($x);\n )*\n temp_vec\n }\n };\n}\n\n#![allow(unused)]\nfn main() {\n let v: Vec = vec![1, 2, 3];\n}\n```\n\n### example 2, unless\n```rust\nmacro_rules! unless {\n ( ($arg:expr) => $branch:tt ) => ( if !$arg {$branch};);\n}\n\nfn cmp(a:i32, b:i32) {\n unless!{\n (a>b) => {println!(\"{} < {}\", a,b);}\n }\n}\nfn main() {\n cmp(1,2); /// print \"1<2\" as the condition is true !(a>b)\n cmp(3,2); /// print nothing\n}\n```\n### example 3, HashMap\n```rust\nmacro_rules! hashmap {\n // match for \"a\" => 1, \"b\" => 2,\n ( $($key:expr => $value:expr,)* ) =>\n { hashmap!($($key => $value),*) }; // recuisive\n // match for \"a\" => 1, \"b\" => 2\n ( $($key:expr => $value:expr),* ) => { \n {\n let mut _map = ::std::collections::HashMap::new();\n $(\n _map.insert($key, $value);\n )*\n _map\n }\n \n };\n}\n\nmacro_rules! hashmap_equivalent {\n ( $($key:expr => $value:expr),* $(,)*) => { \n {\n let mut _map = ::std::collections::HashMap::new();\n $(\n _map.insert($key, $value);\n )*\n _map\n }\n \n };\n}\nfn main() {\n let map = hashmap!{\n \"a\" => 1,\n \"b\" => 2, // with or without ,\n };\n let map_2 = hashmap_equivalent!{\n \"a\" => 1, \n \"b\" => 2, // with or without ,\n };\n}\n```\n\n### metavariables\n- item: an Item\n- stmt: a Statement without the trailing semicolon (except for item statements that require semicolons)\n- expr: an Expression\n- ty: a Type\n- ident: an IDENTIFIER_OR_KEYWORD or RAW_IDENTIFIER\n- path: a TypePath style path\n- tt: a TokenTree (a single token or tokens in matching delimiters (), [], or {})\n- meta: an Attr, the contents of an attribute\n- lifetime: a LIFETIME_TOKEN\n- vis: a possibly empty Visibility qualifier\n- literal: matches LiteralExpression\ndetails to be found [here](https://doc.rust-lang.org/reference/macros-by-example.html)\n\n## procedures macro\n\n## references\n1. [rust trait object](https://web.mit.edu/rust-lang_v1.25/arch/amd64_ubuntu1404/share/doc/rust/html/book/first-edition/trait-objects.html)","slug":"rust/rust-07-macro","published":1,"updated":"2023-11-05T04:21:13.734Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8ds0017qwsj9lqm1w2r","content":"

reflect

trait object

Rust provides dynamic dispatch through a feature called trait objects. Trait objects, like &Foo or Box, are normal values that store a value of any type that implements the given trait, where the precise type can only be known at runtime. more details can be found on [references 1]

\n

any

This module (std::any) contains the Any trait, which enables dynamic typing of any 'static type through runtime reflection. It also contains the Provider trait and accompanying API, which enable trait objects to provide data based on typed requests, an alternate form of runtime reflection.
Any itself can be used to get a TypeId

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
use std::fmt::Debug;
use std::any::Any;

fn log<T: Any + Debug>(value: &Any) {
    let value_any = value as &dyn Any;  // &dyn Any (a borrowed trait object), Note that &dyn Any is limited to testing whether a value is of a specified concrete type, and cannot be used to test whether a type implements a trait.

    match value_any.downcast_ref::<String>() {
        Some(val_str) -> {
            // do with string
        },
        None => {
            // 
        }
    }
}
\n\n

porpular crates using Any

    \n
  • oso
    The Oso Library is a batteries-included framework for building authorization in your application
    • \n
  • bevy
    A refreshingly simple data-driven game engine built in Rust
    • \n
\n

macro

rust compile process

\"rust

\n

front end: rustc

    \n
  1. lexical analysis: Code Text -> TokenStream
    2. \n
  2. syntax analysis: TokenStream -> AST (abstract syntax tree)
    3. \n
  3. semantic analyzer:
    AST -> HIR (High-Level Intermediate Representation) -> Type HIR (static type analysis, syntactic desugar, e.g for changed to loop) -> MIR: (Mid-Level Intermediate Representation, scope, reference & borrow check)
    4. \n
\n

back end: LLVM

LLVM IR -> machine code

\n

macros in compiling

    \n
  • declarative macros: TokenStream - expand -> TokenStream
    • \n
  • procedule macros: self defined AST with the help or third party crate such as syn, quote
    • \n
\n

declarative macro: macro_rules!

Declarative macros allow us to write match-like code. The match expression is a control structure that receives an expression and then matches the result of the expression with multiple patterns. Once a pattern is matched, the code associated with the pattern will be executed

\n
1
2
3
4
5
6
7
8
9
10
11
12
#![allow(unused)]
fn main() {
    match target {
        match_pattern_1 => expr_1,
        match_pattern_2 => {
            statement1;
            statement2;
            expr_2
        },
        _ => expr_3
    }
}
\n\n

example 1, simplified vec!

below example use macro_rules to implement a simplified version of vec!

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
#[macro_export]
macro_rules! vec {
    ( $( $x:expr ),* ) => {
        {
            let mut temp_vec = Vec::new();
            $(
                temp_vec.push($x);
            )*
            temp_vec
        }
    };
}

#![allow(unused)]
fn main() {
    let v: Vec<u32> = vec![1, 2, 3];
}
\n\n

example 2, unless

1
2
3
4
5
6
7
8
9
10
11
12
13
macro_rules! unless {
    ( ($arg:expr) => $branch:tt ) => ( if !$arg {$branch};);
}

fn cmp(a:i32, b:i32) {
    unless!{
        (a>b) => {println!("{} < {}", a,b);}
    }
}
fn main() {
    cmp(1,2);  /// print "1<2" as the condition is true !(a>b)
    cmp(3,2);  /// print nothing
}
\n

example 3, HashMap

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
macro_rules! hashmap {
    // match for "a" => 1, "b" => 2,
    ( $($key:expr => $value:expr,)* ) =>
        { hashmap!($($key => $value),*) }; // recuisive
    // match for "a" => 1, "b" => 2
    ( $($key:expr => $value:expr),* ) => { 
        {
            let mut _map = ::std::collections::HashMap::new();
            $(
                _map.insert($key, $value);
            )*
            _map
        }
       
    };
}

macro_rules! hashmap_equivalent {
    ( $($key:expr => $value:expr),* $(,)*) => { 
        {
            let mut _map = ::std::collections::HashMap::new();
            $(
                _map.insert($key, $value);
            )*
            _map
        }
       
    };
}
fn main() {
    let map = hashmap!{
        "a" => 1,
        "b" => 2,  // with or without ,
    };
    let map_2 = hashmap_equivalent!{
        "a" => 1, 
        "b" => 2,  // with or without ,
    };
}
\n\n

metavariables

    \n
  • item: an Item
    • \n
  • stmt: a Statement without the trailing semicolon (except for item statements that require semicolons)
    • \n
  • expr: an Expression
    • \n
  • ty: a Type
    • \n
  • ident: an IDENTIFIER_OR_KEYWORD or RAW_IDENTIFIER
    • \n
  • path: a TypePath style path
    • \n
  • tt: a TokenTree (a single token or tokens in matching delimiters (), [], or {})
    • \n
  • meta: an Attr, the contents of an attribute
    • \n
  • lifetime: a LIFETIME_TOKEN
    • \n
  • vis: a possibly empty Visibility qualifier
    • \n
  • literal: matches LiteralExpression
    details to be found here
    • \n
\n

procedures macro

references

    \n
  1. rust trait object
    2. \n
\n","site":{"data":{}},"excerpt":"","more":"

reflect

trait object

Rust provides dynamic dispatch through a feature called trait objects. Trait objects, like &Foo or Box, are normal values that store a value of any type that implements the given trait, where the precise type can only be known at runtime. more details can be found on [references 1]

\n

any

This module (std::any) contains the Any trait, which enables dynamic typing of any 'static type through runtime reflection. It also contains the Provider trait and accompanying API, which enable trait objects to provide data based on typed requests, an alternate form of runtime reflection.
Any itself can be used to get a TypeId

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
use std::fmt::Debug;
use std::any::Any;

fn log<T: Any + Debug>(value: &Any) {
    let value_any = value as &dyn Any;  // &dyn Any (a borrowed trait object), Note that &dyn Any is limited to testing whether a value is of a specified concrete type, and cannot be used to test whether a type implements a trait.

    match value_any.downcast_ref::<String>() {
        Some(val_str) -> {
            // do with string
        },
        None => {
            // 
        }
    }
}
\n\n

porpular crates using Any

    \n
  • oso
    The Oso Library is a batteries-included framework for building authorization in your application
    • \n
  • bevy
    A refreshingly simple data-driven game engine built in Rust
    • \n
\n

macro

rust compile process

\"rust

\n

front end: rustc

    \n
  1. lexical analysis: Code Text -> TokenStream
    2. \n
  2. syntax analysis: TokenStream -> AST (abstract syntax tree)
    3. \n
  3. semantic analyzer:
    AST -> HIR (High-Level Intermediate Representation) -> Type HIR (static type analysis, syntactic desugar, e.g for changed to loop) -> MIR: (Mid-Level Intermediate Representation, scope, reference & borrow check)
    4. \n
\n

back end: LLVM

LLVM IR -> machine code

\n

macros in compiling

    \n
  • declarative macros: TokenStream - expand -> TokenStream
    • \n
  • procedule macros: self defined AST with the help or third party crate such as syn, quote
    • \n
\n

declarative macro: macro_rules!

Declarative macros allow us to write match-like code. The match expression is a control structure that receives an expression and then matches the result of the expression with multiple patterns. Once a pattern is matched, the code associated with the pattern will be executed

\n
1
2
3
4
5
6
7
8
9
10
11
12
#![allow(unused)]
fn main() {
    match target {
        match_pattern_1 => expr_1,
        match_pattern_2 => {
            statement1;
            statement2;
            expr_2
        },
        _ => expr_3
    }
}
\n\n

example 1, simplified vec!

below example use macro_rules to implement a simplified version of vec!

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
#[macro_export]
macro_rules! vec {
    ( $( $x:expr ),* ) => {
        {
            let mut temp_vec = Vec::new();
            $(
                temp_vec.push($x);
            )*
            temp_vec
        }
    };
}

#![allow(unused)]
fn main() {
    let v: Vec<u32> = vec![1, 2, 3];
}
\n\n

example 2, unless

1
2
3
4
5
6
7
8
9
10
11
12
13
macro_rules! unless {
    ( ($arg:expr) => $branch:tt ) => ( if !$arg {$branch};);
}

fn cmp(a:i32, b:i32) {
    unless!{
        (a>b) => {println!("{} < {}", a,b);}
    }
}
fn main() {
    cmp(1,2);  /// print "1<2" as the condition is true !(a>b)
    cmp(3,2);  /// print nothing
}
\n

example 3, HashMap

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
macro_rules! hashmap {
    // match for "a" => 1, "b" => 2,
    ( $($key:expr => $value:expr,)* ) =>
        { hashmap!($($key => $value),*) }; // recuisive
    // match for "a" => 1, "b" => 2
    ( $($key:expr => $value:expr),* ) => { 
        {
            let mut _map = ::std::collections::HashMap::new();
            $(
                _map.insert($key, $value);
            )*
            _map
        }
       
    };
}

macro_rules! hashmap_equivalent {
    ( $($key:expr => $value:expr),* $(,)*) => { 
        {
            let mut _map = ::std::collections::HashMap::new();
            $(
                _map.insert($key, $value);
            )*
            _map
        }
       
    };
}
fn main() {
    let map = hashmap!{
        "a" => 1,
        "b" => 2,  // with or without ,
    };
    let map_2 = hashmap_equivalent!{
        "a" => 1, 
        "b" => 2,  // with or without ,
    };
}
\n\n

metavariables

    \n
  • item: an Item
    • \n
  • stmt: a Statement without the trailing semicolon (except for item statements that require semicolons)
    • \n
  • expr: an Expression
    • \n
  • ty: a Type
    • \n
  • ident: an IDENTIFIER_OR_KEYWORD or RAW_IDENTIFIER
    • \n
  • path: a TypePath style path
    • \n
  • tt: a TokenTree (a single token or tokens in matching delimiters (), [], or {})
    • \n
  • meta: an Attr, the contents of an attribute
    • \n
  • lifetime: a LIFETIME_TOKEN
    • \n
  • vis: a possibly empty Visibility qualifier
    • \n
  • literal: matches LiteralExpression
    details to be found here
    • \n
\n

procedures macro

references

    \n
  1. rust trait object
    2. \n
\n"},{"title":"rust project management","date":"2022-11-06T07:33:55.000Z","_content":"\n## basic concepts\n- **Packages**: A Cargo feature that lets you build, test, and share crates\n- **Crates**: A tree of modules that produces a library or executable\n- **Modules and use**: Let you control the organization, scope, and privacy of paths\n- **Paths**: A way of naming an item, such as a struct, function, or module\n\n### package\n1. one package can only has one library Crate. default is src/lib.rs\n2. one package can have multiple binary Crates (under src/bin). default src/main.rs\n3. one package has at least one crate，no matter lib or bin。\n4. one package could simultaneously have src/main.rs and src/lib.rs (crate name same to package name)\n```\ncargo new my-project --lib ## create a library project\n```\n### crate\n- binary or library\n- The crate root is a source file that the Rust compiler starts from and makes up the root module of your crate\n\n### module\n- keyword `mod` \n- module could be nested。\n- module includes(struct、enum、const、trait、func, etc)\n\n### path\n- absolute path: use crate name or `crate`\n- relative path:，use self, super etc\n```rust\nfn serve_order() {}\nmod back_of_house {\n fn fix_incorrect_order() {\n cook_order();\n super::serve_order();\n }\n fn cook_order() {}\n}\n```\n```rust\nmod front_of_house {\n pub mod hosting {\n pub fn add_to_waitlist() {}\n }\n}\npub fn eat_at_restaurant() {\n // Absolute path\n crate::front_of_house::hosting::add_to_waitlist();\n // Relative path\n front_of_house::hosting::add_to_waitlist();\n}\n```\n\n### Bringing Paths into Scope with the use Keyword\n```rust\nuse std::io;\nuse std::io::Write;\nuse std::io:: { self, Write} ;\nuse std::{cmp::Ordering, io};\nuse std::fmt::Result;\nuse std::io::Result as IoResult;\nuse std::collections::*;\n\npub use crate::front_of_house::hosting; // re-exporting names with pub use\n```\n### Separating Modules into Different Files\nFilename: src/lib.rs\n```rust\nmod front_of_house;\n\npub use crate::front_of_house::hosting;\n\npub fn eat_at_restaurant() {\n hosting::add_to_waitlist();\n hosting::add_to_waitlist();\n hosting::add_to_waitlist();\n}\n```\nFilename: src/front_of_house.rs\n```rust\npub mod hosting {\n pub fn add_to_waitlist() {}\n}\n```\n\n## profile\n```\ncargo build ## dev: [unoptimized + debuginfo]\ncargo build --release ## [optimized]\n```\n- config\n```\n[profile.dev]\nopt-level = 0\n\n[profile.release]\nopt-level = 3\n\n# profile for the wasm example (from project ethers-rs)\n[profile.release.package.ethers-wasm]\nopt-level = \"s\" # Tell `rustc` to optimize for small code size.\n```\n","source":"_posts/rust/rust-08-project-management.md","raw":"---\ntitle: rust project management\ndate: 2022-11-06 15:33:55\ntags: [rust]\n---\n\n## basic concepts\n- **Packages**: A Cargo feature that lets you build, test, and share crates\n- **Crates**: A tree of modules that produces a library or executable\n- **Modules and use**: Let you control the organization, scope, and privacy of paths\n- **Paths**: A way of naming an item, such as a struct, function, or module\n\n### package\n1. one package can only has one library Crate. default is src/lib.rs\n2. one package can have multiple binary Crates (under src/bin). default src/main.rs\n3. one package has at least one crate，no matter lib or bin。\n4. one package could simultaneously have src/main.rs and src/lib.rs (crate name same to package name)\n```\ncargo new my-project --lib ## create a library project\n```\n### crate\n- binary or library\n- The crate root is a source file that the Rust compiler starts from and makes up the root module of your crate\n\n### module\n- keyword `mod` \n- module could be nested。\n- module includes(struct、enum、const、trait、func, etc)\n\n### path\n- absolute path: use crate name or `crate`\n- relative path:，use self, super etc\n```rust\nfn serve_order() {}\nmod back_of_house {\n fn fix_incorrect_order() {\n cook_order();\n super::serve_order();\n }\n fn cook_order() {}\n}\n```\n```rust\nmod front_of_house {\n pub mod hosting {\n pub fn add_to_waitlist() {}\n }\n}\npub fn eat_at_restaurant() {\n // Absolute path\n crate::front_of_house::hosting::add_to_waitlist();\n // Relative path\n front_of_house::hosting::add_to_waitlist();\n}\n```\n\n### Bringing Paths into Scope with the use Keyword\n```rust\nuse std::io;\nuse std::io::Write;\nuse std::io:: { self, Write} ;\nuse std::{cmp::Ordering, io};\nuse std::fmt::Result;\nuse std::io::Result as IoResult;\nuse std::collections::*;\n\npub use crate::front_of_house::hosting; // re-exporting names with pub use\n```\n### Separating Modules into Different Files\nFilename: src/lib.rs\n```rust\nmod front_of_house;\n\npub use crate::front_of_house::hosting;\n\npub fn eat_at_restaurant() {\n hosting::add_to_waitlist();\n hosting::add_to_waitlist();\n hosting::add_to_waitlist();\n}\n```\nFilename: src/front_of_house.rs\n```rust\npub mod hosting {\n pub fn add_to_waitlist() {}\n}\n```\n\n## profile\n```\ncargo build ## dev: [unoptimized + debuginfo]\ncargo build --release ## [optimized]\n```\n- config\n```\n[profile.dev]\nopt-level = 0\n\n[profile.release]\nopt-level = 3\n\n# profile for the wasm example (from project ethers-rs)\n[profile.release.package.ethers-wasm]\nopt-level = \"s\" # Tell `rustc` to optimize for small code size.\n```\n","slug":"rust/rust-08-project-management","published":1,"updated":"2023-11-05T04:21:13.734Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8ds0018qwsj3eer6n3l","content":"

basic concepts

    \n
  • Packages: A Cargo feature that lets you build, test, and share crates
    • \n
  • Crates: A tree of modules that produces a library or executable
    • \n
  • Modules and use: Let you control the organization, scope, and privacy of paths
    • \n
  • Paths: A way of naming an item, such as a struct, function, or module
    • \n
\n

package

    \n
  1. one package can only has one library Crate. default is src/lib.rs
    2. \n
  2. one package can have multiple binary Crates (under src/bin). default src/main.rs
    3. \n
  3. one package has at least one crate，no matter lib or bin。
    4. \n
  4. one package could simultaneously have src/main.rs and src/lib.rs (crate name same to package name)
    1
    cargo new my-project --lib  ## create a library project
    5. \n
\n

crate

    \n
  • binary or library
    • \n
  • The crate root is a source file that the Rust compiler starts from and makes up the root module of your crate
    • \n
\n

module

    \n
  • keyword mod
    • \n
  • module could be nested。
    • \n
  • module includes(struct、enum、const、trait、func, etc)
    • \n
\n

path

    \n
  • absolute path: use crate name or crate
    • \n
  • relative path:，use self, super etc
    1
    2
    3
    4
    5
    6
    7
    8
    fn serve_order() {}
    mod back_of_house {
        fn fix_incorrect_order() {
            cook_order();
            super::serve_order();
        }
        fn cook_order() {}
    }
    \n
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    mod front_of_house {
        pub mod hosting {
            pub fn add_to_waitlist() {}
        }
    }
    pub fn eat_at_restaurant() {
        // Absolute path
        crate::front_of_house::hosting::add_to_waitlist();
        // Relative path
        front_of_house::hosting::add_to_waitlist();
    }
    • \n
\n

Bringing Paths into Scope with the use Keyword

1
2
3
4
5
6
7
8
9
use std::io;
use std::io::Write;
use std::io:: { self, Write} ;
use std::{cmp::Ordering, io};
use std::fmt::Result;
use std::io::Result as IoResult;
use std::collections::*;

pub use crate::front_of_house::hosting; // re-exporting names with pub use
\n

Separating Modules into Different Files

Filename: src/lib.rs

\n
1
2
3
4
5
6
7
8
9
mod front_of_house;

pub use crate::front_of_house::hosting;

pub fn eat_at_restaurant() {
    hosting::add_to_waitlist();
    hosting::add_to_waitlist();
    hosting::add_to_waitlist();
}
\n

Filename: src/front_of_house.rs

\n
1
2
3
pub mod hosting {
    pub fn add_to_waitlist() {}
}
\n\n

profile

1
2
cargo build ## dev: [unoptimized + debuginfo]
cargo build --release ## [optimized]
\n
    \n
  • config
    1
    2
    3
    4
    5
    6
    7
    8
    9
    [profile.dev]
    opt-level = 0

    [profile.release]
    opt-level = 3

    # profile for the wasm example (from project ethers-rs)
    [profile.release.package.ethers-wasm]
    opt-level = "s"  # Tell `rustc` to optimize for small code size.
    • \n
\n","site":{"data":{}},"excerpt":"","more":"

basic concepts

    \n
  • Packages: A Cargo feature that lets you build, test, and share crates
    • \n
  • Crates: A tree of modules that produces a library or executable
    • \n
  • Modules and use: Let you control the organization, scope, and privacy of paths
    • \n
  • Paths: A way of naming an item, such as a struct, function, or module
    • \n
\n

package

    \n
  1. one package can only has one library Crate. default is src/lib.rs
    2. \n
  2. one package can have multiple binary Crates (under src/bin). default src/main.rs
    3. \n
  3. one package has at least one crate，no matter lib or bin。
    4. \n
  4. one package could simultaneously have src/main.rs and src/lib.rs (crate name same to package name)
    1
    cargo new my-project --lib  ## create a library project
    5. \n
\n

crate

    \n
  • binary or library
    • \n
  • The crate root is a source file that the Rust compiler starts from and makes up the root module of your crate
    • \n
\n

module

    \n
  • keyword mod
    • \n
  • module could be nested。
    • \n
  • module includes(struct、enum、const、trait、func, etc)
    • \n
\n

path

    \n
  • absolute path: use crate name or crate
    • \n
  • relative path:，use self, super etc
    1
    2
    3
    4
    5
    6
    7
    8
    fn serve_order() {}
    mod back_of_house {
        fn fix_incorrect_order() {
            cook_order();
            super::serve_order();
        }
        fn cook_order() {}
    }
    \n
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    mod front_of_house {
        pub mod hosting {
            pub fn add_to_waitlist() {}
        }
    }
    pub fn eat_at_restaurant() {
        // Absolute path
        crate::front_of_house::hosting::add_to_waitlist();
        // Relative path
        front_of_house::hosting::add_to_waitlist();
    }
    • \n
\n

Bringing Paths into Scope with the use Keyword

1
2
3
4
5
6
7
8
9
use std::io;
use std::io::Write;
use std::io:: { self, Write} ;
use std::{cmp::Ordering, io};
use std::fmt::Result;
use std::io::Result as IoResult;
use std::collections::*;

pub use crate::front_of_house::hosting; // re-exporting names with pub use
\n

Separating Modules into Different Files

Filename: src/lib.rs

\n
1
2
3
4
5
6
7
8
9
mod front_of_house;

pub use crate::front_of_house::hosting;

pub fn eat_at_restaurant() {
    hosting::add_to_waitlist();
    hosting::add_to_waitlist();
    hosting::add_to_waitlist();
}
\n

Filename: src/front_of_house.rs

\n
1
2
3
pub mod hosting {
    pub fn add_to_waitlist() {}
}
\n\n

profile

1
2
cargo build ## dev: [unoptimized + debuginfo]
cargo build --release ## [optimized]
\n
    \n
  • config
    1
    2
    3
    4
    5
    6
    7
    8
    9
    [profile.dev]
    opt-level = 0

    [profile.release]
    opt-level = 3

    # profile for the wasm example (from project ethers-rs)
    [profile.release.package.ethers-wasm]
    opt-level = "s"  # Tell `rustc` to optimize for small code size.
    • \n
\n"},{"title":"rust functional programming","date":"2023-04-11T14:04:38.000Z","_content":"\n## iterator\n- `iter()` Returns an iterator over the **slice**\n- `into_iter()` Creates a consuming iterator, that is, one that moves each value out of the vector\n- `iter_mut()` Returns an iterator that allows modifying each value.\n\n## flat_map\n```rust\n/// Creates an iterator that works like map, but flattens nested structure.\n///\n/// The [`map`] adapter is very useful, but only when the closure\n/// argument produces values. If it produces an iterator instead, there's\n/// an extra layer of indirection. `flat_map()` will remove this extra layer\n/// on its own.\n```\n### Examples\n```rust\nlet words = [\"alpha\", \"beta\", \"gamma\"];\n// chars() returns an iterator\nlet merged: String = words.iter()\n .flat_map(|s| s.chars())\n .collect();\nassert_eq!(merged, \"alphabetagamma\");\n```\n","source":"_posts/rust/rust-09-functional.md","raw":"---\ntitle: rust functional programming\ndate: 2023-04-11 22:04:38\ntags: [rust]\n---\n\n## iterator\n- `iter()` Returns an iterator over the **slice**\n- `into_iter()` Creates a consuming iterator, that is, one that moves each value out of the vector\n- `iter_mut()` Returns an iterator that allows modifying each value.\n\n## flat_map\n```rust\n/// Creates an iterator that works like map, but flattens nested structure.\n///\n/// The [`map`] adapter is very useful, but only when the closure\n/// argument produces values. If it produces an iterator instead, there's\n/// an extra layer of indirection. `flat_map()` will remove this extra layer\n/// on its own.\n```\n### Examples\n```rust\nlet words = [\"alpha\", \"beta\", \"gamma\"];\n// chars() returns an iterator\nlet merged: String = words.iter()\n .flat_map(|s| s.chars())\n .collect();\nassert_eq!(merged, \"alphabetagamma\");\n```\n","slug":"rust/rust-09-functional","published":1,"updated":"2023-11-05T04:21:13.735Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8ds001aqwsjdxc70zfv","content":"

iterator

    \n
  • iter() Returns an iterator over the slice
    • \n
  • into_iter() Creates a consuming iterator, that is, one that moves each value out of the vector
    • \n
  • iter_mut() Returns an iterator that allows modifying each value.
    • \n
\n

flat_map

1
2
3
4
5
6
/// Creates an iterator that works like map, but flattens nested structure.
///
/// The [`map`] adapter is very useful, but only when the closure
/// argument produces values. If it produces an iterator instead, there's
/// an extra layer of indirection. `flat_map()` will remove this extra layer
/// on its own.
\n

Examples

1
2
3
4
5
6
let words = ["alpha", "beta", "gamma"];
// chars() returns an iterator
let merged: String = words.iter()
                          .flat_map(|s| s.chars())
                          .collect();
assert_eq!(merged, "alphabetagamma");
\n","site":{"data":{}},"excerpt":"","more":"

iterator

    \n
  • iter() Returns an iterator over the slice
    • \n
  • into_iter() Creates a consuming iterator, that is, one that moves each value out of the vector
    • \n
  • iter_mut() Returns an iterator that allows modifying each value.
    • \n
\n

flat_map

1
2
3
4
5
6
/// Creates an iterator that works like map, but flattens nested structure.
///
/// The [`map`] adapter is very useful, but only when the closure
/// argument produces values. If it produces an iterator instead, there's
/// an extra layer of indirection. `flat_map()` will remove this extra layer
/// on its own.
\n

Examples

1
2
3
4
5
6
let words = ["alpha", "beta", "gamma"];
// chars() returns an iterator
let merged: String = words.iter()
                          .flat_map(|s| s.chars())
                          .collect();
assert_eq!(merged, "alphabetagamma");
\n"},{"title":"rust async","date":"2023-01-13T09:17:10.000Z","_content":" \n\n## I/O\nI/O：在计算机中指Input/Output。由于程序和运行时数据是在内存中驻留，由cpu来执行，涉及到数据交换的地方，通常是磁盘、网卡等，就需要IO接口\n\n\n### I/O 模型\n```plantuml\n@startmindmap\n+ **I/O模型**\n++ 同步I/O\n'tag::details[]\n+++_ 阻塞I/O (BIO)\n+++_ 非阻塞I/O (NIO)\n+++_ I/O多路复用\n+++_ 信号驱动I/O\n'end::details[]\n++ 异步I/O\n'tag::details[]\n+++_ linux (AIO, io_uring)\n+++_ windows (IOCP)\n'end::details[]\n@endmindmap\n```\n#### 同步阻塞\n- 当用户线程发起IO请求后，会进行系统调用（system call）来让内核（Kernel）进行IO操作（系统调用是用户空间和内核空间的一个通道）；\n- 此时用户线程阻塞，等待内核将数据准备好；\n- 内核将数据准备好后会将数据从内核空间拷贝到用户空间，并返回给用户线程结束阻塞。\n\n#### 同步非阻塞\n- 由用户线程发起IO请求，进行系统调用来让内核进行IO操作；\n- 此时如果内核没有准备好数据则会直接返回error，并不会阻塞用户线程，用户线程可以重复发起IO请求；\n- 当用户线程发起请求并且内核已经将数据准备好后，会将数据从内核空间拷贝到用户空间（这个过程是需要阻塞用户线程的），返回给用户。\n\n#### 同步多路复用\n- 用户线程调用select后进行系统调用（内核会监视所有select负责的socket）\n- 当用户将数据准备好后就会返回，并通知用户线程进行读取操作，此时内核将数据拷贝到用户空间并返回。此时用户线程被阻塞；\n\n#### 异步I/O\n- 用户线程进行aio_read，进行系统调用切换到内核；\n- 内核立即返回，并不会阻塞用户线程；\n- 内核准备好数据后会将数据从内核空间拷贝到用户空间并通知用户线程（发送信号）操作已完成。\n\n### 流程图\n#### 同步blocking I/O\n\n```plantuml\n@startuml Test Diagram\n\nparticipant \"application\" as app\nparticipant \"kernel\" as kernel\n\nactivate app\nactivate kernel\napp -> kernel: syscall: Read recvfrom\nkernel -> kernel: wait for data (no datagram ready)\nkernel -> kernel: copy datagram to user (datagram ready)\nkernel -> app: return\n@enduml\n```\n\n#### I/O多路复用\n\n### 异步编程\n\n\n## the Future Trait\nA Future is an asynchronous computation that can produce a value. A simplified version of the future trait might look something like this\n\n```rust\n\ntrait SimpleFuture {\n type Output;\n fn poll(&mut self, wake: fn()) -> Poll;\n}\n\nenum Poll {\n Ready(T),\n Pending,\n}\n```\n\nFor example, consider the case where we want to read from a socket that may or may not have data available already.\n```rust\npub struct SocketRead<'a> {\n socket: &'a Socket,\n}\n\nimpl SimpleFuture for SocketRead<'_> {\n type Output = Vec;\n\n fn poll(&mut self, wake: fn()) -> Poll {\n if self.socket.has_data_to_read() {\n // The socket has data -- read it into a buffer and return it.\n Poll::Ready(self.socket.read_buf())\n } else {\n // The socket does not yet have data.\n //\n // Arrange for `wake` to be called once data is available.\n // When data becomes available, `wake` will be called, and the\n // user of this `Future` will know to call `poll` again and\n // receive data.\n self.socket.set_readable_callback(wake);\n Poll::Pending\n }\n }\n}\n\n```\n\n the real Future trait and how it is different\n```rust\ntrait Future {\n type Output;\n fn poll(\n // Note the change from `&mut self` to `Pin<&mut Self>`:\n self: Pin<&mut Self>,\n // and the change from `wake: fn()` to `cx: &mut Context<'_>`:\n cx: &mut Context<'_>,\n ) -> Poll;\n}\n\n```\nThe first change you'll notice is that our self type is no longer &mut Self, but has changed to Pin<&mut Self>. We'll talk more about pinning in a later section, but for now know that it allows us to create futures that are immovable. Immovable objects can store pointers between their fields, e.g. struct MyFut { a: i32, ptr_to_a: *const i32 }. Pinning is necessary to enable async/await.\n\nSecondly, wake: fn() has changed to &mut Context<'_>. In SimpleFuture, we used a call to a function pointer (fn()) to tell the future executor that the future in question should be polled. However, since fn() is just a function pointer, it can't store any data about which Future called wake.\n\nIn a real-world scenario, a complex application like a web server may have thousands of different connections whose wakeups should all be managed separately. The Context type solves this by providing access to a value of type Waker, which can be used to wake up a specific task.\n\n\n## task wakeups with Waker\nWaker provides a wake() method that can be used to tell the executor that the associated task should be awoken. When wake() is called, the executor knows that the task associated with the Waker is ready to make progress, and its future should be polled again.\n\n## referencs\n[csdn blog](https://blog.csdn.net/XMJYever/article/details/111560976)\n","source":"_posts/rust/rust-async.md","raw":"---\ntitle: rust async\ndate: 2023-01-13 17:17:10\ntags: [rust]\n--- \n\n## I/O\nI/O：在计算机中指Input/Output。由于程序和运行时数据是在内存中驻留，由cpu来执行，涉及到数据交换的地方，通常是磁盘、网卡等，就需要IO接口\n\n\n### I/O 模型\n```plantuml\n@startmindmap\n+ **I/O模型**\n++ 同步I/O\n'tag::details[]\n+++_ 阻塞I/O (BIO)\n+++_ 非阻塞I/O (NIO)\n+++_ I/O多路复用\n+++_ 信号驱动I/O\n'end::details[]\n++ 异步I/O\n'tag::details[]\n+++_ linux (AIO, io_uring)\n+++_ windows (IOCP)\n'end::details[]\n@endmindmap\n```\n#### 同步阻塞\n- 当用户线程发起IO请求后，会进行系统调用（system call）来让内核（Kernel）进行IO操作（系统调用是用户空间和内核空间的一个通道）；\n- 此时用户线程阻塞，等待内核将数据准备好；\n- 内核将数据准备好后会将数据从内核空间拷贝到用户空间，并返回给用户线程结束阻塞。\n\n#### 同步非阻塞\n- 由用户线程发起IO请求，进行系统调用来让内核进行IO操作；\n- 此时如果内核没有准备好数据则会直接返回error，并不会阻塞用户线程，用户线程可以重复发起IO请求；\n- 当用户线程发起请求并且内核已经将数据准备好后，会将数据从内核空间拷贝到用户空间（这个过程是需要阻塞用户线程的），返回给用户。\n\n#### 同步多路复用\n- 用户线程调用select后进行系统调用（内核会监视所有select负责的socket）\n- 当用户将数据准备好后就会返回，并通知用户线程进行读取操作，此时内核将数据拷贝到用户空间并返回。此时用户线程被阻塞；\n\n#### 异步I/O\n- 用户线程进行aio_read，进行系统调用切换到内核；\n- 内核立即返回，并不会阻塞用户线程；\n- 内核准备好数据后会将数据从内核空间拷贝到用户空间并通知用户线程（发送信号）操作已完成。\n\n### 流程图\n#### 同步blocking I/O\n\n```plantuml\n@startuml Test Diagram\n\nparticipant \"application\" as app\nparticipant \"kernel\" as kernel\n\nactivate app\nactivate kernel\napp -> kernel: syscall: Read recvfrom\nkernel -> kernel: wait for data (no datagram ready)\nkernel -> kernel: copy datagram to user (datagram ready)\nkernel -> app: return\n@enduml\n```\n\n#### I/O多路复用\n\n### 异步编程\n\n\n## the Future Trait\nA Future is an asynchronous computation that can produce a value. A simplified version of the future trait might look something like this\n\n```rust\n\ntrait SimpleFuture {\n type Output;\n fn poll(&mut self, wake: fn()) -> Poll;\n}\n\nenum Poll {\n Ready(T),\n Pending,\n}\n```\n\nFor example, consider the case where we want to read from a socket that may or may not have data available already.\n```rust\npub struct SocketRead<'a> {\n socket: &'a Socket,\n}\n\nimpl SimpleFuture for SocketRead<'_> {\n type Output = Vec;\n\n fn poll(&mut self, wake: fn()) -> Poll {\n if self.socket.has_data_to_read() {\n // The socket has data -- read it into a buffer and return it.\n Poll::Ready(self.socket.read_buf())\n } else {\n // The socket does not yet have data.\n //\n // Arrange for `wake` to be called once data is available.\n // When data becomes available, `wake` will be called, and the\n // user of this `Future` will know to call `poll` again and\n // receive data.\n self.socket.set_readable_callback(wake);\n Poll::Pending\n }\n }\n}\n\n```\n\n the real Future trait and how it is different\n```rust\ntrait Future {\n type Output;\n fn poll(\n // Note the change from `&mut self` to `Pin<&mut Self>`:\n self: Pin<&mut Self>,\n // and the change from `wake: fn()` to `cx: &mut Context<'_>`:\n cx: &mut Context<'_>,\n ) -> Poll;\n}\n\n```\nThe first change you'll notice is that our self type is no longer &mut Self, but has changed to Pin<&mut Self>. We'll talk more about pinning in a later section, but for now know that it allows us to create futures that are immovable. Immovable objects can store pointers between their fields, e.g. struct MyFut { a: i32, ptr_to_a: *const i32 }. Pinning is necessary to enable async/await.\n\nSecondly, wake: fn() has changed to &mut Context<'_>. In SimpleFuture, we used a call to a function pointer (fn()) to tell the future executor that the future in question should be polled. However, since fn() is just a function pointer, it can't store any data about which Future called wake.\n\nIn a real-world scenario, a complex application like a web server may have thousands of different connections whose wakeups should all be managed separately. The Context type solves this by providing access to a value of type Waker, which can be used to wake up a specific task.\n\n\n## task wakeups with Waker\nWaker provides a wake() method that can be used to tell the executor that the associated task should be awoken. When wake() is called, the executor knows that the task associated with the Waker is ready to make progress, and its future should be polled again.\n\n## referencs\n[csdn blog](https://blog.csdn.net/XMJYever/article/details/111560976)\n","slug":"rust/rust-async","published":1,"updated":"2023-11-05T04:21:13.735Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8dt001cqwsj0grhcnus","content":"

I/O

I/O：在计算机中指Input/Output。由于程序和运行时数据是在内存中驻留，由cpu来执行，涉及到数据交换的地方，通常是磁盘、网卡等，就需要IO接口

\n

I/O 模型

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
@startmindmap
+ **I/O模型**
++ 同步I/O
'tag::details[]
+++_ 阻塞I/O (BIO)
+++_ 非阻塞I/O (NIO)
+++_ I/O多路复用
+++_ 信号驱动I/O
'end::details[]
++ 异步I/O
'tag::details[]
+++_ linux (AIO, io_uring)
+++_ windows (IOCP)
'end::details[]
@endmindmap
\n

同步阻塞

    \n
  • 当用户线程发起IO请求后，会进行系统调用（system call）来让内核（Kernel）进行IO操作（系统调用是用户空间和内核空间的一个通道）；
    • \n
  • 此时用户线程阻塞，等待内核将数据准备好；
    • \n
  • 内核将数据准备好后会将数据从内核空间拷贝到用户空间，并返回给用户线程结束阻塞。
    • \n
\n

同步非阻塞

    \n
  • 由用户线程发起IO请求，进行系统调用来让内核进行IO操作；
    • \n
  • 此时如果内核没有准备好数据则会直接返回error，并不会阻塞用户线程，用户线程可以重复发起IO请求；
    • \n
  • 当用户线程发起请求并且内核已经将数据准备好后，会将数据从内核空间拷贝到用户空间（这个过程是需要阻塞用户线程的），返回给用户。
    • \n
\n

同步多路复用

    \n
  • 用户线程调用select后进行系统调用（内核会监视所有select负责的socket）
    • \n
  • 当用户将数据准备好后就会返回，并通知用户线程进行读取操作，此时内核将数据拷贝到用户空间并返回。此时用户线程被阻塞；
    • \n
\n

异步I/O

    \n
  • 用户线程进行aio_read，进行系统调用切换到内核；
    • \n
  • 内核立即返回，并不会阻塞用户线程；
    • \n
  • 内核准备好数据后会将数据从内核空间拷贝到用户空间并通知用户线程（发送信号）操作已完成。
    • \n
\n

流程图

同步blocking I/O

1
2
3
4
5
6
7
8
9
10
11
12
@startuml Test Diagram

participant "application" as app
participant "kernel" as kernel

activate app
activate kernel
app -> kernel: syscall: Read recvfrom
kernel -> kernel: wait for data (no datagram ready)
kernel -> kernel: copy datagram to user (datagram ready)
kernel -> app: return
@enduml
\n\n

I/O多路复用

异步编程

the Future Trait

A Future is an asynchronous computation that can produce a value. A simplified version of the future trait might look something like this

\n
1
2
3
4
5
6
7
8
9
10

trait SimpleFuture {
    type Output;
    fn poll(&mut self, wake: fn()) -> Poll<Self::Output>;
}

enum Poll<T> {
    Ready(T),
    Pending,
}
\n\n

For example, consider the case where we want to read from a socket that may or may not have data available already.

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
pub struct SocketRead<'a> {
    socket: &'a Socket,
}

impl SimpleFuture for SocketRead<'_> {
    type Output = Vec<u8>;

    fn poll(&mut self, wake: fn()) -> Poll<Self::Output> {
        if self.socket.has_data_to_read() {
            // The socket has data -- read it into a buffer and return it.
            Poll::Ready(self.socket.read_buf())
        } else {
            // The socket does not yet have data.
            //
            // Arrange for `wake` to be called once data is available.
            // When data becomes available, `wake` will be called, and the
            // user of this `Future` will know to call `poll` again and
            // receive data.
            self.socket.set_readable_callback(wake);
            Poll::Pending
        }
    }
}

\n\n

the real Future trait and how it is different

\n
1
2
3
4
5
6
7
8
9
10
trait Future {
    type Output;
    fn poll(
        // Note the change from `&mut self` to `Pin<&mut Self>`:
        self: Pin<&mut Self>,
        // and the change from `wake: fn()` to `cx: &mut Context<'_>`:
        cx: &mut Context<'_>,
    ) -> Poll<Self::Output>;
}

\n

The first change you’ll notice is that our self type is no longer &mut Self, but has changed to Pin<&mut Self>. We’ll talk more about pinning in a later section, but for now know that it allows us to create futures that are immovable. Immovable objects can store pointers between their fields, e.g. struct MyFut { a: i32, ptr_to_a: *const i32 }. Pinning is necessary to enable async/await.

\n

Secondly, wake: fn() has changed to &mut Context<’_>. In SimpleFuture, we used a call to a function pointer (fn()) to tell the future executor that the future in question should be polled. However, since fn() is just a function pointer, it can’t store any data about which Future called wake.

\n

In a real-world scenario, a complex application like a web server may have thousands of different connections whose wakeups should all be managed separately. The Context type solves this by providing access to a value of type Waker, which can be used to wake up a specific task.

\n

task wakeups with Waker

Waker provides a wake() method that can be used to tell the executor that the associated task should be awoken. When wake() is called, the executor knows that the task associated with the Waker is ready to make progress, and its future should be polled again.

\n

referencs

csdn blog

\n","site":{"data":{}},"excerpt":"","more":"

I/O

I/O：在计算机中指Input/Output。由于程序和运行时数据是在内存中驻留，由cpu来执行，涉及到数据交换的地方，通常是磁盘、网卡等，就需要IO接口

\n

I/O 模型

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
@startmindmap
+ **I/O模型**
++ 同步I/O
'tag::details[]
+++_ 阻塞I/O (BIO)
+++_ 非阻塞I/O (NIO)
+++_ I/O多路复用
+++_ 信号驱动I/O
'end::details[]
++ 异步I/O
'tag::details[]
+++_ linux (AIO, io_uring)
+++_ windows (IOCP)
'end::details[]
@endmindmap
\n

同步阻塞

    \n
  • 当用户线程发起IO请求后，会进行系统调用（system call）来让内核（Kernel）进行IO操作（系统调用是用户空间和内核空间的一个通道）；
    • \n
  • 此时用户线程阻塞，等待内核将数据准备好；
    • \n
  • 内核将数据准备好后会将数据从内核空间拷贝到用户空间，并返回给用户线程结束阻塞。
    • \n
\n

同步非阻塞

    \n
  • 由用户线程发起IO请求，进行系统调用来让内核进行IO操作；
    • \n
  • 此时如果内核没有准备好数据则会直接返回error，并不会阻塞用户线程，用户线程可以重复发起IO请求；
    • \n
  • 当用户线程发起请求并且内核已经将数据准备好后，会将数据从内核空间拷贝到用户空间（这个过程是需要阻塞用户线程的），返回给用户。
    • \n
\n

同步多路复用

    \n
  • 用户线程调用select后进行系统调用（内核会监视所有select负责的socket）
    • \n
  • 当用户将数据准备好后就会返回，并通知用户线程进行读取操作，此时内核将数据拷贝到用户空间并返回。此时用户线程被阻塞；
    • \n
\n

异步I/O

    \n
  • 用户线程进行aio_read，进行系统调用切换到内核；
    • \n
  • 内核立即返回，并不会阻塞用户线程；
    • \n
  • 内核准备好数据后会将数据从内核空间拷贝到用户空间并通知用户线程（发送信号）操作已完成。
    • \n
\n

流程图

同步blocking I/O

1
2
3
4
5
6
7
8
9
10
11
12
@startuml Test Diagram

participant "application" as app
participant "kernel" as kernel

activate app
activate kernel
app -> kernel: syscall: Read recvfrom
kernel -> kernel: wait for data (no datagram ready)
kernel -> kernel: copy datagram to user (datagram ready)
kernel -> app: return
@enduml
\n\n

I/O多路复用

异步编程

the Future Trait

A Future is an asynchronous computation that can produce a value. A simplified version of the future trait might look something like this

\n
1
2
3
4
5
6
7
8
9
10

trait SimpleFuture {
    type Output;
    fn poll(&mut self, wake: fn()) -> Poll<Self::Output>;
}

enum Poll<T> {
    Ready(T),
    Pending,
}
\n\n

For example, consider the case where we want to read from a socket that may or may not have data available already.

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
pub struct SocketRead<'a> {
    socket: &'a Socket,
}

impl SimpleFuture for SocketRead<'_> {
    type Output = Vec<u8>;

    fn poll(&mut self, wake: fn()) -> Poll<Self::Output> {
        if self.socket.has_data_to_read() {
            // The socket has data -- read it into a buffer and return it.
            Poll::Ready(self.socket.read_buf())
        } else {
            // The socket does not yet have data.
            //
            // Arrange for `wake` to be called once data is available.
            // When data becomes available, `wake` will be called, and the
            // user of this `Future` will know to call `poll` again and
            // receive data.
            self.socket.set_readable_callback(wake);
            Poll::Pending
        }
    }
}

\n\n

the real Future trait and how it is different

\n
1
2
3
4
5
6
7
8
9
10
trait Future {
    type Output;
    fn poll(
        // Note the change from `&mut self` to `Pin<&mut Self>`:
        self: Pin<&mut Self>,
        // and the change from `wake: fn()` to `cx: &mut Context<'_>`:
        cx: &mut Context<'_>,
    ) -> Poll<Self::Output>;
}

\n

The first change you’ll notice is that our self type is no longer &mut Self, but has changed to Pin<&mut Self>. We’ll talk more about pinning in a later section, but for now know that it allows us to create futures that are immovable. Immovable objects can store pointers between their fields, e.g. struct MyFut { a: i32, ptr_to_a: *const i32 }. Pinning is necessary to enable async/await.

\n

Secondly, wake: fn() has changed to &mut Context<’_>. In SimpleFuture, we used a call to a function pointer (fn()) to tell the future executor that the future in question should be polled. However, since fn() is just a function pointer, it can’t store any data about which Future called wake.

\n

In a real-world scenario, a complex application like a web server may have thousands of different connections whose wakeups should all be managed separately. The Context type solves this by providing access to a value of type Waker, which can be used to wake up a specific task.

\n

task wakeups with Waker

Waker provides a wake() method that can be used to tell the executor that the associated task should be awoken. When wake() is called, the executor knows that the task associated with the Waker is ready to make progress, and its future should be polled again.

\n

referencs

csdn blog

\n"},{"title":"rust concurrency","date":"2023-06-01T14:04:38.000Z","_content":"\n## Send and Sync\n- A type is Send if it is safe to send it to another thread.\n- A type is Sync if it is safe to share between threads (T is Sync if and only if &T is Send).\n- raw pointers are neither Send nor Sync (because they have no safety guards).\n- UnsafeCell isn't Sync (and therefore Cell and RefCell aren't).\n- Rc isn't Send or Sync (because the refcount is shared and unsynchronized).\n\nTypes that aren't automatically derived can simply implement them if desired:\n```rust\nstruct MyBox(*mut u8);\n\nunsafe impl Send for MyBox {}\nunsafe impl Sync for MyBox {}\n```\none can also unimplement Send and Sync:\n```rust\n#![feature(negative_impls)]\n\n// I have some magic semantics for some synchronization primitive!\nstruct SpecialThreadToken(u8);\n\nimpl !Send for SpecialThreadToken {}\nimpl !Sync for SpecialThreadToken {}\n```\n\n## reference\n- [rustonomicon](https://doc.rust-lang.org/nomicon/send-and-sync.html)","source":"_posts/rust/rust-10-concurrency.md","raw":"---\ntitle: rust concurrency\ndate: 2023-06-01 22:04:38\ntags: [rust]\n---\n\n## Send and Sync\n- A type is Send if it is safe to send it to another thread.\n- A type is Sync if it is safe to share between threads (T is Sync if and only if &T is Send).\n- raw pointers are neither Send nor Sync (because they have no safety guards).\n- UnsafeCell isn't Sync (and therefore Cell and RefCell aren't).\n- Rc isn't Send or Sync (because the refcount is shared and unsynchronized).\n\nTypes that aren't automatically derived can simply implement them if desired:\n```rust\nstruct MyBox(*mut u8);\n\nunsafe impl Send for MyBox {}\nunsafe impl Sync for MyBox {}\n```\none can also unimplement Send and Sync:\n```rust\n#![feature(negative_impls)]\n\n// I have some magic semantics for some synchronization primitive!\nstruct SpecialThreadToken(u8);\n\nimpl !Send for SpecialThreadToken {}\nimpl !Sync for SpecialThreadToken {}\n```\n\n## reference\n- [rustonomicon](https://doc.rust-lang.org/nomicon/send-and-sync.html)","slug":"rust/rust-10-concurrency","published":1,"updated":"2023-11-05T04:21:13.735Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8dt001fqwsjbi7t4e7s","content":"

Send and Sync

    \n
  • A type is Send if it is safe to send it to another thread.
    • \n
  • A type is Sync if it is safe to share between threads (T is Sync if and only if &T is Send).
    • \n
  • raw pointers are neither Send nor Sync (because they have no safety guards).
    • \n
  • UnsafeCell isn’t Sync (and therefore Cell and RefCell aren’t).
    • \n
  • Rc isn’t Send or Sync (because the refcount is shared and unsynchronized).
    • \n
\n

Types that aren’t automatically derived can simply implement them if desired:

\n
1
2
3
4
struct MyBox(*mut u8);

unsafe impl Send for MyBox {}
unsafe impl Sync for MyBox {}
\n

one can also unimplement Send and Sync:

\n
1
2
3
4
5
6
7
#![feature(negative_impls)]

// I have some magic semantics for some synchronization primitive!
struct SpecialThreadToken(u8);

impl !Send for SpecialThreadToken {}
impl !Sync for SpecialThreadToken {}
\n\n

reference

\n","site":{"data":{}},"excerpt":"","more":"

Send and Sync

    \n
  • A type is Send if it is safe to send it to another thread.
    • \n
  • A type is Sync if it is safe to share between threads (T is Sync if and only if &T is Send).
    • \n
  • raw pointers are neither Send nor Sync (because they have no safety guards).
    • \n
  • UnsafeCell isn’t Sync (and therefore Cell and RefCell aren’t).
    • \n
  • Rc isn’t Send or Sync (because the refcount is shared and unsynchronized).
    • \n
\n

Types that aren’t automatically derived can simply implement them if desired:

\n
1
2
3
4
struct MyBox(*mut u8);

unsafe impl Send for MyBox {}
unsafe impl Sync for MyBox {}
\n

one can also unimplement Send and Sync:

\n
1
2
3
4
5
6
7
#![feature(negative_impls)]

// I have some magic semantics for some synchronization primitive!
struct SpecialThreadToken(u8);

impl !Send for SpecialThreadToken {}
impl !Sync for SpecialThreadToken {}
\n\n

reference

\n"},{"title":"rust cargo all in one","date":"2022-12-06T09:05:07.000Z","_content":"\n## useful cmd\n```\ncargo new ${crate_name} --lib ## create a lib crate\ncargo build --verbose ## print out each rustc invocation\n```\n\n## specifying dependencies\n- specifying dependencies from crates.io\n```toml\n[dependencies]\ntime = \"0.1.12\"\n```\n- specifying dependencies from other registries\n```toml\n[dependencies]\nsome-crate = { version = \"1.0\", registry = \"my-registry\" }\n```\n- specifying dependencies form git repositories\n```toml\n[dependencies]\nregex = { git = \"https://github.com/rust-lang/regex.git\" }\n```\n- path dependencies\n```toml\n[dependencies]\nhello_utils = { path = \"hello_utils\" }\n```\n- platform specific dependencies\n```toml\n[target.'cfg(unix)'.dependencies]\nopenssl = \"1.0.1\"\n\n[target.'cfg(target_arch = \"x86\")'.dependencies]\nnative-i686 = { path = \"native/i686\" }\n```\nLike with Rust, the syntax here supports the not, any, and all operators to combine various cfg name/value pairs.\nIf you want to know which cfg targets are available on your platform, run ```rustc --print=cfg``` from the command line.\nIf you want to know which cfg targets are available for another platform, such as 64-bit Windows, run ```rustc --print=cfg --target=x86_64-pc-windows-msvc```\n- custom target specifications\n```toml\n[target.bar.dependencies]\nwinhttp = \"0.4.0\"\n\n[target.my-special-i686-platform.dependencies]\nopenssl = \"1.0.1\"\nnative = { path = \"native/i686\" }\n```\n- development dependencies\n [dev-dependencies]\n Dev-dependencies are not used when compiling a package for building, but are used for compiling tests, examples, and benchmarks.\n These dependencies are not propagated to other packages which depend on this package.\n ```toml\n [target.'cfg(unix)'.dev-dependencies]\n mio = \"0.0.1\"\n ```\n\n- build dependencies\n ```toml\n [build-dependencies]\n cc = \"1.0.3\"\n ```\n The build script does not have access to the dependencies listed in the dependencies or dev-dependencies section. Build dependencies will likewise not be available to the package itself unless listed under the dependencies section as well.\n- choosing features\n ```toml\n [dependencies.awesome]\n version = \"1.3.5\"\n default-features = false # do not include the default features, and optionally\n # cherry-pick individual features\n features = [\"secure-password\", \"civet\"]\n ```\n- renaming dependencies in Cargo.toml\n When writing a [dependencies] section in Cargo.toml the key you write for a dependency typically matches up to the name of the crate you import from in the code. For some projects, though, you may wish to reference the crate with a different name in the code regardless of how it's published on crates.io. For example you may wish to: Avoid the need to use foo as bar in Rust source.\n (more to be found in original book)\n\n## references\n[cargo book](https://doc.rust-lang.org/cargo/)","source":"_posts/rust/rust-cargo-all-in-one.md","raw":"---\ntitle: rust cargo all in one\ndate: 2022-12-06 17:05:07\ntags: [rust]\n---\n\n## useful cmd\n```\ncargo new ${crate_name} --lib ## create a lib crate\ncargo build --verbose ## print out each rustc invocation\n```\n\n## specifying dependencies\n- specifying dependencies from crates.io\n```toml\n[dependencies]\ntime = \"0.1.12\"\n```\n- specifying dependencies from other registries\n```toml\n[dependencies]\nsome-crate = { version = \"1.0\", registry = \"my-registry\" }\n```\n- specifying dependencies form git repositories\n```toml\n[dependencies]\nregex = { git = \"https://github.com/rust-lang/regex.git\" }\n```\n- path dependencies\n```toml\n[dependencies]\nhello_utils = { path = \"hello_utils\" }\n```\n- platform specific dependencies\n```toml\n[target.'cfg(unix)'.dependencies]\nopenssl = \"1.0.1\"\n\n[target.'cfg(target_arch = \"x86\")'.dependencies]\nnative-i686 = { path = \"native/i686\" }\n```\nLike with Rust, the syntax here supports the not, any, and all operators to combine various cfg name/value pairs.\nIf you want to know which cfg targets are available on your platform, run ```rustc --print=cfg``` from the command line.\nIf you want to know which cfg targets are available for another platform, such as 64-bit Windows, run ```rustc --print=cfg --target=x86_64-pc-windows-msvc```\n- custom target specifications\n```toml\n[target.bar.dependencies]\nwinhttp = \"0.4.0\"\n\n[target.my-special-i686-platform.dependencies]\nopenssl = \"1.0.1\"\nnative = { path = \"native/i686\" }\n```\n- development dependencies\n [dev-dependencies]\n Dev-dependencies are not used when compiling a package for building, but are used for compiling tests, examples, and benchmarks.\n These dependencies are not propagated to other packages which depend on this package.\n ```toml\n [target.'cfg(unix)'.dev-dependencies]\n mio = \"0.0.1\"\n ```\n\n- build dependencies\n ```toml\n [build-dependencies]\n cc = \"1.0.3\"\n ```\n The build script does not have access to the dependencies listed in the dependencies or dev-dependencies section. Build dependencies will likewise not be available to the package itself unless listed under the dependencies section as well.\n- choosing features\n ```toml\n [dependencies.awesome]\n version = \"1.3.5\"\n default-features = false # do not include the default features, and optionally\n # cherry-pick individual features\n features = [\"secure-password\", \"civet\"]\n ```\n- renaming dependencies in Cargo.toml\n When writing a [dependencies] section in Cargo.toml the key you write for a dependency typically matches up to the name of the crate you import from in the code. For some projects, though, you may wish to reference the crate with a different name in the code regardless of how it's published on crates.io. For example you may wish to: Avoid the need to use foo as bar in Rust source.\n (more to be found in original book)\n\n## references\n[cargo book](https://doc.rust-lang.org/cargo/)","slug":"rust/rust-cargo-all-in-one","published":1,"updated":"2023-11-05T04:21:13.735Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8dt001hqwsj4zoehfes","content":"

useful cmd

1
2
cargo new ${crate_name} --lib ## create a lib crate
cargo build --verbose ## print out each rustc invocation
\n\n

specifying dependencies

    \n

  • specifying dependencies from crates.io

    \n
    1
    2
    [dependencies]
    time = "0.1.12"
    • \n

  • specifying dependencies from other registries

    \n
    1
    2
    [dependencies]
    some-crate = { version = "1.0", registry = "my-registry" }
    • \n

  • specifying dependencies form git repositories

    \n
    1
    2
    [dependencies]
    regex = { git = "https://github.com/rust-lang/regex.git" }
    • \n

  • path dependencies

    \n
    1
    2
    [dependencies]
    hello_utils = { path = "hello_utils" }
    • \n

  • platform specific dependencies

    \n
    1
    2
    3
    4
    5
    [target.'cfg(unix)'.dependencies]
    openssl = "1.0.1"

    [target.'cfg(target_arch = "x86")'.dependencies]
    native-i686 = { path = "native/i686" }
    \n

    Like with Rust, the syntax here supports the not, any, and all operators to combine various cfg name/value pairs.
    If you want to know which cfg targets are available on your platform, run rustc --print=cfg from the command line.
    If you want to know which cfg targets are available for another platform, such as 64-bit Windows, run rustc --print=cfg --target=x86_64-pc-windows-msvc

    \n
    • \n

  • custom target specifications

    \n
    1
    2
    3
    4
    5
    6
    [target.bar.dependencies]
    winhttp = "0.4.0"

    [target.my-special-i686-platform.dependencies]
    openssl = "1.0.1"
    native = { path = "native/i686" }
    • \n

  • development dependencies
    [dev-dependencies]
    Dev-dependencies are not used when compiling a package for building, but are used for compiling tests, examples, and benchmarks.
    These dependencies are not propagated to other packages which depend on this package.

    \n 
    1
    2
    [target.'cfg(unix)'.dev-dependencies]
    mio = "0.0.1"
    \n
    • \n

  • build dependencies

    \n 
    1
    2
    [build-dependencies]
    cc = "1.0.3"
    \n

    The build script does not have access to the dependencies listed in the dependencies or dev-dependencies section. Build dependencies will likewise not be available to the package itself unless listed under the dependencies section as well.

    \n
    • \n

  • choosing features

    \n 
    1
    2
    3
    4
    5
    [dependencies.awesome]
    version = "1.3.5"
    default-features = false # do not include the default features, and optionally
                            # cherry-pick individual features
    features = ["secure-password", "civet"]
    • \n

  • renaming dependencies in Cargo.toml
    When writing a [dependencies] section in Cargo.toml the key you write for a dependency typically matches up to the name of the crate you import from in the code. For some projects, though, you may wish to reference the crate with a different name in the code regardless of how it’s published on crates.io. For example you may wish to: Avoid the need to use foo as bar in Rust source.
    (more to be found in original book)

    \n
    • \n
\n

references

cargo book

\n","site":{"data":{}},"excerpt":"","more":"

useful cmd

1
2
cargo new ${crate_name} --lib ## create a lib crate
cargo build --verbose ## print out each rustc invocation
\n\n

specifying dependencies

    \n

  • specifying dependencies from crates.io

    \n
    1
    2
    [dependencies]
    time = "0.1.12"
    • \n

  • specifying dependencies from other registries

    \n
    1
    2
    [dependencies]
    some-crate = { version = "1.0", registry = "my-registry" }
    • \n

  • specifying dependencies form git repositories

    \n
    1
    2
    [dependencies]
    regex = { git = "https://github.com/rust-lang/regex.git" }
    • \n

  • path dependencies

    \n
    1
    2
    [dependencies]
    hello_utils = { path = "hello_utils" }
    • \n

  • platform specific dependencies

    \n
    1
    2
    3
    4
    5
    [target.'cfg(unix)'.dependencies]
    openssl = "1.0.1"

    [target.'cfg(target_arch = "x86")'.dependencies]
    native-i686 = { path = "native/i686" }
    \n

    Like with Rust, the syntax here supports the not, any, and all operators to combine various cfg name/value pairs.
    If you want to know which cfg targets are available on your platform, run rustc --print=cfg from the command line.
    If you want to know which cfg targets are available for another platform, such as 64-bit Windows, run rustc --print=cfg --target=x86_64-pc-windows-msvc

    \n
    • \n

  • custom target specifications

    \n
    1
    2
    3
    4
    5
    6
    [target.bar.dependencies]
    winhttp = "0.4.0"

    [target.my-special-i686-platform.dependencies]
    openssl = "1.0.1"
    native = { path = "native/i686" }
    • \n

  • development dependencies
    [dev-dependencies]
    Dev-dependencies are not used when compiling a package for building, but are used for compiling tests, examples, and benchmarks.
    These dependencies are not propagated to other packages which depend on this package.

    \n 
    1
    2
    [target.'cfg(unix)'.dev-dependencies]
    mio = "0.0.1"
    \n
    • \n

  • build dependencies

    \n 
    1
    2
    [build-dependencies]
    cc = "1.0.3"
    \n

    The build script does not have access to the dependencies listed in the dependencies or dev-dependencies section. Build dependencies will likewise not be available to the package itself unless listed under the dependencies section as well.

    \n
    • \n

  • choosing features

    \n 
    1
    2
    3
    4
    5
    [dependencies.awesome]
    version = "1.3.5"
    default-features = false # do not include the default features, and optionally
                            # cherry-pick individual features
    features = ["secure-password", "civet"]
    • \n

  • renaming dependencies in Cargo.toml
    When writing a [dependencies] section in Cargo.toml the key you write for a dependency typically matches up to the name of the crate you import from in the code. For some projects, though, you may wish to reference the crate with a different name in the code regardless of how it’s published on crates.io. For example you may wish to: Avoid the need to use foo as bar in Rust source.
    (more to be found in original book)

    \n
    • \n
\n

references

cargo book

\n"},{"title":"rust similar concepts comparison","date":"2022-11-23T07:52:34.000Z","_content":"\n## ref vs &\n`ref` annotates pattern bindings to make them borrow rather than move. It is **not** a part of the pattern as far as matching is concerned: it does not affect whether a value is matched, only how it is matched.\nBy default, match statements consume all they can, which can sometimes be a problem, when you don't really need the value to be moved and owned:\n```rust\nlet maybe_name = Some(String::from(\"Alice\"));\n// Using `ref`, the value is borrowed, not moved ...\nmatch maybe_name {\n Some(ref n) => println!(\"Hello, {n}\"),\n _ => println!(\"Hello, world\"),\n}\n// ... so it's available here!\nprintln!(\"Hello again, {}\", maybe_name.unwrap_or(\"world\".into()));\n```\n\n- `&` denotes that your pattern expects a reference to an object. Hence `&` is a part of said pattern: `&Foo` matches different objects than `Foo` does.\n- `ref` indicates that you want a reference to an unpacked value. It is not matched against: `Foo(ref foo)` matches the same objects as `Foo(foo)`.\n\n## Clone vs Copy\n### Copy 的含义\n`Copy` 的全名是 `std::marker::Copy`。`std::marker` 这个模块里面的所有的 trait 都是特殊的。目前稳定的有四个，它们是 `Copy`、`Send`、`Sized`、`Sync`。它们的特殊之处在于它们是跟编译器密切绑定的，impl 这些 trait 对编译器的行为有重要影响。这几个 trait 内部都没有方法，它们的唯一任务是，给类型打一个“标记”，表明它符合某种约定。\n\n如果一个类型 impl 了 Copy trait，意味着任何时候，我们可以通过简单的内存拷贝(C语言的按位拷贝memcpy)实现该类型的复制，而不会产生任何问题。\n\n一旦一个类型实现了 Copy trait，那么它在变量绑定、函数参数传递、函数返回值传递等场景下，它都是 copy 语义，而不再是默认的 move 语义。\n\n### Copy 的实现条件\n并不是所有的类型都可以实现Copy trait。Rust规定，对于自定义类型，只有所有的成员都实现了 Copy trait，这个类型才有资格实现 Copy trait。\n\n常见的数字类型、bool类型、共享借用指针&，都是具有 Copy 属性的类型。而 Box、Vec、可写借用指针&mut 等类型都是不具备 Copy 属性的类型。\n\n我们可以认为，Rust中只有 POD(C++语言中的Plain Old Data) 类型才有资格实现Copy trait。在Rust中，如果一个类型只包含POD数据类型的成员，没有指针类型的成员，并且没有自定义析构函数（实现Drop trait），那它就是POD类型。比如整数、浮点数、只包含POD类型的数组等。而Box、 String、 Vec等，不能按 bit 位拷贝的类型，都不属于POD类型。但是，反过来讲，并不是所有的POD类型都应该实现Copy trait。\n\n### Clone 的含义\nClone 的全名是 std::clone::Clone。它的完整声明是这样的：\n```rust\npub trait Clone : Sized {\n fn clone(&self) -> Self;\n fn clone_from(&mut self, source: &Self) {\n *self = source.clone()\n }\n}\n```\n它有两个关联方法，其中 clone_from 是有默认实现的，它依赖于 clone 方法的实现。clone 方法没有默认实现，需要我们手动实现。\n\nclone 方法一般用于“基于语义的复制”操作。所以，它做什么事情，跟具体类型的作用息息相关。比如对于 Box 类型，clone 就是执行的“深拷贝”，而对于 Rc 类型，clone 做的事情就是把引用计数值加1。\n\n虽然说，Rust中 clone 方法一般是用来执行复制操作的，但是你如果在自定义的 clone 函数中做点什么别的工作编译器也没法禁止，你可以根据情况在 clone 函数中编写任意的逻辑。但是有一条规则需要注意：对于实现了 Copy 的类型，它的 clone 方法应该跟 Copy 语义相容，等同于按位拷贝。\n\n### 自动 derive\n绝大多数情况下，实现 Copy Clone 这样的 trait 都是一个重复而无聊的工作。因此，Rust提供了一个 attribute，让我们可以利用编译器自动生成这部分代码。示例如下：\n\n```rust\n#[derive(Copy, Clone)]\nstruct MyStruct(i32);\n```\n这里的 derive 会让编译器帮我们自动生成 impl Copy 和 impl Clone 这样的代码。自动生成的 clone 方法，就是依次调用每个成员的 clone 方法。\n\n通过 derive 方式自动实现 Copy 和手工实现 Copy 有一丁点的微小区别。当类型具有泛型参数的时候，比如 struct MyStruct{}，通过 derive 自动生成的代码会自动添加一个 T: Copy 的约束。\n\n目前，只有一部分固定的特殊 trait 可以通过 derive 来自动实现。将来 Rust 会允许自定义的 derive 行为，让我们自己的 trait 也可以通过 derive 的方式自动实现。\n\n## Cell vs RefCell\n- Cell 是操作T(values), RefCell操作&T(references). Cell get的时候要求T impl Copy。比如String类型没有实现Copy trait, 那么Cell::new(String::from(\"Hello\")).get()会报错\n- Cell 在编译器检查，运行时不会panic；RefCell在运行时检查，使用不当会发生panic\n- 一般来说，Cell内部实现会发生内存的分配，性能较之RefCell有点大\n\n## AsRef vs Borrow\n[WIP]\n","source":"_posts/rust/rust-similar-concepts-comparison.md","raw":"---\ntitle: rust similar concepts comparison\ndate: 2022-11-23 15:52:34\ntags: [rust]\n---\n\n## ref vs &\n`ref` annotates pattern bindings to make them borrow rather than move. It is **not** a part of the pattern as far as matching is concerned: it does not affect whether a value is matched, only how it is matched.\nBy default, match statements consume all they can, which can sometimes be a problem, when you don't really need the value to be moved and owned:\n```rust\nlet maybe_name = Some(String::from(\"Alice\"));\n// Using `ref`, the value is borrowed, not moved ...\nmatch maybe_name {\n Some(ref n) => println!(\"Hello, {n}\"),\n _ => println!(\"Hello, world\"),\n}\n// ... so it's available here!\nprintln!(\"Hello again, {}\", maybe_name.unwrap_or(\"world\".into()));\n```\n\n- `&` denotes that your pattern expects a reference to an object. Hence `&` is a part of said pattern: `&Foo` matches different objects than `Foo` does.\n- `ref` indicates that you want a reference to an unpacked value. It is not matched against: `Foo(ref foo)` matches the same objects as `Foo(foo)`.\n\n## Clone vs Copy\n### Copy 的含义\n`Copy` 的全名是 `std::marker::Copy`。`std::marker` 这个模块里面的所有的 trait 都是特殊的。目前稳定的有四个，它们是 `Copy`、`Send`、`Sized`、`Sync`。它们的特殊之处在于它们是跟编译器密切绑定的，impl 这些 trait 对编译器的行为有重要影响。这几个 trait 内部都没有方法，它们的唯一任务是，给类型打一个“标记”，表明它符合某种约定。\n\n如果一个类型 impl 了 Copy trait，意味着任何时候，我们可以通过简单的内存拷贝(C语言的按位拷贝memcpy)实现该类型的复制，而不会产生任何问题。\n\n一旦一个类型实现了 Copy trait，那么它在变量绑定、函数参数传递、函数返回值传递等场景下，它都是 copy 语义，而不再是默认的 move 语义。\n\n### Copy 的实现条件\n并不是所有的类型都可以实现Copy trait。Rust规定，对于自定义类型，只有所有的成员都实现了 Copy trait，这个类型才有资格实现 Copy trait。\n\n常见的数字类型、bool类型、共享借用指针&，都是具有 Copy 属性的类型。而 Box、Vec、可写借用指针&mut 等类型都是不具备 Copy 属性的类型。\n\n我们可以认为，Rust中只有 POD(C++语言中的Plain Old Data) 类型才有资格实现Copy trait。在Rust中，如果一个类型只包含POD数据类型的成员，没有指针类型的成员，并且没有自定义析构函数（实现Drop trait），那它就是POD类型。比如整数、浮点数、只包含POD类型的数组等。而Box、 String、 Vec等，不能按 bit 位拷贝的类型，都不属于POD类型。但是，反过来讲，并不是所有的POD类型都应该实现Copy trait。\n\n### Clone 的含义\nClone 的全名是 std::clone::Clone。它的完整声明是这样的：\n```rust\npub trait Clone : Sized {\n fn clone(&self) -> Self;\n fn clone_from(&mut self, source: &Self) {\n *self = source.clone()\n }\n}\n```\n它有两个关联方法，其中 clone_from 是有默认实现的，它依赖于 clone 方法的实现。clone 方法没有默认实现，需要我们手动实现。\n\nclone 方法一般用于“基于语义的复制”操作。所以，它做什么事情，跟具体类型的作用息息相关。比如对于 Box 类型，clone 就是执行的“深拷贝”，而对于 Rc 类型，clone 做的事情就是把引用计数值加1。\n\n虽然说，Rust中 clone 方法一般是用来执行复制操作的，但是你如果在自定义的 clone 函数中做点什么别的工作编译器也没法禁止，你可以根据情况在 clone 函数中编写任意的逻辑。但是有一条规则需要注意：对于实现了 Copy 的类型，它的 clone 方法应该跟 Copy 语义相容，等同于按位拷贝。\n\n### 自动 derive\n绝大多数情况下，实现 Copy Clone 这样的 trait 都是一个重复而无聊的工作。因此，Rust提供了一个 attribute，让我们可以利用编译器自动生成这部分代码。示例如下：\n\n```rust\n#[derive(Copy, Clone)]\nstruct MyStruct(i32);\n```\n这里的 derive 会让编译器帮我们自动生成 impl Copy 和 impl Clone 这样的代码。自动生成的 clone 方法，就是依次调用每个成员的 clone 方法。\n\n通过 derive 方式自动实现 Copy 和手工实现 Copy 有一丁点的微小区别。当类型具有泛型参数的时候，比如 struct MyStruct{}，通过 derive 自动生成的代码会自动添加一个 T: Copy 的约束。\n\n目前，只有一部分固定的特殊 trait 可以通过 derive 来自动实现。将来 Rust 会允许自定义的 derive 行为，让我们自己的 trait 也可以通过 derive 的方式自动实现。\n\n## Cell vs RefCell\n- Cell 是操作T(values), RefCell操作&T(references). Cell get的时候要求T impl Copy。比如String类型没有实现Copy trait, 那么Cell::new(String::from(\"Hello\")).get()会报错\n- Cell 在编译器检查，运行时不会panic；RefCell在运行时检查，使用不当会发生panic\n- 一般来说，Cell内部实现会发生内存的分配，性能较之RefCell有点大\n\n## AsRef vs Borrow\n[WIP]\n","slug":"rust/rust-similar-concepts-comparison","published":1,"updated":"2023-11-05T04:21:13.735Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8dt001kqwsj843kh526","content":"

ref vs &

ref annotates pattern bindings to make them borrow rather than move. It is not a part of the pattern as far as matching is concerned: it does not affect whether a value is matched, only how it is matched.
By default, match statements consume all they can, which can sometimes be a problem, when you don’t really need the value to be moved and owned:

\n
1
2
3
4
5
6
7
8
let maybe_name = Some(String::from("Alice"));
// Using `ref`, the value is borrowed, not moved ...
match maybe_name {
    Some(ref n) => println!("Hello, {n}"),
    _ => println!("Hello, world"),
}
// ... so it's available here!
println!("Hello again, {}", maybe_name.unwrap_or("world".into()));
\n\n
    \n
  • & denotes that your pattern expects a reference to an object. Hence & is a part of said pattern: &Foo matches different objects than Foo does.
    • \n
  • ref indicates that you want a reference to an unpacked value. It is not matched against: Foo(ref foo) matches the same objects as Foo(foo).
    • \n
\n

Clone vs Copy

Copy 的含义

Copy 的全名是 std::marker::Copy。std::marker 这个模块里面的所有的 trait 都是特殊的。目前稳定的有四个，它们是 Copy、Send、Sized、Sync。它们的特殊之处在于它们是跟编译器密切绑定的，impl 这些 trait 对编译器的行为有重要影响。这几个 trait 内部都没有方法，它们的唯一任务是，给类型打一个“标记”，表明它符合某种约定。

\n

如果一个类型 impl 了 Copy trait，意味着任何时候，我们可以通过简单的内存拷贝(C语言的按位拷贝memcpy)实现该类型的复制，而不会产生任何问题。

\n

一旦一个类型实现了 Copy trait，那么它在变量绑定、函数参数传递、函数返回值传递等场景下，它都是 copy 语义，而不再是默认的 move 语义。

\n

Copy 的实现条件

并不是所有的类型都可以实现Copy trait。Rust规定，对于自定义类型，只有所有的成员都实现了 Copy trait，这个类型才有资格实现 Copy trait。

\n

常见的数字类型、bool类型、共享借用指针&，都是具有 Copy 属性的类型。而 Box、Vec、可写借用指针&mut 等类型都是不具备 Copy 属性的类型。

\n

我们可以认为，Rust中只有 POD(C++语言中的Plain Old Data) 类型才有资格实现Copy trait。在Rust中，如果一个类型只包含POD数据类型的成员，没有指针类型的成员，并且没有自定义析构函数（实现Drop trait），那它就是POD类型。比如整数、浮点数、只包含POD类型的数组等。而Box、 String、 Vec等，不能按 bit 位拷贝的类型，都不属于POD类型。但是，反过来讲，并不是所有的POD类型都应该实现Copy trait。

\n

Clone 的含义

Clone 的全名是 std::clone::Clone。它的完整声明是这样的：

\n
1
2
3
4
5
6
pub trait Clone : Sized {
    fn clone(&self) -> Self;
    fn clone_from(&mut self, source: &Self) {
        *self = source.clone()
    }
}
\n

它有两个关联方法，其中 clone_from 是有默认实现的，它依赖于 clone 方法的实现。clone 方法没有默认实现，需要我们手动实现。

\n

clone 方法一般用于“基于语义的复制”操作。所以，它做什么事情，跟具体类型的作用息息相关。比如对于 Box 类型，clone 就是执行的“深拷贝”，而对于 Rc 类型，clone 做的事情就是把引用计数值加1。

\n

虽然说，Rust中 clone 方法一般是用来执行复制操作的，但是你如果在自定义的 clone 函数中做点什么别的工作编译器也没法禁止，你可以根据情况在 clone 函数中编写任意的逻辑。但是有一条规则需要注意：对于实现了 Copy 的类型，它的 clone 方法应该跟 Copy 语义相容，等同于按位拷贝。

\n

自动 derive

绝大多数情况下，实现 Copy Clone 这样的 trait 都是一个重复而无聊的工作。因此，Rust提供了一个 attribute，让我们可以利用编译器自动生成这部分代码。示例如下：

\n
1
2
#[derive(Copy, Clone)]
struct MyStruct(i32);
\n

这里的 derive 会让编译器帮我们自动生成 impl Copy 和 impl Clone 这样的代码。自动生成的 clone 方法，就是依次调用每个成员的 clone 方法。

\n

通过 derive 方式自动实现 Copy 和手工实现 Copy 有一丁点的微小区别。当类型具有泛型参数的时候，比如 struct MyStruct{}，通过 derive 自动生成的代码会自动添加一个 T: Copy 的约束。

\n

目前，只有一部分固定的特殊 trait 可以通过 derive 来自动实现。将来 Rust 会允许自定义的 derive 行为，让我们自己的 trait 也可以通过 derive 的方式自动实现。

\n

Cell vs RefCell

    \n
  • Cell 是操作T(values), RefCell操作&T(references). Cell get的时候要求T impl Copy。比如String类型没有实现Copy trait, 那么Cell::new(String::from(“Hello”)).get()会报错
    • \n
  • Cell 在编译器检查，运行时不会panic；RefCell在运行时检查，使用不当会发生panic
    • \n
  • 一般来说，Cell内部实现会发生内存的分配，性能较之RefCell有点大
    • \n
\n

AsRef vs Borrow

[WIP]

\n","site":{"data":{}},"excerpt":"","more":"

ref vs &

ref annotates pattern bindings to make them borrow rather than move. It is not a part of the pattern as far as matching is concerned: it does not affect whether a value is matched, only how it is matched.
By default, match statements consume all they can, which can sometimes be a problem, when you don’t really need the value to be moved and owned:

\n
1
2
3
4
5
6
7
8
let maybe_name = Some(String::from("Alice"));
// Using `ref`, the value is borrowed, not moved ...
match maybe_name {
    Some(ref n) => println!("Hello, {n}"),
    _ => println!("Hello, world"),
}
// ... so it's available here!
println!("Hello again, {}", maybe_name.unwrap_or("world".into()));
\n\n
    \n
  • & denotes that your pattern expects a reference to an object. Hence & is a part of said pattern: &Foo matches different objects than Foo does.
    • \n
  • ref indicates that you want a reference to an unpacked value. It is not matched against: Foo(ref foo) matches the same objects as Foo(foo).
    • \n
\n

Clone vs Copy

Copy 的含义

Copy 的全名是 std::marker::Copy。std::marker 这个模块里面的所有的 trait 都是特殊的。目前稳定的有四个，它们是 Copy、Send、Sized、Sync。它们的特殊之处在于它们是跟编译器密切绑定的，impl 这些 trait 对编译器的行为有重要影响。这几个 trait 内部都没有方法，它们的唯一任务是，给类型打一个“标记”，表明它符合某种约定。

\n

如果一个类型 impl 了 Copy trait，意味着任何时候，我们可以通过简单的内存拷贝(C语言的按位拷贝memcpy)实现该类型的复制，而不会产生任何问题。

\n

一旦一个类型实现了 Copy trait，那么它在变量绑定、函数参数传递、函数返回值传递等场景下，它都是 copy 语义，而不再是默认的 move 语义。

\n

Copy 的实现条件

并不是所有的类型都可以实现Copy trait。Rust规定，对于自定义类型，只有所有的成员都实现了 Copy trait，这个类型才有资格实现 Copy trait。

\n

常见的数字类型、bool类型、共享借用指针&，都是具有 Copy 属性的类型。而 Box、Vec、可写借用指针&mut 等类型都是不具备 Copy 属性的类型。

\n

我们可以认为，Rust中只有 POD(C++语言中的Plain Old Data) 类型才有资格实现Copy trait。在Rust中，如果一个类型只包含POD数据类型的成员，没有指针类型的成员，并且没有自定义析构函数（实现Drop trait），那它就是POD类型。比如整数、浮点数、只包含POD类型的数组等。而Box、 String、 Vec等，不能按 bit 位拷贝的类型，都不属于POD类型。但是，反过来讲，并不是所有的POD类型都应该实现Copy trait。

\n

Clone 的含义

Clone 的全名是 std::clone::Clone。它的完整声明是这样的：

\n
1
2
3
4
5
6
pub trait Clone : Sized {
    fn clone(&self) -> Self;
    fn clone_from(&mut self, source: &Self) {
        *self = source.clone()
    }
}
\n

它有两个关联方法，其中 clone_from 是有默认实现的，它依赖于 clone 方法的实现。clone 方法没有默认实现，需要我们手动实现。

\n

clone 方法一般用于“基于语义的复制”操作。所以，它做什么事情，跟具体类型的作用息息相关。比如对于 Box 类型，clone 就是执行的“深拷贝”，而对于 Rc 类型，clone 做的事情就是把引用计数值加1。

\n

虽然说，Rust中 clone 方法一般是用来执行复制操作的，但是你如果在自定义的 clone 函数中做点什么别的工作编译器也没法禁止，你可以根据情况在 clone 函数中编写任意的逻辑。但是有一条规则需要注意：对于实现了 Copy 的类型，它的 clone 方法应该跟 Copy 语义相容，等同于按位拷贝。

\n

自动 derive

绝大多数情况下，实现 Copy Clone 这样的 trait 都是一个重复而无聊的工作。因此，Rust提供了一个 attribute，让我们可以利用编译器自动生成这部分代码。示例如下：

\n
1
2
#[derive(Copy, Clone)]
struct MyStruct(i32);
\n

这里的 derive 会让编译器帮我们自动生成 impl Copy 和 impl Clone 这样的代码。自动生成的 clone 方法，就是依次调用每个成员的 clone 方法。

\n

通过 derive 方式自动实现 Copy 和手工实现 Copy 有一丁点的微小区别。当类型具有泛型参数的时候，比如 struct MyStruct{}，通过 derive 自动生成的代码会自动添加一个 T: Copy 的约束。

\n

目前，只有一部分固定的特殊 trait 可以通过 derive 来自动实现。将来 Rust 会允许自定义的 derive 行为，让我们自己的 trait 也可以通过 derive 的方式自动实现。

\n

Cell vs RefCell

    \n
  • Cell 是操作T(values), RefCell操作&T(references). Cell get的时候要求T impl Copy。比如String类型没有实现Copy trait, 那么Cell::new(String::from(“Hello”)).get()会报错
    • \n
  • Cell 在编译器检查，运行时不会panic；RefCell在运行时检查，使用不当会发生panic
    • \n
  • 一般来说，Cell内部实现会发生内存的分配，性能较之RefCell有点大
    • \n
\n

AsRef vs Borrow

[WIP]

\n"},{"title":"cargo doc","date":"2022-11-13T07:41:59.000Z","_content":"\n## 文档注释\n### 用于生成文档\n - 使用 ///\n - 支持 Markdown\n - 放置在被说没条目之前\n### 例子\n```rust\n/// adds one to the number given\n/// \n/// # Examples\n/// ```\n/// let arg = 5;\n/// let answer = my_crate::add_one(arg);\n/// \n/// assert_eq!(6, answer);\n/// ```\npub fn add_one(x: i32) -> i32 {\n x + 1;\n}\n```\n### 命令\n```\ncargo doc ## 生成的html放在 target/doc 目录下\ncargo doc --open ## 构建当前crate的文档 (也包含crate依赖项的文档)\n```\n### 常用章节\n- `# Examples`\n- `Panics`: 可能panic的场景\n- `Errors`: 如果fn返回Result, 描述可能的错误种类, 以及导致错误的条件\n- `Safety`: 如果fn出入unsafe调用, 解释unsafe的原因, 以及调用者确保的使用前提\n\n### 文档注释作为测试\n- 运行cargo test, doc中用# Example标记的实例代码会用来测试运行\n\n### 为包含注释的项添加文档注释\n- 符号: //!\n- 这类注释通常用描述crate和模块:\n - crate root (按惯例 src/lib.rs)\n - 一个模块内，将crate火模块作为一个整体进行记录\n\n## 注释\n//! - 模块级稳定注释, 置于模块头部\n//!! - 模块级稳定注释, 但是和上面注释置于同一行\n\n//! - 模块级稳定注释, 会换行\n\n/*! - 模块级稳定注释 */\n/*!！ - 模块级稳定注释, 和上面同一行 */\n\n// 普通行注释\n/// 行级文档注释\n//// 普通行注释\n\n/* 普通块级注释 */\n/** 会级文档注释 */","source":"_posts/rust/rust-cargo-doc.md","raw":"---\ntitle: cargo doc\ndate: 2022-11-13 15:41:59\ntags: [rust,cargo]\n---\n\n## 文档注释\n### 用于生成文档\n - 使用 ///\n - 支持 Markdown\n - 放置在被说没条目之前\n### 例子\n```rust\n/// adds one to the number given\n/// \n/// # Examples\n/// ```\n/// let arg = 5;\n/// let answer = my_crate::add_one(arg);\n/// \n/// assert_eq!(6, answer);\n/// ```\npub fn add_one(x: i32) -> i32 {\n x + 1;\n}\n```\n### 命令\n```\ncargo doc ## 生成的html放在 target/doc 目录下\ncargo doc --open ## 构建当前crate的文档 (也包含crate依赖项的文档)\n```\n### 常用章节\n- `# Examples`\n- `Panics`: 可能panic的场景\n- `Errors`: 如果fn返回Result, 描述可能的错误种类, 以及导致错误的条件\n- `Safety`: 如果fn出入unsafe调用, 解释unsafe的原因, 以及调用者确保的使用前提\n\n### 文档注释作为测试\n- 运行cargo test, doc中用# Example标记的实例代码会用来测试运行\n\n### 为包含注释的项添加文档注释\n- 符号: //!\n- 这类注释通常用描述crate和模块:\n - crate root (按惯例 src/lib.rs)\n - 一个模块内，将crate火模块作为一个整体进行记录\n\n## 注释\n//! - 模块级稳定注释, 置于模块头部\n//!! - 模块级稳定注释, 但是和上面注释置于同一行\n\n//! - 模块级稳定注释, 会换行\n\n/*! - 模块级稳定注释 */\n/*!！ - 模块级稳定注释, 和上面同一行 */\n\n// 普通行注释\n/// 行级文档注释\n//// 普通行注释\n\n/* 普通块级注释 */\n/** 会级文档注释 */","slug":"rust/rust-cargo-doc","published":1,"updated":"2023-11-05T04:21:13.735Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8dt001mqwsjfk9x8s37","content":"

文档注释

用于生成文档

    \n
  • 使用 ///
    • \n
  • 支持 Markdown
    • \n
  • 放置在被说没条目之前
    • \n
\n

例子

1
2
3
4
5
6
7
8
9
10
11
12
/// adds one to the number given
/// 
/// # Examples
/// ```
/// let arg = 5;
/// let answer = my_crate::add_one(arg);
/// 
/// assert_eq!(6, answer);
/// ```
pub fn add_one(x: i32) -> i32 {
    x + 1;
}
\n

命令

1
2
cargo doc  ## 生成的html放在 target/doc 目录下
cargo doc --open ## 构建当前crate的文档 (也包含crate依赖项的文档)
\n

常用章节

    \n
  • # Examples
    • \n
  • Panics: 可能panic的场景
    • \n
  • Errors: 如果fn返回Result, 描述可能的错误种类, 以及导致错误的条件
    • \n
  • Safety: 如果fn出入unsafe调用, 解释unsafe的原因, 以及调用者确保的使用前提
    • \n
\n

文档注释作为测试

    \n
  • 运行cargo test, doc中用# Example标记的实例代码会用来测试运行
    • \n
\n

为包含注释的项添加文档注释

    \n
  • 符号: //!
    • \n
  • 这类注释通常用描述crate和模块:
      \n
    • crate root (按惯例 src/lib.rs)
      • \n
    • 一个模块内，将crate火模块作为一个整体进行记录
      • \n
    \n
    • \n
\n

注释

//! - 模块级稳定注释, 置于模块头部
//!! - 模块级稳定注释, 但是和上面注释置于同一行

\n

//! - 模块级稳定注释, 会换行

\n

/*! - 模块级稳定注释 /
/!！ - 模块级稳定注释, 和上面同一行 */

\n

// 普通行注释
/// 行级文档注释
//// 普通行注释

\n

/* 普通块级注释 /
/* 会级文档注释 */

\n","site":{"data":{}},"excerpt":"","more":"

文档注释

用于生成文档

    \n
  • 使用 ///
    • \n
  • 支持 Markdown
    • \n
  • 放置在被说没条目之前
    • \n
\n

例子

1
2
3
4
5
6
7
8
9
10
11
12
/// adds one to the number given
/// 
/// # Examples
/// ```
/// let arg = 5;
/// let answer = my_crate::add_one(arg);
/// 
/// assert_eq!(6, answer);
/// ```
pub fn add_one(x: i32) -> i32 {
    x + 1;
}
\n

命令

1
2
cargo doc  ## 生成的html放在 target/doc 目录下
cargo doc --open ## 构建当前crate的文档 (也包含crate依赖项的文档)
\n

常用章节

    \n
  • # Examples
    • \n
  • Panics: 可能panic的场景
    • \n
  • Errors: 如果fn返回Result, 描述可能的错误种类, 以及导致错误的条件
    • \n
  • Safety: 如果fn出入unsafe调用, 解释unsafe的原因, 以及调用者确保的使用前提
    • \n
\n

文档注释作为测试

    \n
  • 运行cargo test, doc中用# Example标记的实例代码会用来测试运行
    • \n
\n

为包含注释的项添加文档注释

    \n
  • 符号: //!
    • \n
  • 这类注释通常用描述crate和模块:
      \n
    • crate root (按惯例 src/lib.rs)
      • \n
    • 一个模块内，将crate火模块作为一个整体进行记录
      • \n
    \n
    • \n
\n

注释

//! - 模块级稳定注释, 置于模块头部
//!! - 模块级稳定注释, 但是和上面注释置于同一行

\n

//! - 模块级稳定注释, 会换行

\n

/*! - 模块级稳定注释 /
/!！ - 模块级稳定注释, 和上面同一行 */

\n

// 普通行注释
/// 行级文档注释
//// 普通行注释

\n

/* 普通块级注释 /
/* 会级文档注释 */

\n"},{"title":"rust sugar","date":"2022-11-17T07:47:13.000Z","_content":"\n## formatted print\n```rust\n// Positional arguments can be used\nprintln!(\"{0}, this is {1}. {1}, this is {0}\", \"Alice\", \"Bob\");\n// As can named arguments.\nprintln!(\"{subject} {verb} {object}\",\n object=\"the lazy dog\",\n subject=\"the quick brown fox\",\n verb=\"jumps over\");\n// Different formatting can be invoked by specifying the format character\n// after a `:`.\nprintln!(\"Base 10: {}\", 69420); // 69420\nprintln!(\"Base 2 (binary): {:b}\", 69420); // 10000111100101100\nprintln!(\"Base 8 (octal): {:o}\", 69420); // 207454\nprintln!(\"Base 16 (hexadecimal): {:x}\", 69420); // 10f2c\nprintln!(\"Base 16 (hexadecimal): {:X}\", 69420); // 10F2C\n// You can right-justify text with a specified width. This will\n// output \" 1\". (Four white spaces and a \"1\", for a total width of 5.)\nprintln!(\"{number:>5}\", number=1);\n// You can pad numbers with extra zeroes,\n// and left-adjust by flipping the sign. This will output \"10000\".\nprintln!(\"{number:0<5}\", number=1);\n// You can use named arguments in the format specifier by appending a `$`.\nprintln!(\"{number:0>width$}\", number=1, width=5);\n```\n- [reference](https://doc.rust-lang.org/rust-by-example/hello/print.html)\n\n\n## syntax\n```rust\n/// print to stderr\neprintln!(\"server error: {}\", e);\n\nstruct MyTupleStruct(M);\nlet myTupleStruct = MyTupleStruct::(String::from(\"hello\"));\n\nvec.iter().position()\nvec.iter().find()\nvec.iter().any()\n\n\nstatic mut A: u32 = 0;\n```\n\n## attributes\n```rust\n#![allow(warnings)]\n#[allow(dead_code)]\n#![allow(unused)]\n// Suppress all warnings from casts which overflow.\n#![allow(overflowing_literals)]\n#![allow(unreachable_code)]\n```\n\n## memory\nThe compiler will not rearrange the memory layout\n```rust\n#[repr(C)]\nstruct A {\n a:u8,\n b:u32,\n c:u16\n}\n```\n\n## ptr\n```rust\nNonNull::new_unchecked()\n\n#![feature(new_uninit)]\nlet mut five = Box::::new_uninit();\nlet five = unsafe {\n // Deferred initialization:\n five.as_mut_ptr().write(5);\n five.assume_init()\n};\nassert_eq!(*five, 5)\n\nlet zero = Box::::new_zeroed();\nlet zero = unsafe { zero.assume_init() };\nassert_eq!(*zero, 0)\n\nuse std::alloc::{alloc, Layout};\nunsafe {\n let ptr = alloc(Layout::new::()) as *mut i32;\n ptr.write(5);\n let x = Box::from_raw(ptr);\n}\n```","source":"_posts/rust/rust-sugar.md","raw":"---\ntitle: rust sugar\ndate: 2022-11-17 15:47:13\ntags: [rust]\n---\n\n## formatted print\n```rust\n// Positional arguments can be used\nprintln!(\"{0}, this is {1}. {1}, this is {0}\", \"Alice\", \"Bob\");\n// As can named arguments.\nprintln!(\"{subject} {verb} {object}\",\n object=\"the lazy dog\",\n subject=\"the quick brown fox\",\n verb=\"jumps over\");\n// Different formatting can be invoked by specifying the format character\n// after a `:`.\nprintln!(\"Base 10: {}\", 69420); // 69420\nprintln!(\"Base 2 (binary): {:b}\", 69420); // 10000111100101100\nprintln!(\"Base 8 (octal): {:o}\", 69420); // 207454\nprintln!(\"Base 16 (hexadecimal): {:x}\", 69420); // 10f2c\nprintln!(\"Base 16 (hexadecimal): {:X}\", 69420); // 10F2C\n// You can right-justify text with a specified width. This will\n// output \" 1\". (Four white spaces and a \"1\", for a total width of 5.)\nprintln!(\"{number:>5}\", number=1);\n// You can pad numbers with extra zeroes,\n// and left-adjust by flipping the sign. This will output \"10000\".\nprintln!(\"{number:0<5}\", number=1);\n// You can use named arguments in the format specifier by appending a `$`.\nprintln!(\"{number:0>width$}\", number=1, width=5);\n```\n- [reference](https://doc.rust-lang.org/rust-by-example/hello/print.html)\n\n\n## syntax\n```rust\n/// print to stderr\neprintln!(\"server error: {}\", e);\n\nstruct MyTupleStruct(M);\nlet myTupleStruct = MyTupleStruct::(String::from(\"hello\"));\n\nvec.iter().position()\nvec.iter().find()\nvec.iter().any()\n\n\nstatic mut A: u32 = 0;\n```\n\n## attributes\n```rust\n#![allow(warnings)]\n#[allow(dead_code)]\n#![allow(unused)]\n// Suppress all warnings from casts which overflow.\n#![allow(overflowing_literals)]\n#![allow(unreachable_code)]\n```\n\n## memory\nThe compiler will not rearrange the memory layout\n```rust\n#[repr(C)]\nstruct A {\n a:u8,\n b:u32,\n c:u16\n}\n```\n\n## ptr\n```rust\nNonNull::new_unchecked()\n\n#![feature(new_uninit)]\nlet mut five = Box::::new_uninit();\nlet five = unsafe {\n // Deferred initialization:\n five.as_mut_ptr().write(5);\n five.assume_init()\n};\nassert_eq!(*five, 5)\n\nlet zero = Box::::new_zeroed();\nlet zero = unsafe { zero.assume_init() };\nassert_eq!(*zero, 0)\n\nuse std::alloc::{alloc, Layout};\nunsafe {\n let ptr = alloc(Layout::new::()) as *mut i32;\n ptr.write(5);\n let x = Box::from_raw(ptr);\n}\n```","slug":"rust/rust-sugar","published":1,"updated":"2023-11-05T04:21:13.735Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8du001pqwsj1k4uhzw0","content":"

formatted print

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
// Positional arguments can be used
println!("{0}, this is {1}. {1}, this is {0}", "Alice", "Bob");
// As can named arguments.
println!("{subject} {verb} {object}",
            object="the lazy dog",
            subject="the quick brown fox",
            verb="jumps over");
// Different formatting can be invoked by specifying the format character
// after a `:`.
println!("Base 10:               {}",   69420); // 69420
println!("Base 2 (binary):       {:b}", 69420); // 10000111100101100
println!("Base 8 (octal):        {:o}", 69420); // 207454
println!("Base 16 (hexadecimal): {:x}", 69420); // 10f2c
println!("Base 16 (hexadecimal): {:X}", 69420); // 10F2C
// You can right-justify text with a specified width. This will
// output "    1". (Four white spaces and a "1", for a total width of 5.)
println!("{number:>5}", number=1);
// You can pad numbers with extra zeroes,
// and left-adjust by flipping the sign. This will output "10000".
println!("{number:0<5}", number=1);
// You can use named arguments in the format specifier by appending a `$`.
println!("{number:0>width$}", number=1, width=5);
\n\n

syntax

1
2
3
4
5
6
7
8
9
10
11
12
/// print to stderr
eprintln!("server error: {}", e);

struct MyTupleStruct<M>(M);
let myTupleStruct =  MyTupleStruct::<String>(String::from("hello"));

vec.iter().position()
vec.iter().find()
vec.iter().any()


static mut A: u32 = 0;
\n\n

attributes

1
2
3
4
5
6
#![allow(warnings)]
#[allow(dead_code)]
#![allow(unused)]
// Suppress all warnings from casts which overflow.
#![allow(overflowing_literals)]
#![allow(unreachable_code)]
\n\n

memory

The compiler will not rearrange the memory layout

\n
1
2
3
4
5
6
#[repr(C)]
struct A {
    a:u8,
    b:u32,
    c:u16
}
\n\n

ptr

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
NonNull::new_unchecked()

#![feature(new_uninit)]
let mut five = Box::<u32>::new_uninit();
let five = unsafe {
    // Deferred initialization:
    five.as_mut_ptr().write(5);
    five.assume_init()
};
assert_eq!(*five, 5)

let zero = Box::<u32>::new_zeroed();
let zero = unsafe { zero.assume_init() };
assert_eq!(*zero, 0)

use std::alloc::{alloc, Layout};
unsafe {
    let ptr = alloc(Layout::new::<i32>()) as *mut i32;
    ptr.write(5);
    let x = Box::from_raw(ptr);
}
","site":{"data":{}},"excerpt":"","more":"

formatted print

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
// Positional arguments can be used
println!("{0}, this is {1}. {1}, this is {0}", "Alice", "Bob");
// As can named arguments.
println!("{subject} {verb} {object}",
            object="the lazy dog",
            subject="the quick brown fox",
            verb="jumps over");
// Different formatting can be invoked by specifying the format character
// after a `:`.
println!("Base 10:               {}",   69420); // 69420
println!("Base 2 (binary):       {:b}", 69420); // 10000111100101100
println!("Base 8 (octal):        {:o}", 69420); // 207454
println!("Base 16 (hexadecimal): {:x}", 69420); // 10f2c
println!("Base 16 (hexadecimal): {:X}", 69420); // 10F2C
// You can right-justify text with a specified width. This will
// output "    1". (Four white spaces and a "1", for a total width of 5.)
println!("{number:>5}", number=1);
// You can pad numbers with extra zeroes,
// and left-adjust by flipping the sign. This will output "10000".
println!("{number:0<5}", number=1);
// You can use named arguments in the format specifier by appending a `$`.
println!("{number:0>width$}", number=1, width=5);
\n\n

syntax

1
2
3
4
5
6
7
8
9
10
11
12
/// print to stderr
eprintln!("server error: {}", e);

struct MyTupleStruct<M>(M);
let myTupleStruct =  MyTupleStruct::<String>(String::from("hello"));

vec.iter().position()
vec.iter().find()
vec.iter().any()


static mut A: u32 = 0;
\n\n

attributes

1
2
3
4
5
6
#![allow(warnings)]
#[allow(dead_code)]
#![allow(unused)]
// Suppress all warnings from casts which overflow.
#![allow(overflowing_literals)]
#![allow(unreachable_code)]
\n\n

memory

The compiler will not rearrange the memory layout

\n
1
2
3
4
5
6
#[repr(C)]
struct A {
    a:u8,
    b:u32,
    c:u16
}
\n\n

ptr

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
NonNull::new_unchecked()

#![feature(new_uninit)]
let mut five = Box::<u32>::new_uninit();
let five = unsafe {
    // Deferred initialization:
    five.as_mut_ptr().write(5);
    five.assume_init()
};
assert_eq!(*five, 5)

let zero = Box::<u32>::new_zeroed();
let zero = unsafe { zero.assume_init() };
assert_eq!(*zero, 0)

use std::alloc::{alloc, Layout};
unsafe {
    let ptr = alloc(Layout::new::<i32>()) as *mut i32;
    ptr.write(5);
    let x = Box::from_raw(ptr);
}
"},{"title":"rust tools","date":"2022-11-20T09:14:05.000Z","_content":"\n## tools\n- cargo-edit\nThis tool extends Cargo to allow you to add, remove, and upgrade dependencies by modifying your Cargo.toml file from the command line\n\n- cargo whatfeatures ${crate}\n eg: `cargo whatfeatures hyper`\n","source":"_posts/rust/rust-tools.md","raw":"---\ntitle: rust tools\ndate: 2022-11-20 17:14:05\ntags: [rust]\n---\n\n## tools\n- cargo-edit\nThis tool extends Cargo to allow you to add, remove, and upgrade dependencies by modifying your Cargo.toml file from the command line\n\n- cargo whatfeatures ${crate}\n eg: `cargo whatfeatures hyper`\n","slug":"rust/rust-tools","published":1,"updated":"2023-11-05T04:21:13.735Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8du001rqwsj7syrcry3","content":"

tools

    \n

  • cargo-edit
    This tool extends Cargo to allow you to add, remove, and upgrade dependencies by modifying your Cargo.toml file from the command line

    \n
    • \n

  • cargo whatfeatures ${crate}
    eg: cargo whatfeatures hyper

    \n
    • \n
\n","site":{"data":{}},"excerpt":"","more":"

tools

    \n

  • cargo-edit
    This tool extends Cargo to allow you to add, remove, and upgrade dependencies by modifying your Cargo.toml file from the command line

    \n
    • \n

  • cargo whatfeatures ${crate}
    eg: cargo whatfeatures hyper

    \n
    • \n
\n"},{"title":"markdown & latex","date":"2023-10-01T01:22:35.000Z","_content":"\n\n\n\n## markdown by example\n_italic_\ncolor\n\n\n\n## latex by example\n## symbol\n\\\\( \\mu \\\\)\n\\\\( \\omega \\\\)\n\\\\( \\sigma \\\\)\n\\\\( \\epsilon \\\\)\n\\\\( \\zeta \\\\)\n\\\\( \\eta \\\\)\n\\\\( \\theta \\\\)\n\\\\( \\kappa \\\\)\n\\\\( \\nu \\\\)\n\\\\( \\omicron \\\\)\n\\\\( \\rho \\\\)\n\\\\( \\delta \\\\)\n\\\\( \\tau \\\\)\n\\\\( \\upsilon \\\\)\n\\\\( \\phi \\\\)\n\\\\( \\chi \\\\)\n\\\\( \\psi \\\\)\n\\\\(\\mathcal O\\\\)\n\n## font & effects\n\\\\( \\mathbb{G_{1}}\\\\)\n\\\\( \\boldsymbol{F}\\\\)\n\\\\( \\hat{h} \\\\)\n\n## equation\n\\\\[ \\tag{1.1} X = F \\cdot x^{T}\\\\]\n## blob\n\\\\[\n\\begin{bmatrix}\n\\displaylines{\n \\omega_{N}^{0\\cdot0} & \\omega_{N}^{0\\cdot1} & \\dots & \\omega_{N}^{0\\cdot (N-1)}\\\\\\\\\n \\omega_{N}^{1\\cdot0} & \\omega_{N}^{1\\cdot1} & \\dots & \\omega_{N}^{2\\cdot (N-1)}\\\\\\\\\n \\vdots & \\vdots & \\ddots & \\vdots\\\\\\\\\n \\omega_{N}^{N-1\\cdot0} & \\omega_{N}^{N-1\\cdot1} & \\dots & \\omega_{N}^{N-1\\cdot (N-1)}\n}\n\\end{bmatrix}\n\\\\]","source":"_posts/tools/markdown_and_latex.md","raw":"---\ntitle: markdown & latex\ndate: 2023-10-01 09:22:35\ntags: [tool]\n---\n\n\n\n\n## markdown by example\n_italic_\ncolor\n\n\n\n## latex by example\n## symbol\n\\\\( \\mu \\\\)\n\\\\( \\omega \\\\)\n\\\\( \\sigma \\\\)\n\\\\( \\epsilon \\\\)\n\\\\( \\zeta \\\\)\n\\\\( \\eta \\\\)\n\\\\( \\theta \\\\)\n\\\\( \\kappa \\\\)\n\\\\( \\nu \\\\)\n\\\\( \\omicron \\\\)\n\\\\( \\rho \\\\)\n\\\\( \\delta \\\\)\n\\\\( \\tau \\\\)\n\\\\( \\upsilon \\\\)\n\\\\( \\phi \\\\)\n\\\\( \\chi \\\\)\n\\\\( \\psi \\\\)\n\\\\(\\mathcal O\\\\)\n\n## font & effects\n\\\\( \\mathbb{G_{1}}\\\\)\n\\\\( \\boldsymbol{F}\\\\)\n\\\\( \\hat{h} \\\\)\n\n## equation\n\\\\[ \\tag{1.1} X = F \\cdot x^{T}\\\\]\n## blob\n\\\\[\n\\begin{bmatrix}\n\\displaylines{\n \\omega_{N}^{0\\cdot0} & \\omega_{N}^{0\\cdot1} & \\dots & \\omega_{N}^{0\\cdot (N-1)}\\\\\\\\\n \\omega_{N}^{1\\cdot0} & \\omega_{N}^{1\\cdot1} & \\dots & \\omega_{N}^{2\\cdot (N-1)}\\\\\\\\\n \\vdots & \\vdots & \\ddots & \\vdots\\\\\\\\\n \\omega_{N}^{N-1\\cdot0} & \\omega_{N}^{N-1\\cdot1} & \\dots & \\omega_{N}^{N-1\\cdot (N-1)}\n}\n\\end{bmatrix}\n\\\\]","slug":"tools/markdown_and_latex","published":1,"updated":"2023-12-01T06:06:41.564Z","_id":"clokyy8dv001zqwsj9hdfghdd","comments":1,"layout":"post","photos":[],"link":"","content":"\n\n\n

markdown by example

italic
color

\n

latex by example

symbol

\\( \\mu \\)
\\( \\omega \\)
\\( \\sigma \\)
\\( \\epsilon \\)
\\( \\zeta \\)
\\( \\eta \\)
\\( \\theta \\)
\\( \\kappa \\)
\\( \\nu \\)
\\( \\omicron \\)
\\( \\rho \\)
\\( \\delta \\)
\\( \\tau \\)
\\( \\upsilon \\)
\\( \\phi \\)
\\( \\chi \\)
\\( \\psi \\)
\\(\\mathcal O\\)

\n

font & effects

\\( \\mathbb{G_{1}}\\)
\\( \\boldsymbol{F}\\)
\\( \\hat{h} \\)

\n

equation

\\[ \\tag{1.1} X = F \\cdot x^{T}\\]

\n

blob

\\[
\\begin{bmatrix}
\\displaylines{
\\omega_{N}^{0\\cdot0} & \\omega_{N}^{0\\cdot1} & \\dots & \\omega_{N}^{0\\cdot (N-1)}\\\\
\\omega_{N}^{1\\cdot0} & \\omega_{N}^{1\\cdot1} & \\dots & \\omega_{N}^{2\\cdot (N-1)}\\\\
\\vdots & \\vdots & \\ddots & \\vdots\\\\
\\omega_{N}^{N-1\\cdot0} & \\omega_{N}^{N-1\\cdot1} & \\dots & \\omega_{N}^{N-1\\cdot (N-1)}
}
\\end{bmatrix}
\\]

\n","site":{"data":{}},"excerpt":"","more":"\n\n\n

markdown by example

italic
color

\n

latex by example

symbol

\\( \\mu \\)
\\( \\omega \\)
\\( \\sigma \\)
\\( \\epsilon \\)
\\( \\zeta \\)
\\( \\eta \\)
\\( \\theta \\)
\\( \\kappa \\)
\\( \\nu \\)
\\( \\omicron \\)
\\( \\rho \\)
\\( \\delta \\)
\\( \\tau \\)
\\( \\upsilon \\)
\\( \\phi \\)
\\( \\chi \\)
\\( \\psi \\)
\\(\\mathcal O\\)

\n

font & effects

\\( \\mathbb{G_{1}}\\)
\\( \\boldsymbol{F}\\)
\\( \\hat{h} \\)

\n

equation

\\[ \\tag{1.1} X = F \\cdot x^{T}\\]

\n

blob

\\[
\\begin{bmatrix}
\\displaylines{
\\omega_{N}^{0\\cdot0} & \\omega_{N}^{0\\cdot1} & \\dots & \\omega_{N}^{0\\cdot (N-1)}\\\\
\\omega_{N}^{1\\cdot0} & \\omega_{N}^{1\\cdot1} & \\dots & \\omega_{N}^{2\\cdot (N-1)}\\\\
\\vdots & \\vdots & \\ddots & \\vdots\\\\
\\omega_{N}^{N-1\\cdot0} & \\omega_{N}^{N-1\\cdot1} & \\dots & \\omega_{N}^{N-1\\cdot (N-1)}
}
\\end{bmatrix}
\\]

\n"},{"title":"fourier series, DFT and FFT","date":"2023-10-12T03:13:17.000Z","_content":"\n\n\n\n## Fourier Series\n### innter product of two functions\n\\\\[ = \\int_{a}^{b} f(x) \\bar{g}(x)dx\\\\]\n\\\\(\\bar{g}\\\\) is the congugate of g on complex number system\n\n### Fourier series definition\nlet \\\\(f(x)\\\\) be a periodic function over a period of \\\\([-\\pi, +\\pi]\\\\)\n![f(x)](/images/math/fourier_series/f_x.png)\n\n\\\\( f(x) \\\\) can be transformed into below\n\\\\[ f(x) = \\frac{A_{0}}{2} + \\sum_{k=1}^{\\infty}A_{k}cos(kx) + B_{k}sin(kx)\\\\],\nwhere\n\\\\[ A_{k} = \\frac{1}{\\pi} \\int_{-\\pi}^{+\\pi} f(x)cos(kx)dx = \\frac{1}{||cos(kx)||^2} \\\\]\n\\\\[ B_{k} = \\frac{1}{\\pi} \\int_{-\\pi}^{+\\pi} f(x)sin(kx)dx = \\frac{1}{||sin(kx)||^2} \\\\]\nwhere \\\\(A_{k}, B_{k}\\\\) is just the **innver product** of \\\\(f(x)\\\\), and \\\\( cos(kx) \\\\), \\\\(sin(kx)\\\\) respectively. intuitively, it is the **projection** of \\\\(f(x)\\\\) over \\\\(sin(kx)\\\\), or \\\\(cos(kx)\\\\). \n\n### if period is L\n\\\\[ f(x) = \\frac{A_{0}}{2} + \\sum_{k=1}^{\\infty}A_{k}cos(\\frac{2\\pi kx}{L}) + B_{k}sin(\\frac{2\\pi kx}{L})\\\\],\nwhere \n\\\\[ A_{k} = \\frac{2}{L} \\int_{0}^{L} f(x)cos(\\frac{2\\pi kx}{L})dx\\\\]\n\\\\[ B_{k} = \\frac{2}{L} \\int_{0}^{L} f(x)sin(\\frac{2\\pi kx}{L})dx\\\\]\n\n## Complex Fourier Series\nlet's define \\\\[ \\psi_k = e^{ikx} = cos(kx) + i \\cdot sin(kx)\\\\]\nwe can prove that \\\\(\\psi_k, and \\psi_j\\\\) are **orthogonal** as their inner product is zero\n**proof**\n\\\\[ <\\psi_j, \\psi_k> = \\int_{-\\pi}^{+\\pi} e^{jkx}e^{-ikx}dx = \\int_{-\\pi}^{+\\pi} e^{j(j-k)x}dx = \\frac{1}{i(j-k)}[e^{i(j-k)x}]_{-\\pi}^{\\pi}\\\\]\nit is 0 if \\\\(j \\ne k\\\\) , \\\\(2\\pi \\\\) if \\\\(j=k\\\\)\n\n**note** conjugate of \\\\(\\psi_k\\\\) is \\\\(e^{-ikx}\\\\)\n\n\\\\(f(x) = \\frac{1}{2\\pi} \\sum_{k=-\\infty}^{\\infty} c_k \\psi_k\\\\), where \\\\(c_k= = \\int_{-\\pi}^{+\\pi} f(x)e^{-ikx}dx\\\\)\n\n## Discrete Fourier Transform (DFT)\nThe discrete Fourier transform transforms a sequence of N complex numbers \n\\\\[\\lbrace x_{n} \\rbrace := x_0, x_1,..., x_{N-1} \\\\] into another sequence of complex numbers, \n\\\\[\\lbrace X_{k} \\rbrace := X_0, X_1,..., X_{N-1} \\\\] which is defined by\n\\\\[X_{k} = \\sum_{n=0}^{N-1} x_{n} \\cdot e^{-\\frac{i2\\pi}{N}kn}\\\\]\n\nlet the n-th root of unity be \\\\(\\omega_N = e^{-\\frac{2\\pi i}{N}}\\\\), we have\n\\\\[ \\tag{1.1} X_{k} = \\sum_{n=0}^{N-1} x_{n} \\cdot \\omega_N^{kn}\\\\]\n\n### inverse DFT\nThe inverse transform is given by\n\\\\[ \\tag{1.2} x_{n} = \\frac{1}{N}\\sum_{k=0}^{N-1} X_{k} \\cdot \\omega_N^{-kn}\\\\]\n\n\n### the unitary DFT\nAnother way of looking at the DFT is to note that the DFT can be expressed as the DFT matrix, a Vandermonde matrix, introduced by Sylvester in 1867.\n\\\\[\n \\boldsymbol{F} =\n\\begin{bmatrix}\n\\displaylines{\n \\omega_{N}^{0\\cdot0} & \\omega_{N}^{0\\cdot1} & \\dots & \\omega_{N}^{0\\cdot (N-1)}\\\\\\\\\n \\omega_{N}^{1\\cdot0} & \\omega_{N}^{1\\cdot1} & \\dots & \\omega_{N}^{2\\cdot (N-1)}\\\\\\\\\n \\vdots & \\vdots & \\ddots & \\vdots\\\\\\\\\n \\omega_{N}^{N-1\\cdot0} & \\omega_{N}^{N-1\\cdot1} & \\dots & \\omega_{N}^{N-1\\cdot (N-1)}\n}\n\\end{bmatrix}\n\\\\]\nwhere \\\\( \\omega_{N} = e^{-\\frac{i2\\pi}{N}}\\\\) is a primitive Nth root of unity. (any complex number that yields 1 when raised to some positive integer power n)\n\n\\\\[ X = F \\cdot x^{T}\\\\]\n\\\\(x\\\\) is a column vector\n\n\n## Polynomial & DFT\n### Polynomial evaluation & DFT\nlet a polynomial be\n$$f(x)=\\sum_{n=0}^{N-1}x_nx^n$$\nthen,\n$$f(\\omega_N^{k})=\\sum_{n=0}^{N-1}x_n\\omega_N^{kn},\\quad k=0,1,2,...,N-1$$\ncompare this equation with E.q(1.1), we can know that DFT is the evaluations of a polynomial at N points of Nth root of unity, \\\\(\\omega_N^{0},\\omega_N^1,\\omega_N^2,...,\\omega_N^{N-1}\\\\)\n\n$$(\\omega_N^{0},X_0),(\\omega_N^1,X_1),(\\omega_N^2,X_2),...,(\\omega_N^{N-1},X_{N-1})$$\n\n### Polynomial interpolation & IDFT\nAccording to E.q(1.2), IDFT is the polynomial interpolation (getting original polynomial coefficients).\n\n## Fast Fourier Transform (FFT) [3]\nThe problem is to calculate the below \n\\\\[ \\tag{2.1} X_{k} = \\sum_{n=0}^{N-1} x_{n} \\cdot \\omega_N^{kn}, \\quad k= 0,1,...,N-1\\\\]\nA straightforward calculation using would require \\\\(N^2\\\\) operations where \"operation\" means, as it will throughout this note, a complex multiplication followed by a complex addition.\nThe FFT algorithm described here iterates on the array and yields the result in less than \\\\( 2N log_2N \\\\)\n\nTo derive the algorithm, suppose \\\\(N\\\\) is a composite, i.e., \\\\(N = r_1\\cdot r_2\\\\). Then let the indices in (2.1) be expressed\n\\\\[ k = k_1r_1 + k_0, \\quad k_0 = 0,1,...,r_1-1, \\quad k_1 = 0,1,..., r_2 -1 \\\\]\n\\\\[ n = n_1r_2 + n_0, \\quad n_0 = 0,1,...,r_2-1, \\quad n_1 = 0,1,..., r_1 -1 \\\\]\n\nSince \\\\( n = n_1r_2 + n_0\\\\), we can write\n\n\\\\[ \\tag{2.2} X(k_1, k_0) = \\sum_{n_0} \\sum_{n_1}x(n_1,n_0)\\omega_N^{kn} = \\sum_{n_0} \\sum_{n_1}x(n_1,n_0)\\omega_N^{kn_1r_2}\\omega_N^{kn_0} \\\\]\n**note** \\\\( x(n_1,n_0)\\\\) is same to \\\\(x(n)\\\\), just the indexing is transformed form 1D to 2D. You can think original 1D array of size N is resized to a matrix of size \\\\( (r_2, r_1) \\\\), \\\\(r_1\\\\) is number of cols, while \\\\(r_2\\\\) is number of rows\n\nSince \\\\(k = k_1r_1 + k_0\\\\), we have\n\\\\[ \\omega_N^{kn_1r_2} =\\omega_N^{(k_1r_1+k_0)n_1r_2} = \\omega_N^{(k_1r_1)n_1r_2} \\cdot \\omega_N^{k_0n_1r_2}\\\\]\naccording to the property of Nth root of unity, \\\\( \\omega_N^{N} = \\omega_N^{r_1r_2} =1 \\\\), \\\\( \\omega_N^{(k_1r_1)n_1r_2}\\\\) is also 1.\nthen we have \n\\\\[ \\omega_N^{kn_1r_2} = \\omega_N^{k_0n_1r_2}\\\\]\nsubstitute it into E.q 2.2, we get \n\\\\[ \\tag{2.3} X(k_1, k_0) = \\sum_{n_0} \\sum_{n_1}x(n_1,n_0)\\omega_N^{kn} = \\sum_{n_0} \\sum_{n_1}x(n_1,n_0)\\omega_N^{k_0n_1r_2}\\omega_N^{kn_0} \\\\]\nTherefore, the inner sum, over \\\\(n_1\\\\), depends only on \\\\(k_0\\\\) and \\\\(n_0\\\\), and can be defined as a new array,\n\\\\[ \\tag{2.4} x_1(k_0, n_0) = \\sum_{n_1} x(n_1, n_0) \\cdot \\omega_N^{k_0n_1r_2}\\\\]\nE.q(2.3) can be writtern as\n\\\\[ \\tag{2.5} X(k_1, k_0) = \\sum_{n_0} x_1(k_0,n_0)\\omega_N^{kn_0} = \\sum_{n_0} x_1(k_0,n_0)\\omega_N^{(k_1r_1 + k_0)n_0} \\\\]\n\nThere are N elements in the matrix \\\\(x_1\\\\), each requiring \\\\(r_1\\\\) operations, giving a total of \\\\(Nr_1\\\\) operations. Similarly, it takes \\\\(Nr_2\\\\) operations to calculate \\\\(X\\\\) from \\\\(x_1\\\\). Therefore, the two-step algorithm, given by Eq(2.4) and Eq(2.5) requires a total of\n\\\\[ \\tag{2.6} T = N(r_1+r_2) \\\\]\n\nit is easy to see how successive applications of the above procedure (recursively), starting with its appliation to Eq(2.4) give an m-step algorihtm requiring \n\\\\[ \\tag{2.7} T = N(r_1+r_2 + ... + r_m) \\\\]\nwhere \n\\\\[ N = \\prod_{j=1}^{j=m}r_j\\\\]\nif all \\\\(r_j\\\\) are equal to \\\\(r\\\\), i.e \\\\(N = r^m \\\\), which gives \\\\( m = log_rN \\\\)\nE.q(2.7) becomes\n\\\\[ \\tag{2.8} T = N(m \\cdot r) = rNm = rN(log_rN) \\\\]\n\n## radix-2 FFT\nThe algorithm with \\\\(r=2\\\\) is derived by expressing the indices in the form\n\n\\\\[\\tag{3.1} k=k_{m-1} \\cdot 2^{m-1} + ... + k_1 \\cdot 2 + k_0 \\\\]\n\\\\[\\tag{3.2} n=n_{m-1} \\cdot 2^{m-1} + ... + n_1 \\cdot 2 + n_0 \\\\]\nwhere \\\\( k_v \\in [0,1] \\\\), for \\\\( v = 0,...,m-1 \\\\), and \\\\( n_v \\in [0,1] \\\\), for \\\\( v = 0,...,m-1 \\\\)\n**\\\\(k_v\\\\) and \\\\(n_v\\\\) are the contents of the respective bit positions in the binary representation of \\\\(k\\\\) and \\\\(n\\\\)**\n\nAll arrays will now be written as functions of the bits of their indices. With this convention E.q(2.1) is written as \n\\\\[ \\tag{3.3} \n\\displaylines{\n X(k_{m-1}, ..., k_0) = \\sum_{n_0}\\sum_{n_1} ... \\sum_{n_{m-1}} x(n_{m-1}, ...,n_1, n_0) \\cdot \\omega_N^{kn} \\\\\\\\\n = \\sum_{n_0}\\sum_{n_1} ... \\sum_{n_{m-1}} x(n_{m-1}, ...,n_1, n_0) \\cdot \\omega_N^{k(n_{m-1} \\cdot 2^{m-1} + ... + n_1 \\cdot 2 + n_0)} \\\\\\\\\n = \\sum_{n_0}\\sum_{n_1} ... \\sum_{n_{m-1}} x(n_{m-1}, ...,n_1, n_0) \\cdot \\omega_N^{kn_{m-1} \\cdot 2^{m-1} + ... + kn_1 \\cdot 2 + kn_0}\n} \n\\\\]\nwhere the sums are over \\\\(n_v \\in [0,1]\\\\), for \\\\(v = 0,1,...,m-1\\\\)\n\nSince \\\\[ \\displaylines{\n \\omega_N^{kn_{m-1}\\cdot 2^{m-1}} = \\omega_N^{(k_{m-1} \\cdot 2^{m-1} + ... + k_1 \\cdot 2 + k_0)n_{m-1}\\cdot 2^{m-1}} \\\\\\\\\n = \\omega_N^{k_0 n_{m-1} \\cdot 2^{m-1}}\n}\n \\\\]\n(it is easy to show that all other terms are 1 as \\\\( \\omega_N^{2^m} = 1 \\\\), so only \\\\( k_0\\\\) is kept)\n\nthe innermost sum of E.q(3.3) over \\\\(n_{m-1} \\\\), depends only on \\\\( k_0, n_{m-2}, ..., n_0\\\\) and can be writtern\n\\\\[ \\displaylines{\nx_1(k_0, n_{m-2}, ..., n_1, n_0) = \\sum_{n_{m-1}} x(n_{m-1}, ..., n_0) \\cdot \\omega_N^{k_0 n_{m-1} \\cdot 2^{m-1}}\n}\n \\\\]\n\nproceeding to the next innermost sum, over \\\\( n_{m-2} \\\\), and so on, and using \n\\\\[ \\displaylines{\n \\omega_N^{kn_{m-l}\\cdot 2^{m-l}} = \\omega_N^{(k_{m-1} \\cdot 2^{m-1} + ... + k_1 \\cdot 2 + k_0)n_{m-l}\\cdot 2^{m-l}} \\\\\\\\\n = \\omega_N^{(k_{l-1}\\cdot 2^{l-1} + ... + k_0) n_{m-l} \\cdot 2^{m-1}}\n}\n \\\\]\none obtains successive arrays\n\\\\[\\displaylines{\nx_l(k_0, ..., k_{l-1}, n_{m-l-1}, ... , n_0) \\\\\\\\\n= \\sum_{n_{m-l}}x_{l-1}(k_0, ..., k_{l-2}, n_{m-l}, ..., n_0 ) \\cdot \\omega_N^{(k_{l-1}\\cdot 2^{l-1} + ...+ k_0) \\cdot n_{m-l} \\cdot 2^{m-l}}\n}\\\\]\nfor \\\\(l = 1,2,...,m \\\\)\n\nwriting out the sum this appears as \n\\\\[ \\tag{3.4} \\displaylines{\nx_l(k_0, ..., k_{l-1}, n_{m-l-1}, ... , n_0) \\\\\\\\\n= x_{l-1}(k_0, ..., k_{l-2}, 0, n_{m-l -1}, ..., n_0 ) \\cdot \\omega_N^{(k_{l-1}\\cdot 2^{l-1} + ...+ k_0) \\cdot 0 \\cdot 2^{m-l}} \\\\\\\\\n\\+ x_{l-1}(k_0, ..., k_{l-2}, 1, n_{m-l -1}, ..., n_0 ) \\cdot \\omega_N^{(k_{l-1}\\cdot 2^{l-1} + ...+ k_0) \\cdot 1 \\cdot 2^{m-l}} \\\\\\\\\n= x_{l-1}(k_0, ..., k_{l-2}, 0, n_{m-l -1}, ..., n_0 ) \\\\\\\\\n\\+ x_{l-1}(k_0, ..., k_{l-2}, 1, n_{m-l-1}, ..., n_0 ) \\cdot \\omega_N^{(k_{l-1}\\cdot 2^{l-1} + k_{l-2}\\cdot 2^{l-2 }+k_{l-3}\\cdot 2^{l-3} + ...+ k_0) \\cdot 2^{m-l}} \\\\\\\\\n= x_{l-1}(k_0, ..., k_{l-2}, 0, n_{m-l -1}, ..., n_0 ) \\\\\\\\\n\\+ \\omega_N^{k_{l-1}\\cdot 2^{l-1} \\cdot 2^{m-l} } \\cdot \\omega_N^{k_{l-2}\\cdot 2^{l-2 } \\cdot 2^{m-l}} \\cdot x_{l-1}(k_0, ..., k_{l-2}, 1, n_{m-l-1}, ..., n_0 ) \\cdot \\omega_N^{(k_{l-3}\\cdot 2^{l-3} + ...+ k_0) \\cdot 2^{m-l}} \\\\\\\\\n= x_{l-1}(k_0, ..., k_{l-2}, 0, n_{m-l -1}, ..., n_0 ) \\\\\\\\\n\\+ \\omega_N^{k_{l-1}\\cdot 2^{m-1} } \\cdot \\omega_N^{k_{l-2}\\cdot 2^{m-2}} \\cdot x_{l-1}(k_0, ..., k_{l-2}, 1, n_{m-l-1}, ..., n_0 ) \\cdot \\omega_N^{(k_{l-3}\\cdot 2^{l-3} + ...+ k_0) \\cdot 2^{m-l}} \\\\\\\\\n= x_{l-1}(k_0, ..., k_{l-2}, 0, n_{m-l -1}, ..., n_0 ) \\\\\\\\\n\\+ (-1)^{k_{l-1} } \\cdot i^{k_{l-2}} \\cdot x_{l-1}(k_0, ..., k_{l-2}, 1, n_{m-l-1}, ..., n_0 ) \\cdot \\omega_N^{(k_{l-3}\\cdot 2^{l-3} + ...+ k_0) \\cdot 2^{m-l}} \n}\\\\]\naccording to the indexing convension, this is stored in a location whose index is \n\\\\[ k_0 \\cdot 2^{m-1} + ... + k_{l-1} \\cdot 2 ^{m-l} + n_{m-l-1} \\cdot 2^{m-l-1} + ... + n_0 \\\\]\n\nIt can be seen in E.q(3.4) that only the two storage locations with indices having 0 and 1 in the \\\\(2^{m-l}\\\\) bit position are involved in the computation. Parallel computation is permitted since the operation described by E.q(3.4) can be carried out with all values of \\\\(k_0, ..., k_{l-2} \\\\), and \\\\( n_0, ..., n_{m-l-1}\\\\) simultaneously. In some applications it is convenient to use E.q(3.4) to express \\\\(x_l\\\\) in terms of \\\\(x_{l-2}\\\\), giving what is equivalent to an algorithm with \\\\(r = 4\\\\).\nthe last array calculated gives the desired Fourier sums,\n\n\\\\[\\tag{3.5}\nX(k_{m-1}, ..., k_0) = x_{m}(k_0, ..., k_{m-1})\n\\\\]\nin such an order that the index of an X must have its binary bits put in reverse order to yield its index in the array \\\\(x_m\\\\)\n## references\n- [1] [youtube video by Steve Brunton, Fourier Series](https://www.youtube.com/watch?v=MB6XGQWLV04)\n- [2] [DFT wikipedia](https://en.wikipedia.org/wiki/Discrete_Fourier_transform)\n- [3] [Cooley–Tukey FFT algorithm wikipedia](https://en.wikipedia.org/wiki/Cooley%E2%80%93Tukey_FFT_algorithm)\n- [4] [1965, Cooley & Turkey: an algorithm for the machine calculation of complex fourier series](https://www.ams.org/journals/mcom/1965-19-090/S0025-5718-1965-0178586-1/S0025-5718-1965-0178586-1.pdf)\n- [5] [Split-radix FFT algorithm wikipedia](https://en.wikipedia.org/wiki/Split-radix_FFT_algorithm)\n- [6] [csdn blog of FFT notes](https://blog.csdn.net/weixin_43870101/article/details/106095644?utm_medium=distribute.pc_relevant.none-task-blog-2~default~baidujs_utm_term~default-2-106095644-blog-132357995.235^v38^pc_relevant_sort_base1&spm=1001.2101.3001.4242.2&utm_relevant_index=5)\n\n\n","source":"_posts/math/fourier-series.md","raw":"---\ntitle: fourier series, DFT and FFT\ndate: 2023-10-12 11:13:17\ntags: [math]\n---\n\n\n\n\n## Fourier Series\n### innter product of two functions\n\\\\[ = \\int_{a}^{b} f(x) \\bar{g}(x)dx\\\\]\n\\\\(\\bar{g}\\\\) is the congugate of g on complex number system\n\n### Fourier series definition\nlet \\\\(f(x)\\\\) be a periodic function over a period of \\\\([-\\pi, +\\pi]\\\\)\n![f(x)](/images/math/fourier_series/f_x.png)\n\n\\\\( f(x) \\\\) can be transformed into below\n\\\\[ f(x) = \\frac{A_{0}}{2} + \\sum_{k=1}^{\\infty}A_{k}cos(kx) + B_{k}sin(kx)\\\\],\nwhere\n\\\\[ A_{k} = \\frac{1}{\\pi} \\int_{-\\pi}^{+\\pi} f(x)cos(kx)dx = \\frac{1}{||cos(kx)||^2} \\\\]\n\\\\[ B_{k} = \\frac{1}{\\pi} \\int_{-\\pi}^{+\\pi} f(x)sin(kx)dx = \\frac{1}{||sin(kx)||^2} \\\\]\nwhere \\\\(A_{k}, B_{k}\\\\) is just the **innver product** of \\\\(f(x)\\\\), and \\\\( cos(kx) \\\\), \\\\(sin(kx)\\\\) respectively. intuitively, it is the **projection** of \\\\(f(x)\\\\) over \\\\(sin(kx)\\\\), or \\\\(cos(kx)\\\\). \n\n### if period is L\n\\\\[ f(x) = \\frac{A_{0}}{2} + \\sum_{k=1}^{\\infty}A_{k}cos(\\frac{2\\pi kx}{L}) + B_{k}sin(\\frac{2\\pi kx}{L})\\\\],\nwhere \n\\\\[ A_{k} = \\frac{2}{L} \\int_{0}^{L} f(x)cos(\\frac{2\\pi kx}{L})dx\\\\]\n\\\\[ B_{k} = \\frac{2}{L} \\int_{0}^{L} f(x)sin(\\frac{2\\pi kx}{L})dx\\\\]\n\n## Complex Fourier Series\nlet's define \\\\[ \\psi_k = e^{ikx} = cos(kx) + i \\cdot sin(kx)\\\\]\nwe can prove that \\\\(\\psi_k, and \\psi_j\\\\) are **orthogonal** as their inner product is zero\n**proof**\n\\\\[ <\\psi_j, \\psi_k> = \\int_{-\\pi}^{+\\pi} e^{jkx}e^{-ikx}dx = \\int_{-\\pi}^{+\\pi} e^{j(j-k)x}dx = \\frac{1}{i(j-k)}[e^{i(j-k)x}]_{-\\pi}^{\\pi}\\\\]\nit is 0 if \\\\(j \\ne k\\\\) , \\\\(2\\pi \\\\) if \\\\(j=k\\\\)\n\n**note** conjugate of \\\\(\\psi_k\\\\) is \\\\(e^{-ikx}\\\\)\n\n\\\\(f(x) = \\frac{1}{2\\pi} \\sum_{k=-\\infty}^{\\infty} c_k \\psi_k\\\\), where \\\\(c_k= = \\int_{-\\pi}^{+\\pi} f(x)e^{-ikx}dx\\\\)\n\n## Discrete Fourier Transform (DFT)\nThe discrete Fourier transform transforms a sequence of N complex numbers \n\\\\[\\lbrace x_{n} \\rbrace := x_0, x_1,..., x_{N-1} \\\\] into another sequence of complex numbers, \n\\\\[\\lbrace X_{k} \\rbrace := X_0, X_1,..., X_{N-1} \\\\] which is defined by\n\\\\[X_{k} = \\sum_{n=0}^{N-1} x_{n} \\cdot e^{-\\frac{i2\\pi}{N}kn}\\\\]\n\nlet the n-th root of unity be \\\\(\\omega_N = e^{-\\frac{2\\pi i}{N}}\\\\), we have\n\\\\[ \\tag{1.1} X_{k} = \\sum_{n=0}^{N-1} x_{n} \\cdot \\omega_N^{kn}\\\\]\n\n### inverse DFT\nThe inverse transform is given by\n\\\\[ \\tag{1.2} x_{n} = \\frac{1}{N}\\sum_{k=0}^{N-1} X_{k} \\cdot \\omega_N^{-kn}\\\\]\n\n\n### the unitary DFT\nAnother way of looking at the DFT is to note that the DFT can be expressed as the DFT matrix, a Vandermonde matrix, introduced by Sylvester in 1867.\n\\\\[\n \\boldsymbol{F} =\n\\begin{bmatrix}\n\\displaylines{\n \\omega_{N}^{0\\cdot0} & \\omega_{N}^{0\\cdot1} & \\dots & \\omega_{N}^{0\\cdot (N-1)}\\\\\\\\\n \\omega_{N}^{1\\cdot0} & \\omega_{N}^{1\\cdot1} & \\dots & \\omega_{N}^{2\\cdot (N-1)}\\\\\\\\\n \\vdots & \\vdots & \\ddots & \\vdots\\\\\\\\\n \\omega_{N}^{N-1\\cdot0} & \\omega_{N}^{N-1\\cdot1} & \\dots & \\omega_{N}^{N-1\\cdot (N-1)}\n}\n\\end{bmatrix}\n\\\\]\nwhere \\\\( \\omega_{N} = e^{-\\frac{i2\\pi}{N}}\\\\) is a primitive Nth root of unity. (any complex number that yields 1 when raised to some positive integer power n)\n\n\\\\[ X = F \\cdot x^{T}\\\\]\n\\\\(x\\\\) is a column vector\n\n\n## Polynomial & DFT\n### Polynomial evaluation & DFT\nlet a polynomial be\n$$f(x)=\\sum_{n=0}^{N-1}x_nx^n$$\nthen,\n$$f(\\omega_N^{k})=\\sum_{n=0}^{N-1}x_n\\omega_N^{kn},\\quad k=0,1,2,...,N-1$$\ncompare this equation with E.q(1.1), we can know that DFT is the evaluations of a polynomial at N points of Nth root of unity, \\\\(\\omega_N^{0},\\omega_N^1,\\omega_N^2,...,\\omega_N^{N-1}\\\\)\n\n$$(\\omega_N^{0},X_0),(\\omega_N^1,X_1),(\\omega_N^2,X_2),...,(\\omega_N^{N-1},X_{N-1})$$\n\n### Polynomial interpolation & IDFT\nAccording to E.q(1.2), IDFT is the polynomial interpolation (getting original polynomial coefficients).\n\n## Fast Fourier Transform (FFT) [3]\nThe problem is to calculate the below \n\\\\[ \\tag{2.1} X_{k} = \\sum_{n=0}^{N-1} x_{n} \\cdot \\omega_N^{kn}, \\quad k= 0,1,...,N-1\\\\]\nA straightforward calculation using would require \\\\(N^2\\\\) operations where \"operation\" means, as it will throughout this note, a complex multiplication followed by a complex addition.\nThe FFT algorithm described here iterates on the array and yields the result in less than \\\\( 2N log_2N \\\\)\n\nTo derive the algorithm, suppose \\\\(N\\\\) is a composite, i.e., \\\\(N = r_1\\cdot r_2\\\\). Then let the indices in (2.1) be expressed\n\\\\[ k = k_1r_1 + k_0, \\quad k_0 = 0,1,...,r_1-1, \\quad k_1 = 0,1,..., r_2 -1 \\\\]\n\\\\[ n = n_1r_2 + n_0, \\quad n_0 = 0,1,...,r_2-1, \\quad n_1 = 0,1,..., r_1 -1 \\\\]\n\nSince \\\\( n = n_1r_2 + n_0\\\\), we can write\n\n\\\\[ \\tag{2.2} X(k_1, k_0) = \\sum_{n_0} \\sum_{n_1}x(n_1,n_0)\\omega_N^{kn} = \\sum_{n_0} \\sum_{n_1}x(n_1,n_0)\\omega_N^{kn_1r_2}\\omega_N^{kn_0} \\\\]\n**note** \\\\( x(n_1,n_0)\\\\) is same to \\\\(x(n)\\\\), just the indexing is transformed form 1D to 2D. You can think original 1D array of size N is resized to a matrix of size \\\\( (r_2, r_1) \\\\), \\\\(r_1\\\\) is number of cols, while \\\\(r_2\\\\) is number of rows\n\nSince \\\\(k = k_1r_1 + k_0\\\\), we have\n\\\\[ \\omega_N^{kn_1r_2} =\\omega_N^{(k_1r_1+k_0)n_1r_2} = \\omega_N^{(k_1r_1)n_1r_2} \\cdot \\omega_N^{k_0n_1r_2}\\\\]\naccording to the property of Nth root of unity, \\\\( \\omega_N^{N} = \\omega_N^{r_1r_2} =1 \\\\), \\\\( \\omega_N^{(k_1r_1)n_1r_2}\\\\) is also 1.\nthen we have \n\\\\[ \\omega_N^{kn_1r_2} = \\omega_N^{k_0n_1r_2}\\\\]\nsubstitute it into E.q 2.2, we get \n\\\\[ \\tag{2.3} X(k_1, k_0) = \\sum_{n_0} \\sum_{n_1}x(n_1,n_0)\\omega_N^{kn} = \\sum_{n_0} \\sum_{n_1}x(n_1,n_0)\\omega_N^{k_0n_1r_2}\\omega_N^{kn_0} \\\\]\nTherefore, the inner sum, over \\\\(n_1\\\\), depends only on \\\\(k_0\\\\) and \\\\(n_0\\\\), and can be defined as a new array,\n\\\\[ \\tag{2.4} x_1(k_0, n_0) = \\sum_{n_1} x(n_1, n_0) \\cdot \\omega_N^{k_0n_1r_2}\\\\]\nE.q(2.3) can be writtern as\n\\\\[ \\tag{2.5} X(k_1, k_0) = \\sum_{n_0} x_1(k_0,n_0)\\omega_N^{kn_0} = \\sum_{n_0} x_1(k_0,n_0)\\omega_N^{(k_1r_1 + k_0)n_0} \\\\]\n\nThere are N elements in the matrix \\\\(x_1\\\\), each requiring \\\\(r_1\\\\) operations, giving a total of \\\\(Nr_1\\\\) operations. Similarly, it takes \\\\(Nr_2\\\\) operations to calculate \\\\(X\\\\) from \\\\(x_1\\\\). Therefore, the two-step algorithm, given by Eq(2.4) and Eq(2.5) requires a total of\n\\\\[ \\tag{2.6} T = N(r_1+r_2) \\\\]\n\nit is easy to see how successive applications of the above procedure (recursively), starting with its appliation to Eq(2.4) give an m-step algorihtm requiring \n\\\\[ \\tag{2.7} T = N(r_1+r_2 + ... + r_m) \\\\]\nwhere \n\\\\[ N = \\prod_{j=1}^{j=m}r_j\\\\]\nif all \\\\(r_j\\\\) are equal to \\\\(r\\\\), i.e \\\\(N = r^m \\\\), which gives \\\\( m = log_rN \\\\)\nE.q(2.7) becomes\n\\\\[ \\tag{2.8} T = N(m \\cdot r) = rNm = rN(log_rN) \\\\]\n\n## radix-2 FFT\nThe algorithm with \\\\(r=2\\\\) is derived by expressing the indices in the form\n\n\\\\[\\tag{3.1} k=k_{m-1} \\cdot 2^{m-1} + ... + k_1 \\cdot 2 + k_0 \\\\]\n\\\\[\\tag{3.2} n=n_{m-1} \\cdot 2^{m-1} + ... + n_1 \\cdot 2 + n_0 \\\\]\nwhere \\\\( k_v \\in [0,1] \\\\), for \\\\( v = 0,...,m-1 \\\\), and \\\\( n_v \\in [0,1] \\\\), for \\\\( v = 0,...,m-1 \\\\)\n**\\\\(k_v\\\\) and \\\\(n_v\\\\) are the contents of the respective bit positions in the binary representation of \\\\(k\\\\) and \\\\(n\\\\)**\n\nAll arrays will now be written as functions of the bits of their indices. With this convention E.q(2.1) is written as \n\\\\[ \\tag{3.3} \n\\displaylines{\n X(k_{m-1}, ..., k_0) = \\sum_{n_0}\\sum_{n_1} ... \\sum_{n_{m-1}} x(n_{m-1}, ...,n_1, n_0) \\cdot \\omega_N^{kn} \\\\\\\\\n = \\sum_{n_0}\\sum_{n_1} ... \\sum_{n_{m-1}} x(n_{m-1}, ...,n_1, n_0) \\cdot \\omega_N^{k(n_{m-1} \\cdot 2^{m-1} + ... + n_1 \\cdot 2 + n_0)} \\\\\\\\\n = \\sum_{n_0}\\sum_{n_1} ... \\sum_{n_{m-1}} x(n_{m-1}, ...,n_1, n_0) \\cdot \\omega_N^{kn_{m-1} \\cdot 2^{m-1} + ... + kn_1 \\cdot 2 + kn_0}\n} \n\\\\]\nwhere the sums are over \\\\(n_v \\in [0,1]\\\\), for \\\\(v = 0,1,...,m-1\\\\)\n\nSince \\\\[ \\displaylines{\n \\omega_N^{kn_{m-1}\\cdot 2^{m-1}} = \\omega_N^{(k_{m-1} \\cdot 2^{m-1} + ... + k_1 \\cdot 2 + k_0)n_{m-1}\\cdot 2^{m-1}} \\\\\\\\\n = \\omega_N^{k_0 n_{m-1} \\cdot 2^{m-1}}\n}\n \\\\]\n(it is easy to show that all other terms are 1 as \\\\( \\omega_N^{2^m} = 1 \\\\), so only \\\\( k_0\\\\) is kept)\n\nthe innermost sum of E.q(3.3) over \\\\(n_{m-1} \\\\), depends only on \\\\( k_0, n_{m-2}, ..., n_0\\\\) and can be writtern\n\\\\[ \\displaylines{\nx_1(k_0, n_{m-2}, ..., n_1, n_0) = \\sum_{n_{m-1}} x(n_{m-1}, ..., n_0) \\cdot \\omega_N^{k_0 n_{m-1} \\cdot 2^{m-1}}\n}\n \\\\]\n\nproceeding to the next innermost sum, over \\\\( n_{m-2} \\\\), and so on, and using \n\\\\[ \\displaylines{\n \\omega_N^{kn_{m-l}\\cdot 2^{m-l}} = \\omega_N^{(k_{m-1} \\cdot 2^{m-1} + ... + k_1 \\cdot 2 + k_0)n_{m-l}\\cdot 2^{m-l}} \\\\\\\\\n = \\omega_N^{(k_{l-1}\\cdot 2^{l-1} + ... + k_0) n_{m-l} \\cdot 2^{m-1}}\n}\n \\\\]\none obtains successive arrays\n\\\\[\\displaylines{\nx_l(k_0, ..., k_{l-1}, n_{m-l-1}, ... , n_0) \\\\\\\\\n= \\sum_{n_{m-l}}x_{l-1}(k_0, ..., k_{l-2}, n_{m-l}, ..., n_0 ) \\cdot \\omega_N^{(k_{l-1}\\cdot 2^{l-1} + ...+ k_0) \\cdot n_{m-l} \\cdot 2^{m-l}}\n}\\\\]\nfor \\\\(l = 1,2,...,m \\\\)\n\nwriting out the sum this appears as \n\\\\[ \\tag{3.4} \\displaylines{\nx_l(k_0, ..., k_{l-1}, n_{m-l-1}, ... , n_0) \\\\\\\\\n= x_{l-1}(k_0, ..., k_{l-2}, 0, n_{m-l -1}, ..., n_0 ) \\cdot \\omega_N^{(k_{l-1}\\cdot 2^{l-1} + ...+ k_0) \\cdot 0 \\cdot 2^{m-l}} \\\\\\\\\n\\+ x_{l-1}(k_0, ..., k_{l-2}, 1, n_{m-l -1}, ..., n_0 ) \\cdot \\omega_N^{(k_{l-1}\\cdot 2^{l-1} + ...+ k_0) \\cdot 1 \\cdot 2^{m-l}} \\\\\\\\\n= x_{l-1}(k_0, ..., k_{l-2}, 0, n_{m-l -1}, ..., n_0 ) \\\\\\\\\n\\+ x_{l-1}(k_0, ..., k_{l-2}, 1, n_{m-l-1}, ..., n_0 ) \\cdot \\omega_N^{(k_{l-1}\\cdot 2^{l-1} + k_{l-2}\\cdot 2^{l-2 }+k_{l-3}\\cdot 2^{l-3} + ...+ k_0) \\cdot 2^{m-l}} \\\\\\\\\n= x_{l-1}(k_0, ..., k_{l-2}, 0, n_{m-l -1}, ..., n_0 ) \\\\\\\\\n\\+ \\omega_N^{k_{l-1}\\cdot 2^{l-1} \\cdot 2^{m-l} } \\cdot \\omega_N^{k_{l-2}\\cdot 2^{l-2 } \\cdot 2^{m-l}} \\cdot x_{l-1}(k_0, ..., k_{l-2}, 1, n_{m-l-1}, ..., n_0 ) \\cdot \\omega_N^{(k_{l-3}\\cdot 2^{l-3} + ...+ k_0) \\cdot 2^{m-l}} \\\\\\\\\n= x_{l-1}(k_0, ..., k_{l-2}, 0, n_{m-l -1}, ..., n_0 ) \\\\\\\\\n\\+ \\omega_N^{k_{l-1}\\cdot 2^{m-1} } \\cdot \\omega_N^{k_{l-2}\\cdot 2^{m-2}} \\cdot x_{l-1}(k_0, ..., k_{l-2}, 1, n_{m-l-1}, ..., n_0 ) \\cdot \\omega_N^{(k_{l-3}\\cdot 2^{l-3} + ...+ k_0) \\cdot 2^{m-l}} \\\\\\\\\n= x_{l-1}(k_0, ..., k_{l-2}, 0, n_{m-l -1}, ..., n_0 ) \\\\\\\\\n\\+ (-1)^{k_{l-1} } \\cdot i^{k_{l-2}} \\cdot x_{l-1}(k_0, ..., k_{l-2}, 1, n_{m-l-1}, ..., n_0 ) \\cdot \\omega_N^{(k_{l-3}\\cdot 2^{l-3} + ...+ k_0) \\cdot 2^{m-l}} \n}\\\\]\naccording to the indexing convension, this is stored in a location whose index is \n\\\\[ k_0 \\cdot 2^{m-1} + ... + k_{l-1} \\cdot 2 ^{m-l} + n_{m-l-1} \\cdot 2^{m-l-1} + ... + n_0 \\\\]\n\nIt can be seen in E.q(3.4) that only the two storage locations with indices having 0 and 1 in the \\\\(2^{m-l}\\\\) bit position are involved in the computation. Parallel computation is permitted since the operation described by E.q(3.4) can be carried out with all values of \\\\(k_0, ..., k_{l-2} \\\\), and \\\\( n_0, ..., n_{m-l-1}\\\\) simultaneously. In some applications it is convenient to use E.q(3.4) to express \\\\(x_l\\\\) in terms of \\\\(x_{l-2}\\\\), giving what is equivalent to an algorithm with \\\\(r = 4\\\\).\nthe last array calculated gives the desired Fourier sums,\n\n\\\\[\\tag{3.5}\nX(k_{m-1}, ..., k_0) = x_{m}(k_0, ..., k_{m-1})\n\\\\]\nin such an order that the index of an X must have its binary bits put in reverse order to yield its index in the array \\\\(x_m\\\\)\n## references\n- [1] [youtube video by Steve Brunton, Fourier Series](https://www.youtube.com/watch?v=MB6XGQWLV04)\n- [2] [DFT wikipedia](https://en.wikipedia.org/wiki/Discrete_Fourier_transform)\n- [3] [Cooley–Tukey FFT algorithm wikipedia](https://en.wikipedia.org/wiki/Cooley%E2%80%93Tukey_FFT_algorithm)\n- [4] [1965, Cooley & Turkey: an algorithm for the machine calculation of complex fourier series](https://www.ams.org/journals/mcom/1965-19-090/S0025-5718-1965-0178586-1/S0025-5718-1965-0178586-1.pdf)\n- [5] [Split-radix FFT algorithm wikipedia](https://en.wikipedia.org/wiki/Split-radix_FFT_algorithm)\n- [6] [csdn blog of FFT notes](https://blog.csdn.net/weixin_43870101/article/details/106095644?utm_medium=distribute.pc_relevant.none-task-blog-2~default~baidujs_utm_term~default-2-106095644-blog-132357995.235^v38^pc_relevant_sort_base1&spm=1001.2101.3001.4242.2&utm_relevant_index=5)\n\n\n","slug":"math/fourier-series","published":1,"updated":"2023-11-26T15:51:47.630Z","_id":"clokyy8dv0021qwsjhghcad7b","comments":1,"layout":"post","photos":[],"link":"","content":"\n\n\n

Fourier Series

innter product of two functions

\\[ <f(x), g(x)> = \\int_{a}^{b} f(x) \\bar{g}(x)dx\\]
\\(\\bar{g}\\) is the congugate of g on complex number system

\n

Fourier series definition

let \\(f(x)\\) be a periodic function over a period of \\([-\\pi, +\\pi]\\)
\"f(x)\"

\n

\\( f(x) \\) can be transformed into below
\\[ f(x) = \\frac{A_{0}}{2} + \\sum_{k=1}^{\\infty}A_{k}cos(kx) + B_{k}sin(kx)\\],
where
\\[ A_{k} = \\frac{1}{\\pi} \\int_{-\\pi}^{+\\pi} f(x)cos(kx)dx = \\frac{1}{||cos(kx)||^2}<f(x),cos(kx)> \\]
\\[ B_{k} = \\frac{1}{\\pi} \\int_{-\\pi}^{+\\pi} f(x)sin(kx)dx = \\frac{1}{||sin(kx)||^2}<f(x),sin(kx)> \\]
where \\(A_{k}, B_{k}\\) is just the innver product of \\(f(x)\\), and \\( cos(kx) \\), \\(sin(kx)\\) respectively. intuitively, it is the projection of \\(f(x)\\) over \\(sin(kx)\\), or \\(cos(kx)\\).

\n

if period is L

\\[ f(x) = \\frac{A_{0}}{2} + \\sum_{k=1}^{\\infty}A_{k}cos(\\frac{2\\pi kx}{L}) + B_{k}sin(\\frac{2\\pi kx}{L})\\],
where
\\[ A_{k} = \\frac{2}{L} \\int_{0}^{L} f(x)cos(\\frac{2\\pi kx}{L})dx\\]
\\[ B_{k} = \\frac{2}{L} \\int_{0}^{L} f(x)sin(\\frac{2\\pi kx}{L})dx\\]

\n

Complex Fourier Series

let’s define \\[ \\psi_k = e^{ikx} = cos(kx) + i \\cdot sin(kx)\\]
we can prove that \\(\\psi_k, and \\psi_j\\) are orthogonal as their inner product is zero
proof
\\[ <\\psi_j, \\psi_k> = \\int_{-\\pi}^{+\\pi} e^{jkx}e^{-ikx}dx = \\int_{-\\pi}^{+\\pi} e^{j(j-k)x}dx = \\frac{1}{i(j-k)}[e^{i(j-k)x}]_{-\\pi}^{\\pi}\\]
it is 0 if \\(j \\ne k\\) , \\(2\\pi \\) if \\(j=k\\)

\n

note conjugate of \\(\\psi_k\\) is \\(e^{-ikx}\\)

\n

\\(f(x) = \\frac{1}{2\\pi} \\sum_{k=-\\infty}^{\\infty} c_k \\psi_k\\), where \\(c_k=<f(x), \\psi_k> = \\int_{-\\pi}^{+\\pi} f(x)e^{-ikx}dx\\)

\n

Discrete Fourier Transform (DFT)

The discrete Fourier transform transforms a sequence of N complex numbers
\\[\\lbrace x_{n} \\rbrace := x_0, x_1,…, x_{N-1} \\] into another sequence of complex numbers,
\\[\\lbrace X_{k} \\rbrace := X_0, X_1,…, X_{N-1} \\] which is defined by
\\[X_{k} = \\sum_{n=0}^{N-1} x_{n} \\cdot e^{-\\frac{i2\\pi}{N}kn}\\]

\n

let the n-th root of unity be \\(\\omega_N = e^{-\\frac{2\\pi i}{N}}\\), we have
\\[ \\tag{1.1} X_{k} = \\sum_{n=0}^{N-1} x_{n} \\cdot \\omega_N^{kn}\\]

\n

inverse DFT

The inverse transform is given by
\\[ \\tag{1.2} x_{n} = \\frac{1}{N}\\sum_{k=0}^{N-1} X_{k} \\cdot \\omega_N^{-kn}\\]

\n

the unitary DFT

Another way of looking at the DFT is to note that the DFT can be expressed as the DFT matrix, a Vandermonde matrix, introduced by Sylvester in 1867.
\\[
\\boldsymbol{F} =
\\begin{bmatrix}
\\displaylines{
\\omega_{N}^{0\\cdot0} & \\omega_{N}^{0\\cdot1} & \\dots & \\omega_{N}^{0\\cdot (N-1)}\\\\
\\omega_{N}^{1\\cdot0} & \\omega_{N}^{1\\cdot1} & \\dots & \\omega_{N}^{2\\cdot (N-1)}\\\\
\\vdots & \\vdots & \\ddots & \\vdots\\\\
\\omega_{N}^{N-1\\cdot0} & \\omega_{N}^{N-1\\cdot1} & \\dots & \\omega_{N}^{N-1\\cdot (N-1)}
}
\\end{bmatrix}
\\]
where \\( \\omega_{N} = e^{-\\frac{i2\\pi}{N}}\\) is a primitive Nth root of unity. (any complex number that yields 1 when raised to some positive integer power n)

\n

\\[ X = F \\cdot x^{T}\\]
\\(x\\) is a column vector

\n

Polynomial & DFT

Polynomial evaluation & DFT

let a polynomial be
$$f(x)=\\sum_{n=0}^{N-1}x_nx^n$$
then,
$$f(\\omega_N^{k})=\\sum_{n=0}^{N-1}x_n\\omega_N^{kn},\\quad k=0,1,2,…,N-1$$
compare this equation with E.q(1.1), we can know that DFT is the evaluations of a polynomial at N points of Nth root of unity, \\(\\omega_N^{0},\\omega_N^1,\\omega_N^2,…,\\omega_N^{N-1}\\)

\n

$$(\\omega_N^{0},X_0),(\\omega_N^1,X_1),(\\omega_N^2,X_2),…,(\\omega_N^{N-1},X_{N-1})$$

\n

Polynomial interpolation & IDFT

According to E.q(1.2), IDFT is the polynomial interpolation (getting original polynomial coefficients).

\n

Fast Fourier Transform (FFT) [3]

The problem is to calculate the below
\\[ \\tag{2.1} X_{k} = \\sum_{n=0}^{N-1} x_{n} \\cdot \\omega_N^{kn}, \\quad k= 0,1,…,N-1\\]
A straightforward calculation using would require \\(N^2\\) operations where “operation” means, as it will throughout this note, a complex multiplication followed by a complex addition.
The FFT algorithm described here iterates on the array and yields the result in less than \\( 2N log_2N \\)

\n

To derive the algorithm, suppose \\(N\\) is a composite, i.e., \\(N = r_1\\cdot r_2\\). Then let the indices in (2.1) be expressed
\\[ k = k_1r_1 + k_0, \\quad k_0 = 0,1,…,r_1-1, \\quad k_1 = 0,1,…, r_2 -1 \\]
\\[ n = n_1r_2 + n_0, \\quad n_0 = 0,1,…,r_2-1, \\quad n_1 = 0,1,…, r_1 -1 \\]

\n

Since \\( n = n_1r_2 + n_0\\), we can write

\n

\\[ \\tag{2.2} X(k_1, k_0) = \\sum_{n_0} \\sum_{n_1}x(n_1,n_0)\\omega_N^{kn} = \\sum_{n_0} \\sum_{n_1}x(n_1,n_0)\\omega_N^{kn_1r_2}\\omega_N^{kn_0} \\]
note \\( x(n_1,n_0)\\) is same to \\(x(n)\\), just the indexing is transformed form 1D to 2D. You can think original 1D array of size N is resized to a matrix of size \\( (r_2, r_1) \\), \\(r_1\\) is number of cols, while \\(r_2\\) is number of rows

\n

Since \\(k = k_1r_1 + k_0\\), we have
\\[ \\omega_N^{kn_1r_2} =\\omega_N^{(k_1r_1+k_0)n_1r_2} = \\omega_N^{(k_1r_1)n_1r_2} \\cdot \\omega_N^{k_0n_1r_2}\\]
according to the property of Nth root of unity, \\( \\omega_N^{N} = \\omega_N^{r_1r_2} =1 \\), \\( \\omega_N^{(k_1r_1)n_1r_2}\\) is also 1.
then we have
\\[ \\omega_N^{kn_1r_2} = \\omega_N^{k_0n_1r_2}\\]
substitute it into E.q 2.2, we get
\\[ \\tag{2.3} X(k_1, k_0) = \\sum_{n_0} \\sum_{n_1}x(n_1,n_0)\\omega_N^{kn} = \\sum_{n_0} \\sum_{n_1}x(n_1,n_0)\\omega_N^{k_0n_1r_2}\\omega_N^{kn_0} \\]
Therefore, the inner sum, over \\(n_1\\), depends only on \\(k_0\\) and \\(n_0\\), and can be defined as a new array,
\\[ \\tag{2.4} x_1(k_0, n_0) = \\sum_{n_1} x(n_1, n_0) \\cdot \\omega_N^{k_0n_1r_2}\\]
E.q(2.3) can be writtern as
\\[ \\tag{2.5} X(k_1, k_0) = \\sum_{n_0} x_1(k_0,n_0)\\omega_N^{kn_0} = \\sum_{n_0} x_1(k_0,n_0)\\omega_N^{(k_1r_1 + k_0)n_0} \\]

\n

There are N elements in the matrix \\(x_1\\), each requiring \\(r_1\\) operations, giving a total of \\(Nr_1\\) operations. Similarly, it takes \\(Nr_2\\) operations to calculate \\(X\\) from \\(x_1\\). Therefore, the two-step algorithm, given by Eq(2.4) and Eq(2.5) requires a total of
\\[ \\tag{2.6} T = N(r_1+r_2) \\]

\n

it is easy to see how successive applications of the above procedure (recursively), starting with its appliation to Eq(2.4) give an m-step algorihtm requiring
\\[ \\tag{2.7} T = N(r_1+r_2 + … + r_m) \\]
where
\\[ N = \\prod_{j=1}^{j=m}r_j\\]
if all \\(r_j\\) are equal to \\(r\\), i.e \\(N = r^m \\), which gives \\( m = log_rN \\)
E.q(2.7) becomes
\\[ \\tag{2.8} T = N(m \\cdot r) = rNm = rN(log_rN) \\]

\n

radix-2 FFT

The algorithm with \\(r=2\\) is derived by expressing the indices in the form

\n

\\[\\tag{3.1} k=k_{m-1} \\cdot 2^{m-1} + … + k_1 \\cdot 2 + k_0 \\]
\\[\\tag{3.2} n=n_{m-1} \\cdot 2^{m-1} + … + n_1 \\cdot 2 + n_0 \\]
where \\( k_v \\in [0,1] \\), for \\( v = 0,…,m-1 \\), and \\( n_v \\in [0,1] \\), for \\( v = 0,…,m-1 \\)
\\(k_v\\) and \\(n_v\\) are the contents of the respective bit positions in the binary representation of \\(k\\) and \\(n\\)

\n

All arrays will now be written as functions of the bits of their indices. With this convention E.q(2.1) is written as
\\[ \\tag{3.3}
\\displaylines{
X(k_{m-1}, …, k_0) = \\sum_{n_0}\\sum_{n_1} … \\sum_{n_{m-1}} x(n_{m-1}, …,n_1, n_0) \\cdot \\omega_N^{kn} \\\\
= \\sum_{n_0}\\sum_{n_1} … \\sum_{n_{m-1}} x(n_{m-1}, …,n_1, n_0) \\cdot \\omega_N^{k(n_{m-1} \\cdot 2^{m-1} + … + n_1 \\cdot 2 + n_0)} \\\\
= \\sum_{n_0}\\sum_{n_1} … \\sum_{n_{m-1}} x(n_{m-1}, …,n_1, n_0) \\cdot \\omega_N^{kn_{m-1} \\cdot 2^{m-1} + … + kn_1 \\cdot 2 + kn_0}
}
\\]
where the sums are over \\(n_v \\in [0,1]\\), for \\(v = 0,1,…,m-1\\)

\n

Since \\[ \\displaylines{
\\omega_N^{kn_{m-1}\\cdot 2^{m-1}} = \\omega_N^{(k_{m-1} \\cdot 2^{m-1} + … + k_1 \\cdot 2 + k_0)n_{m-1}\\cdot 2^{m-1}} \\\\
= \\omega_N^{k_0 n_{m-1} \\cdot 2^{m-1}}
}
\\]
(it is easy to show that all other terms are 1 as \\( \\omega_N^{2^m} = 1 \\), so only \\( k_0\\) is kept)

\n

the innermost sum of E.q(3.3) over \\(n_{m-1} \\), depends only on \\( k_0, n_{m-2}, …, n_0\\) and can be writtern
\\[ \\displaylines{
x_1(k_0, n_{m-2}, …, n_1, n_0) = \\sum_{n_{m-1}} x(n_{m-1}, …, n_0) \\cdot \\omega_N^{k_0 n_{m-1} \\cdot 2^{m-1}}
}
\\]

\n

proceeding to the next innermost sum, over \\( n_{m-2} \\), and so on, and using
\\[ \\displaylines{
\\omega_N^{kn_{m-l}\\cdot 2^{m-l}} = \\omega_N^{(k_{m-1} \\cdot 2^{m-1} + … + k_1 \\cdot 2 + k_0)n_{m-l}\\cdot 2^{m-l}} \\\\
= \\omega_N^{(k_{l-1}\\cdot 2^{l-1} + … + k_0) n_{m-l} \\cdot 2^{m-1}}
}
\\]
one obtains successive arrays
\\[\\displaylines{
x_l(k_0, …, k_{l-1}, n_{m-l-1}, … , n_0) \\\\
= \\sum_{n_{m-l}}x_{l-1}(k_0, …, k_{l-2}, n_{m-l}, …, n_0 ) \\cdot \\omega_N^{(k_{l-1}\\cdot 2^{l-1} + …+ k_0) \\cdot n_{m-l} \\cdot 2^{m-l}}
}\\]
for \\(l = 1,2,…,m \\)

\n

writing out the sum this appears as
\\[ \\tag{3.4} \\displaylines{
x_l(k_0, …, k_{l-1}, n_{m-l-1}, … , n_0) \\\\
= x_{l-1}(k_0, …, k_{l-2}, 0, n_{m-l -1}, …, n_0 ) \\cdot \\omega_N^{(k_{l-1}\\cdot 2^{l-1} + …+ k_0) \\cdot 0 \\cdot 2^{m-l}} \\\\
+ x_{l-1}(k_0, …, k_{l-2}, 1, n_{m-l -1}, …, n_0 ) \\cdot \\omega_N^{(k_{l-1}\\cdot 2^{l-1} + …+ k_0) \\cdot 1 \\cdot 2^{m-l}} \\\\
= x_{l-1}(k_0, …, k_{l-2}, 0, n_{m-l -1}, …, n_0 ) \\\\
+ x_{l-1}(k_0, …, k_{l-2}, 1, n_{m-l-1}, …, n_0 ) \\cdot \\omega_N^{(k_{l-1}\\cdot 2^{l-1} + k_{l-2}\\cdot 2^{l-2 }+k_{l-3}\\cdot 2^{l-3} + …+ k_0) \\cdot 2^{m-l}} \\\\
= x_{l-1}(k_0, …, k_{l-2}, 0, n_{m-l -1}, …, n_0 ) \\\\
+ \\omega_N^{k_{l-1}\\cdot 2^{l-1} \\cdot 2^{m-l} } \\cdot \\omega_N^{k_{l-2}\\cdot 2^{l-2 } \\cdot 2^{m-l}} \\cdot x_{l-1}(k_0, …, k_{l-2}, 1, n_{m-l-1}, …, n_0 ) \\cdot \\omega_N^{(k_{l-3}\\cdot 2^{l-3} + …+ k_0) \\cdot 2^{m-l}} \\\\
= x_{l-1}(k_0, …, k_{l-2}, 0, n_{m-l -1}, …, n_0 ) \\\\
+ \\omega_N^{k_{l-1}\\cdot 2^{m-1} } \\cdot \\omega_N^{k_{l-2}\\cdot 2^{m-2}} \\cdot x_{l-1}(k_0, …, k_{l-2}, 1, n_{m-l-1}, …, n_0 ) \\cdot \\omega_N^{(k_{l-3}\\cdot 2^{l-3} + …+ k_0) \\cdot 2^{m-l}} \\\\
= x_{l-1}(k_0, …, k_{l-2}, 0, n_{m-l -1}, …, n_0 ) \\\\
+ (-1)^{k_{l-1} } \\cdot i^{k_{l-2}} \\cdot x_{l-1}(k_0, …, k_{l-2}, 1, n_{m-l-1}, …, n_0 ) \\cdot \\omega_N^{(k_{l-3}\\cdot 2^{l-3} + …+ k_0) \\cdot 2^{m-l}}
}\\]
according to the indexing convension, this is stored in a location whose index is
\\[ k_0 \\cdot 2^{m-1} + … + k_{l-1} \\cdot 2 ^{m-l} + n_{m-l-1} \\cdot 2^{m-l-1} + … + n_0 \\]

\n

It can be seen in E.q(3.4) that only the two storage locations with indices having 0 and 1 in the \\(2^{m-l}\\) bit position are involved in the computation. Parallel computation is permitted since the operation described by E.q(3.4) can be carried out with all values of \\(k_0, …, k_{l-2} \\), and \\( n_0, …, n_{m-l-1}\\) simultaneously. In some applications it is convenient to use E.q(3.4) to express \\(x_l\\) in terms of \\(x_{l-2}\\), giving what is equivalent to an algorithm with \\(r = 4\\).
the last array calculated gives the desired Fourier sums,

\n

\\[\\tag{3.5}
X(k_{m-1}, …, k_0) = x_{m}(k_0, …, k_{m-1})
\\]
in such an order that the index of an X must have its binary bits put in reverse order to yield its index in the array \\(x_m\\)

\n

references

\n","site":{"data":{}},"excerpt":"","more":"\n\n\n

Fourier Series

innter product of two functions

\\[ <f(x), g(x)> = \\int_{a}^{b} f(x) \\bar{g}(x)dx\\]
\\(\\bar{g}\\) is the congugate of g on complex number system

\n

Fourier series definition

let \\(f(x)\\) be a periodic function over a period of \\([-\\pi, +\\pi]\\)
\"f(x)\"

\n

\\( f(x) \\) can be transformed into below
\\[ f(x) = \\frac{A_{0}}{2} + \\sum_{k=1}^{\\infty}A_{k}cos(kx) + B_{k}sin(kx)\\],
where
\\[ A_{k} = \\frac{1}{\\pi} \\int_{-\\pi}^{+\\pi} f(x)cos(kx)dx = \\frac{1}{||cos(kx)||^2}<f(x),cos(kx)> \\]
\\[ B_{k} = \\frac{1}{\\pi} \\int_{-\\pi}^{+\\pi} f(x)sin(kx)dx = \\frac{1}{||sin(kx)||^2}<f(x),sin(kx)> \\]
where \\(A_{k}, B_{k}\\) is just the innver product of \\(f(x)\\), and \\( cos(kx) \\), \\(sin(kx)\\) respectively. intuitively, it is the projection of \\(f(x)\\) over \\(sin(kx)\\), or \\(cos(kx)\\).

\n

if period is L

\\[ f(x) = \\frac{A_{0}}{2} + \\sum_{k=1}^{\\infty}A_{k}cos(\\frac{2\\pi kx}{L}) + B_{k}sin(\\frac{2\\pi kx}{L})\\],
where
\\[ A_{k} = \\frac{2}{L} \\int_{0}^{L} f(x)cos(\\frac{2\\pi kx}{L})dx\\]
\\[ B_{k} = \\frac{2}{L} \\int_{0}^{L} f(x)sin(\\frac{2\\pi kx}{L})dx\\]

\n

Complex Fourier Series

let’s define \\[ \\psi_k = e^{ikx} = cos(kx) + i \\cdot sin(kx)\\]
we can prove that \\(\\psi_k, and \\psi_j\\) are orthogonal as their inner product is zero
proof
\\[ <\\psi_j, \\psi_k> = \\int_{-\\pi}^{+\\pi} e^{jkx}e^{-ikx}dx = \\int_{-\\pi}^{+\\pi} e^{j(j-k)x}dx = \\frac{1}{i(j-k)}[e^{i(j-k)x}]_{-\\pi}^{\\pi}\\]
it is 0 if \\(j \\ne k\\) , \\(2\\pi \\) if \\(j=k\\)

\n

note conjugate of \\(\\psi_k\\) is \\(e^{-ikx}\\)

\n

\\(f(x) = \\frac{1}{2\\pi} \\sum_{k=-\\infty}^{\\infty} c_k \\psi_k\\), where \\(c_k=<f(x), \\psi_k> = \\int_{-\\pi}^{+\\pi} f(x)e^{-ikx}dx\\)

\n

Discrete Fourier Transform (DFT)

The discrete Fourier transform transforms a sequence of N complex numbers
\\[\\lbrace x_{n} \\rbrace := x_0, x_1,…, x_{N-1} \\] into another sequence of complex numbers,
\\[\\lbrace X_{k} \\rbrace := X_0, X_1,…, X_{N-1} \\] which is defined by
\\[X_{k} = \\sum_{n=0}^{N-1} x_{n} \\cdot e^{-\\frac{i2\\pi}{N}kn}\\]

\n

let the n-th root of unity be \\(\\omega_N = e^{-\\frac{2\\pi i}{N}}\\), we have
\\[ \\tag{1.1} X_{k} = \\sum_{n=0}^{N-1} x_{n} \\cdot \\omega_N^{kn}\\]

\n

inverse DFT

The inverse transform is given by
\\[ \\tag{1.2} x_{n} = \\frac{1}{N}\\sum_{k=0}^{N-1} X_{k} \\cdot \\omega_N^{-kn}\\]

\n

the unitary DFT

Another way of looking at the DFT is to note that the DFT can be expressed as the DFT matrix, a Vandermonde matrix, introduced by Sylvester in 1867.
\\[
\\boldsymbol{F} =
\\begin{bmatrix}
\\displaylines{
\\omega_{N}^{0\\cdot0} & \\omega_{N}^{0\\cdot1} & \\dots & \\omega_{N}^{0\\cdot (N-1)}\\\\
\\omega_{N}^{1\\cdot0} & \\omega_{N}^{1\\cdot1} & \\dots & \\omega_{N}^{2\\cdot (N-1)}\\\\
\\vdots & \\vdots & \\ddots & \\vdots\\\\
\\omega_{N}^{N-1\\cdot0} & \\omega_{N}^{N-1\\cdot1} & \\dots & \\omega_{N}^{N-1\\cdot (N-1)}
}
\\end{bmatrix}
\\]
where \\( \\omega_{N} = e^{-\\frac{i2\\pi}{N}}\\) is a primitive Nth root of unity. (any complex number that yields 1 when raised to some positive integer power n)

\n

\\[ X = F \\cdot x^{T}\\]
\\(x\\) is a column vector

\n

Polynomial & DFT

Polynomial evaluation & DFT

let a polynomial be
$$f(x)=\\sum_{n=0}^{N-1}x_nx^n$$
then,
$$f(\\omega_N^{k})=\\sum_{n=0}^{N-1}x_n\\omega_N^{kn},\\quad k=0,1,2,…,N-1$$
compare this equation with E.q(1.1), we can know that DFT is the evaluations of a polynomial at N points of Nth root of unity, \\(\\omega_N^{0},\\omega_N^1,\\omega_N^2,…,\\omega_N^{N-1}\\)

\n

$$(\\omega_N^{0},X_0),(\\omega_N^1,X_1),(\\omega_N^2,X_2),…,(\\omega_N^{N-1},X_{N-1})$$

\n

Polynomial interpolation & IDFT

According to E.q(1.2), IDFT is the polynomial interpolation (getting original polynomial coefficients).

\n

Fast Fourier Transform (FFT) [3]

The problem is to calculate the below
\\[ \\tag{2.1} X_{k} = \\sum_{n=0}^{N-1} x_{n} \\cdot \\omega_N^{kn}, \\quad k= 0,1,…,N-1\\]
A straightforward calculation using would require \\(N^2\\) operations where “operation” means, as it will throughout this note, a complex multiplication followed by a complex addition.
The FFT algorithm described here iterates on the array and yields the result in less than \\( 2N log_2N \\)

\n

To derive the algorithm, suppose \\(N\\) is a composite, i.e., \\(N = r_1\\cdot r_2\\). Then let the indices in (2.1) be expressed
\\[ k = k_1r_1 + k_0, \\quad k_0 = 0,1,…,r_1-1, \\quad k_1 = 0,1,…, r_2 -1 \\]
\\[ n = n_1r_2 + n_0, \\quad n_0 = 0,1,…,r_2-1, \\quad n_1 = 0,1,…, r_1 -1 \\]

\n

Since \\( n = n_1r_2 + n_0\\), we can write

\n

\\[ \\tag{2.2} X(k_1, k_0) = \\sum_{n_0} \\sum_{n_1}x(n_1,n_0)\\omega_N^{kn} = \\sum_{n_0} \\sum_{n_1}x(n_1,n_0)\\omega_N^{kn_1r_2}\\omega_N^{kn_0} \\]
note \\( x(n_1,n_0)\\) is same to \\(x(n)\\), just the indexing is transformed form 1D to 2D. You can think original 1D array of size N is resized to a matrix of size \\( (r_2, r_1) \\), \\(r_1\\) is number of cols, while \\(r_2\\) is number of rows

\n

Since \\(k = k_1r_1 + k_0\\), we have
\\[ \\omega_N^{kn_1r_2} =\\omega_N^{(k_1r_1+k_0)n_1r_2} = \\omega_N^{(k_1r_1)n_1r_2} \\cdot \\omega_N^{k_0n_1r_2}\\]
according to the property of Nth root of unity, \\( \\omega_N^{N} = \\omega_N^{r_1r_2} =1 \\), \\( \\omega_N^{(k_1r_1)n_1r_2}\\) is also 1.
then we have
\\[ \\omega_N^{kn_1r_2} = \\omega_N^{k_0n_1r_2}\\]
substitute it into E.q 2.2, we get
\\[ \\tag{2.3} X(k_1, k_0) = \\sum_{n_0} \\sum_{n_1}x(n_1,n_0)\\omega_N^{kn} = \\sum_{n_0} \\sum_{n_1}x(n_1,n_0)\\omega_N^{k_0n_1r_2}\\omega_N^{kn_0} \\]
Therefore, the inner sum, over \\(n_1\\), depends only on \\(k_0\\) and \\(n_0\\), and can be defined as a new array,
\\[ \\tag{2.4} x_1(k_0, n_0) = \\sum_{n_1} x(n_1, n_0) \\cdot \\omega_N^{k_0n_1r_2}\\]
E.q(2.3) can be writtern as
\\[ \\tag{2.5} X(k_1, k_0) = \\sum_{n_0} x_1(k_0,n_0)\\omega_N^{kn_0} = \\sum_{n_0} x_1(k_0,n_0)\\omega_N^{(k_1r_1 + k_0)n_0} \\]

\n

There are N elements in the matrix \\(x_1\\), each requiring \\(r_1\\) operations, giving a total of \\(Nr_1\\) operations. Similarly, it takes \\(Nr_2\\) operations to calculate \\(X\\) from \\(x_1\\). Therefore, the two-step algorithm, given by Eq(2.4) and Eq(2.5) requires a total of
\\[ \\tag{2.6} T = N(r_1+r_2) \\]

\n

it is easy to see how successive applications of the above procedure (recursively), starting with its appliation to Eq(2.4) give an m-step algorihtm requiring
\\[ \\tag{2.7} T = N(r_1+r_2 + … + r_m) \\]
where
\\[ N = \\prod_{j=1}^{j=m}r_j\\]
if all \\(r_j\\) are equal to \\(r\\), i.e \\(N = r^m \\), which gives \\( m = log_rN \\)
E.q(2.7) becomes
\\[ \\tag{2.8} T = N(m \\cdot r) = rNm = rN(log_rN) \\]

\n

radix-2 FFT

The algorithm with \\(r=2\\) is derived by expressing the indices in the form

\n

\\[\\tag{3.1} k=k_{m-1} \\cdot 2^{m-1} + … + k_1 \\cdot 2 + k_0 \\]
\\[\\tag{3.2} n=n_{m-1} \\cdot 2^{m-1} + … + n_1 \\cdot 2 + n_0 \\]
where \\( k_v \\in [0,1] \\), for \\( v = 0,…,m-1 \\), and \\( n_v \\in [0,1] \\), for \\( v = 0,…,m-1 \\)
\\(k_v\\) and \\(n_v\\) are the contents of the respective bit positions in the binary representation of \\(k\\) and \\(n\\)

\n

All arrays will now be written as functions of the bits of their indices. With this convention E.q(2.1) is written as
\\[ \\tag{3.3}
\\displaylines{
X(k_{m-1}, …, k_0) = \\sum_{n_0}\\sum_{n_1} … \\sum_{n_{m-1}} x(n_{m-1}, …,n_1, n_0) \\cdot \\omega_N^{kn} \\\\
= \\sum_{n_0}\\sum_{n_1} … \\sum_{n_{m-1}} x(n_{m-1}, …,n_1, n_0) \\cdot \\omega_N^{k(n_{m-1} \\cdot 2^{m-1} + … + n_1 \\cdot 2 + n_0)} \\\\
= \\sum_{n_0}\\sum_{n_1} … \\sum_{n_{m-1}} x(n_{m-1}, …,n_1, n_0) \\cdot \\omega_N^{kn_{m-1} \\cdot 2^{m-1} + … + kn_1 \\cdot 2 + kn_0}
}
\\]
where the sums are over \\(n_v \\in [0,1]\\), for \\(v = 0,1,…,m-1\\)

\n

Since \\[ \\displaylines{
\\omega_N^{kn_{m-1}\\cdot 2^{m-1}} = \\omega_N^{(k_{m-1} \\cdot 2^{m-1} + … + k_1 \\cdot 2 + k_0)n_{m-1}\\cdot 2^{m-1}} \\\\
= \\omega_N^{k_0 n_{m-1} \\cdot 2^{m-1}}
}
\\]
(it is easy to show that all other terms are 1 as \\( \\omega_N^{2^m} = 1 \\), so only \\( k_0\\) is kept)

\n

the innermost sum of E.q(3.3) over \\(n_{m-1} \\), depends only on \\( k_0, n_{m-2}, …, n_0\\) and can be writtern
\\[ \\displaylines{
x_1(k_0, n_{m-2}, …, n_1, n_0) = \\sum_{n_{m-1}} x(n_{m-1}, …, n_0) \\cdot \\omega_N^{k_0 n_{m-1} \\cdot 2^{m-1}}
}
\\]

\n

proceeding to the next innermost sum, over \\( n_{m-2} \\), and so on, and using
\\[ \\displaylines{
\\omega_N^{kn_{m-l}\\cdot 2^{m-l}} = \\omega_N^{(k_{m-1} \\cdot 2^{m-1} + … + k_1 \\cdot 2 + k_0)n_{m-l}\\cdot 2^{m-l}} \\\\
= \\omega_N^{(k_{l-1}\\cdot 2^{l-1} + … + k_0) n_{m-l} \\cdot 2^{m-1}}
}
\\]
one obtains successive arrays
\\[\\displaylines{
x_l(k_0, …, k_{l-1}, n_{m-l-1}, … , n_0) \\\\
= \\sum_{n_{m-l}}x_{l-1}(k_0, …, k_{l-2}, n_{m-l}, …, n_0 ) \\cdot \\omega_N^{(k_{l-1}\\cdot 2^{l-1} + …+ k_0) \\cdot n_{m-l} \\cdot 2^{m-l}}
}\\]
for \\(l = 1,2,…,m \\)

\n

writing out the sum this appears as
\\[ \\tag{3.4} \\displaylines{
x_l(k_0, …, k_{l-1}, n_{m-l-1}, … , n_0) \\\\
= x_{l-1}(k_0, …, k_{l-2}, 0, n_{m-l -1}, …, n_0 ) \\cdot \\omega_N^{(k_{l-1}\\cdot 2^{l-1} + …+ k_0) \\cdot 0 \\cdot 2^{m-l}} \\\\
+ x_{l-1}(k_0, …, k_{l-2}, 1, n_{m-l -1}, …, n_0 ) \\cdot \\omega_N^{(k_{l-1}\\cdot 2^{l-1} + …+ k_0) \\cdot 1 \\cdot 2^{m-l}} \\\\
= x_{l-1}(k_0, …, k_{l-2}, 0, n_{m-l -1}, …, n_0 ) \\\\
+ x_{l-1}(k_0, …, k_{l-2}, 1, n_{m-l-1}, …, n_0 ) \\cdot \\omega_N^{(k_{l-1}\\cdot 2^{l-1} + k_{l-2}\\cdot 2^{l-2 }+k_{l-3}\\cdot 2^{l-3} + …+ k_0) \\cdot 2^{m-l}} \\\\
= x_{l-1}(k_0, …, k_{l-2}, 0, n_{m-l -1}, …, n_0 ) \\\\
+ \\omega_N^{k_{l-1}\\cdot 2^{l-1} \\cdot 2^{m-l} } \\cdot \\omega_N^{k_{l-2}\\cdot 2^{l-2 } \\cdot 2^{m-l}} \\cdot x_{l-1}(k_0, …, k_{l-2}, 1, n_{m-l-1}, …, n_0 ) \\cdot \\omega_N^{(k_{l-3}\\cdot 2^{l-3} + …+ k_0) \\cdot 2^{m-l}} \\\\
= x_{l-1}(k_0, …, k_{l-2}, 0, n_{m-l -1}, …, n_0 ) \\\\
+ \\omega_N^{k_{l-1}\\cdot 2^{m-1} } \\cdot \\omega_N^{k_{l-2}\\cdot 2^{m-2}} \\cdot x_{l-1}(k_0, …, k_{l-2}, 1, n_{m-l-1}, …, n_0 ) \\cdot \\omega_N^{(k_{l-3}\\cdot 2^{l-3} + …+ k_0) \\cdot 2^{m-l}} \\\\
= x_{l-1}(k_0, …, k_{l-2}, 0, n_{m-l -1}, …, n_0 ) \\\\
+ (-1)^{k_{l-1} } \\cdot i^{k_{l-2}} \\cdot x_{l-1}(k_0, …, k_{l-2}, 1, n_{m-l-1}, …, n_0 ) \\cdot \\omega_N^{(k_{l-3}\\cdot 2^{l-3} + …+ k_0) \\cdot 2^{m-l}}
}\\]
according to the indexing convension, this is stored in a location whose index is
\\[ k_0 \\cdot 2^{m-1} + … + k_{l-1} \\cdot 2 ^{m-l} + n_{m-l-1} \\cdot 2^{m-l-1} + … + n_0 \\]

\n

It can be seen in E.q(3.4) that only the two storage locations with indices having 0 and 1 in the \\(2^{m-l}\\) bit position are involved in the computation. Parallel computation is permitted since the operation described by E.q(3.4) can be carried out with all values of \\(k_0, …, k_{l-2} \\), and \\( n_0, …, n_{m-l-1}\\) simultaneously. In some applications it is convenient to use E.q(3.4) to express \\(x_l\\) in terms of \\(x_{l-2}\\), giving what is equivalent to an algorithm with \\(r = 4\\).
the last array calculated gives the desired Fourier sums,

\n

\\[\\tag{3.5}
X(k_{m-1}, …, k_0) = x_{m}(k_0, …, k_{m-1})
\\]
in such an order that the index of an X must have its binary bits put in reverse order to yield its index in the array \\(x_m\\)

\n

references

\n"},{"title":"geth start","date":"2022-11-01T10:15:12.000Z","_content":"\n## build from source\n```\ngit clone https://github.com/ethereum/go-ethereum.git\ncd go-ethereum\nmake geth\n```\n\n## understanding geth config\ngeth config type is defined in /cmd/geth/config.go\n```go\ntype gethConfig struct {\n\tEth ethconfig.Config\n\tNode node.Config\n\tEthstats ethstatsConfig\n\tMetrics metrics.Config\n}\n```\n- **ethconfig** (eth/ethconfig/config.go)\ncontains configuration options for of the ETH and LES(light node) protocols, such as NetworkId, SyncMode, txpool.Config, database options\n- **nodeConfig** (node/config.go)\nrepresents a small collection of configuration values to fine tune the P2P network layer of a protocol stack. These values can be further extended by all registered services. such as p2p.Config, DataDir, KeyStoreDir, HTTPHost, HTTPModules(eth,net,web3), WSHost\n- **metrics.Config** (metrics/config.go)\ncontains the configuration for the metric collection, such as InfluxDBEndpoint, etc\n- **ethstatsConfig**\nonly one URL entry\n\ngeth provides default config in the above files. user config file path is given by the below flag\n```go\nconfigFileFlag = &cli.StringFlag{\n\t\tName: \"config\",\n\t\tUsage: \"TOML configuration file\",\n\t\tCategory: flags.EthCategory,\n\t}\n```\n\nThe config file should be a .toml file. A convenient way to create a config file is to get Geth to create one for you and use it as a template. To do this, use the dumpconfig command, saving the result to a .toml file. Note that you also need to explicitly provide the network_id on the command line for the public testnets such as Sepolia or Geoerli:\n```\n./geth --sepolia dumpconfig > geth-config.toml\n```\nto specify path to config file\n```\ngeth --sepolia --config geth-config.toml\n```\n\n## key configs\n- [Eth].TxLookupLimit \nNumber of recent blocks to maintain transactions index for (default = about one year, 0 = entire chain), default: 2350000\n- [Node].BootstrapNodes\nused to establish connectivity with the rest of the network.\ngeth provides default bootstrapNodes in file `params/bootnodes.go`\n- [Metrics_AND_STATS].ethstats\nReporting URL of a ethstats service (nodename:secret@host:port), [more detail](https://geth.ethereum.org/docs/monitoring/ethstats)\n- SyncMode\n- TrieDirtyCache\n- NoPruning\n- TrieCleanCacheJournal e.g triecache\n## how geth starts\n\n![geth starts](/images/geth_starts.drawio.png)\nthe main func is in `cmd/geth/main.go`\n```go\nfunc main() {\n\tif err := app.Run(os.Args); err != nil {\n\t\tfmt.Fprintln(os.Stderr, err)\n\t\tos.Exit(1)\n\t}\n}\n```\nthe main() function is very short, and its main function is to start a tool for parsing command line commands: `gopkg.in/urfave/cli.v1`. Going deeper, we will find that `app.Action = geth` will be called when the cli app is initialized to call the geth() function\n```go\nfunc init() {\n\t// Initialize the CLI app and start Geth\n\tapp.Action = geth\n // ....\n}\n```\ngeth is the main entry point into the system if no special subcommand is run. It creates a default node based on the command line arguments and runs it in blocking mode, waiting for it to be shut down.\n```go\nfunc geth(ctx *cli.Context) error {\n\tif args := ctx.Args().Slice(); len(args) > 0 {\n\t\treturn fmt.Errorf(\"invalid command: %q\", args[0])\n\t}\n\n\tprepare(ctx)\n\tstack, backend := makeFullNode(ctx)\n\tdefer stack.Close()\n\n\tstartNode(ctx, stack, backend, false)\n\tstack.Wait()\n\treturn nil\n}\n```\nIn the geth() function, there are three important function calls, namely: `prepare()`, `makeFullNode()`, and `startNode()`.\n\n### prepare\nThe implementation of the prepare() function is in the current main.go file. It is mainly used to set some configurations required for node initialization.\n\n### makeFullNode\nThe implementation of the `makeFullNode()` function is located in the `cmd/geth/config.go` file. It will load the context of the command and apply user given configuration; and generate instances of `stack` and `backend`. Among them, `stack` is an instance of `Node` type (Node is the top-level instance in the life cycle of geth. It is responsible for managing high-level abstractions such as P2P Server, Http Server, and Database in the node. The definition of the Node type is located in the `node/node.go` file), which is initialized by calling `makeConfigNode()` function through `makeFullNode()` function. inside `makeFullNode`, it calls `node.New(&cfg.Node)` to initiate a node. During instantiating of node, it invokes `rpc.NewServer()` to create a new rpc server and put in the field `inprocHandler`. it registers `rpc` api namespace by default.\n\nThe `backend` here is an interface of `ethapi.Backend` type, which provides the basic functions needed to obtain the runtime of the Ethereum execution layer. Its definition is located in `internal/ethapi/backend.go`. Since there are many functions in this interface, we have selected some of the key functions as below for a glimpse of its functionality. `backend` is created by calling `backend, eth := utils.RegisterEthService(stack, &cfg.Eth)`. Inside, it calls `eth.New(stack, cfg)` to create `backend` instance. During `backend` initiating, it opens database (`chainDb, err := stack.OpenDatabaseWithFreezer(\"chaindata\", config.DatabaseCache, config.DatabaseHandles, config.DatabaseFreezer, \"eth/db/chaindata/\", false)`). Further, it creates consensus engine, `engine := ethconfig.CreateConsensusEngine(stack, ðashConfig, cliqueConfig, config.Miner.Notify, config.Miner.Noverify, chainDb)`. goerli testnet use POA consensus (clique). \n```go\ntype Backend interface {\n\tSyncProgress() ethereum.SyncProgress\n\tSuggestGasTipCap(ctx context.Context) (*big.Int, error)\n\tChainDb() ethdb.Database\n\tAccountManager() *accounts.Manager\n\tExtRPCEnabled() bool\n\tRPCGasCap() uint64 // global gas cap for eth_call over rpc: DoS protection\n\tRPCEVMTimeout() time.Duration // global timeout for eth_call over rpc: DoS protection\n\tRPCTxFeeCap() float64 // global tx fee cap for all transaction related APIs\n\tUnprotectedAllowed() bool // allows only for EIP155 transactions.\n\tSetHead(number uint64)\n\tHeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Header, error)\n\tHeaderByHash(ctx context.Context, hash common.Hash) (*types.Header, error)\n\tHeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*types.Header, error)\n\tCurrentHeader() *types.Header\n\tCurrentBlock() *types.Header\n\tBlockByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Block, error)\n\tBlockByHash(ctx context.Context, hash common.Hash) (*types.Block, error)\n\tBlockByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*types.Block, error)\n\tStateAndHeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*state.StateDB, *types.Header, error)\n\tStateAndHeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*state.StateDB, *types.Header, error)\n\tPendingBlockAndReceipts() (*types.Block, types.Receipts)\n\tGetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error)\n\tGetTd(ctx context.Context, hash common.Hash) *big.Int\n\tGetEVM(ctx context.Context, msg *core.Message, state *state.StateDB, header *types.Header, vmConfig *vm.Config) (*vm.EVM, func() error, error)\n\tSubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription\n\tSubscribeChainHeadEvent(ch chan<- core.ChainHeadEvent) event.Subscription\n\tSubscribeChainSideEvent(ch chan<- core.ChainSideEvent) event.Subscription\n\tSendTx(ctx context.Context, signedTx *types.Transaction) error\n\tGetTransaction(ctx context.Context, txHash common.Hash) (*types.Transaction, common.Hash, uint64, uint64, error)\n\tGetPoolTransactions() (types.Transactions, error)\n\tGetPoolTransaction(txHash common.Hash) *types.Transaction\n\tGetPoolNonce(ctx context.Context, addr common.Address) (uint64, error)\n\tStats() (pending int, queued int)\n\tTxPoolContent() (map[common.Address]types.Transactions, map[common.Address]types.Transactions)\n\tTxPoolContentFrom(addr common.Address) (types.Transactions, types.Transactions)\n\tSubscribeNewTxsEvent(chan<- core.NewTxsEvent) event.Subscription\n\tChainConfig() *params.ChainConfig\n\tEngine() consensus.Engine\n\tGetBody(ctx context.Context, hash common.Hash, number rpc.BlockNumber) (*types.Body, error)\n\tGetLogs(ctx context.Context, blockHash common.Hash, number uint64) ([][]*types.Log, error)\n\tSubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription\n\tSubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription\n\tSubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription\n\tBloomStatus() (uint64, uint64)\n\tServiceFilter(ctx context.Context, session *bloombits.MatcherSession)\n}\n```\n\nIf readers want to customize some new RPC APIs, they can define functions in the /internal/ethapi.Backend interface and add specific implementations to EthAPIBackend\n\n### startNode\nThe last key function, `startNode()`, is to officially start an Ethereum execution layer node. It starts the Stack instance (Node) by calling the utils.StartNode() function which triggers the Node.Start() function. In the Node.Start() function, it traverses the backend instances registered in `Node.lifecycles` and starts them. In addition, in the startNode() function, the unlockAccounts() function is still called, and the unlocked wallet is registered in the stack, and the RPClient module that interacts with local Geth is created through the stack.Attach() function\n\nAt the end of the geth() function, the function executes `stack.Wait()`, so that the main thread enters the blocking state, and the services of other functional modules are distributed to other sub-coroutines for maintenance\n\n## Node\nAs we mentioned earlier, the Node type belongs to the top-level instance in the life cycle of geth, and it is responsible for being the administrator of the high-level abstract module communicating with the outside world, such as managing rpc server, http server, Web Socket, and P2P Server external interface . At the same time, Node maintains the back-end instances and services (lifecycles []Lifecycle) required for node operation, such as the Ethereum type we mentioned above that is responsible for the specific Service\n```go\ntype Node struct {\n\teventmux *event.TypeMux\n\tconfig *Config\n\taccman *accounts.Manager\n\tlog log.Logger\n\tkeyDir string // key store directory\n\tkeyDirTemp bool // If true, key directory will be removed by Stop\n\tdirLock *flock.Flock // prevents concurrent use of instance directory\n\tstop chan struct{} // Channel to wait for termination notifications\n\tserver *p2p.Server // Currently running P2P networking layer\n\tstartStopLock sync.Mutex // Start/Stop are protected by an additional lock\n\tstate int // Tracks state of node lifecycle\n\n\tlock sync.Mutex\n\tlifecycles []Lifecycle // All registered backends, services, and auxiliary services that have a lifecycle\n\trpcAPIs []rpc.API // List of APIs currently provided by the node\n\thttp *httpServer //\n\tws *httpServer //\n\thttpAuth *httpServer //\n\twsAuth *httpServer //\n\tipc *ipcServer // Stores information about the ipc http server\n\tinprocHandler *rpc.Server // In-process RPC request handler to process the API requests\n\n\tdatabases map[*closeTrackingDB]struct{} // All open databases\n}\n```\n\n### close node\nAs mentioned earlier, the main thread of the entire program is blocked because of calling `stack.Wait()`. We can see that a channel called `stop` is declared in the Node structure. Since this Channel has not been assigned a value, the main process of the entire geth enters the blocking state, and continues to execute other business coroutines concurrently\n```go\n// Wait blocks until the node is closed.\nfunc (n *Node) Wait() {\n <-n.stop\n}\n```\nWhen the Channel n.stop is assigned a value, the geth main function will stop the current blocking state and start to perform a series of corresponding resource release operations.\nIt is worth noting that in the current codebase of go-ethereum, the blocking state of the main process is not ended directly by assigning a value to the stop channel, but a more concise and rude way is used: call the close() function directly Close the Channel. We can find the related implementation in node.doClose(). close() is a native function of go language, used when closing Channel.\n```go\n// doClose releases resources acquired by New(), collecting errors.\nfunc (n *Node) doClose(errs []error) error {\n // Close databases. This needs the lock because it needs to\n // synchronize with OpenDatabase*.\n n.lock.Lock()\n n.state = closedState\n errs = append(errs, n.closeDatabases()...)\n n.lock.Unlock()\n\n if err := n.accman.Close(); err != nil {\n errs = append(errs, err)\n }\n if n.keyDirTemp {\n if err := os.RemoveAll(n.keyDir); err != nil {\n errs = append(errs, err)\n }\n }\n\n // Release instance directory lock.\n n.closeDataDir()\n\n // Unblock n.Wait.\n close(n.stop)\n\n // Report any errors that might have occurred.\n switch len(errs) {\n case 0:\n return nil\n case 1:\n return errs[0]\n default:\n return fmt.Errorf(\"%v\", errs)\n }\n}\n```\n\n## Ethereum Backend\nWe can find the definition of the Ethereum structure in eth/backend.go. The member variables and receiving methods contained in this structure implement all the functions and data structures required by an Ethereum full node. We can see in the following code definition that the Ethereum structure contains several core data structures such as TxPool, Blockchain, consensus.Engine, and miner as member variables.\n```go\ntype Ethereum struct {\n\tconfig *ethconfig.Config\n\n\t// Handlers\n\ttxPool *txpool.TxPool\n\tblockchain *core.BlockChain\n\thandler *handler\n\tethDialCandidates enode.Iterator\n\tsnapDialCandidates enode.Iterator\n\tmerger *consensus.Merger\n\n\t// DB interfaces\n\tchainDb ethdb.Database // Block chain database\n\n\teventMux *event.TypeMux\n\tengine consensus.Engine\n\taccountManager *accounts.Manager\n\n\tbloomRequests chan chan *bloombits.Retrieval // Channel receiving bloom data retrieval requests\n\tbloomIndexer *core.ChainIndexer // Bloom indexer operating during block imports\n\tcloseBloomHandler chan struct{}\n\n\tAPIBackend *EthAPIBackend\n\n\tminer *miner.Miner\n\tgasPrice *big.Int\n\tetherbase common.Address\n\n\tnetworkID uint64\n\tnetRPCService *ethapi.NetAPI\n\n\tp2pServer *p2p.Server\n\n\tlock sync.RWMutex // Protects the variadic fields (e.g. gas price and etherbase)\n\n\tshutdownTracker *shutdowncheck.ShutdownTracker // Tracks if and when the node has shutdown ungracefully\n}\n```\nNodes start and stop Mining by calling `Ethereum.StartMining()` and `Ethereum.StopMining()`. Setting the profit account of Mining is achieved by calling `Ethereum.SetEtherbase()`\nHere we pay extra attention to the member variable `handler`. The definition of `handler` is in `eth/handler.go`.\nFrom a macro point of view, the main workflow of a node needs to: 1. Obtain/synchronize Transaction and Block data from the network 2. Add the Block obtained from the network to the Blockchain. The handler is responsible for providing the function of synchronizing blocks and transaction data, for example, `downloader.Downloader` is responsible for synchronizing Block from the network, and `fetcher.TxFetcher` is responsible for synchronizing transactions from the network","source":"_posts/geth/code_analysis/geth.0.get.start.md","raw":"---\ntitle: geth start\ndate: 2022-11-01 18:15:12\ntags: [blockchain,geth]\n---\n\n## build from source\n```\ngit clone https://github.com/ethereum/go-ethereum.git\ncd go-ethereum\nmake geth\n```\n\n## understanding geth config\ngeth config type is defined in /cmd/geth/config.go\n```go\ntype gethConfig struct {\n\tEth ethconfig.Config\n\tNode node.Config\n\tEthstats ethstatsConfig\n\tMetrics metrics.Config\n}\n```\n- **ethconfig** (eth/ethconfig/config.go)\ncontains configuration options for of the ETH and LES(light node) protocols, such as NetworkId, SyncMode, txpool.Config, database options\n- **nodeConfig** (node/config.go)\nrepresents a small collection of configuration values to fine tune the P2P network layer of a protocol stack. These values can be further extended by all registered services. such as p2p.Config, DataDir, KeyStoreDir, HTTPHost, HTTPModules(eth,net,web3), WSHost\n- **metrics.Config** (metrics/config.go)\ncontains the configuration for the metric collection, such as InfluxDBEndpoint, etc\n- **ethstatsConfig**\nonly one URL entry\n\ngeth provides default config in the above files. user config file path is given by the below flag\n```go\nconfigFileFlag = &cli.StringFlag{\n\t\tName: \"config\",\n\t\tUsage: \"TOML configuration file\",\n\t\tCategory: flags.EthCategory,\n\t}\n```\n\nThe config file should be a .toml file. A convenient way to create a config file is to get Geth to create one for you and use it as a template. To do this, use the dumpconfig command, saving the result to a .toml file. Note that you also need to explicitly provide the network_id on the command line for the public testnets such as Sepolia or Geoerli:\n```\n./geth --sepolia dumpconfig > geth-config.toml\n```\nto specify path to config file\n```\ngeth --sepolia --config geth-config.toml\n```\n\n## key configs\n- [Eth].TxLookupLimit \nNumber of recent blocks to maintain transactions index for (default = about one year, 0 = entire chain), default: 2350000\n- [Node].BootstrapNodes\nused to establish connectivity with the rest of the network.\ngeth provides default bootstrapNodes in file `params/bootnodes.go`\n- [Metrics_AND_STATS].ethstats\nReporting URL of a ethstats service (nodename:secret@host:port), [more detail](https://geth.ethereum.org/docs/monitoring/ethstats)\n- SyncMode\n- TrieDirtyCache\n- NoPruning\n- TrieCleanCacheJournal e.g triecache\n## how geth starts\n\n![geth starts](/images/geth_starts.drawio.png)\nthe main func is in `cmd/geth/main.go`\n```go\nfunc main() {\n\tif err := app.Run(os.Args); err != nil {\n\t\tfmt.Fprintln(os.Stderr, err)\n\t\tos.Exit(1)\n\t}\n}\n```\nthe main() function is very short, and its main function is to start a tool for parsing command line commands: `gopkg.in/urfave/cli.v1`. Going deeper, we will find that `app.Action = geth` will be called when the cli app is initialized to call the geth() function\n```go\nfunc init() {\n\t// Initialize the CLI app and start Geth\n\tapp.Action = geth\n // ....\n}\n```\ngeth is the main entry point into the system if no special subcommand is run. It creates a default node based on the command line arguments and runs it in blocking mode, waiting for it to be shut down.\n```go\nfunc geth(ctx *cli.Context) error {\n\tif args := ctx.Args().Slice(); len(args) > 0 {\n\t\treturn fmt.Errorf(\"invalid command: %q\", args[0])\n\t}\n\n\tprepare(ctx)\n\tstack, backend := makeFullNode(ctx)\n\tdefer stack.Close()\n\n\tstartNode(ctx, stack, backend, false)\n\tstack.Wait()\n\treturn nil\n}\n```\nIn the geth() function, there are three important function calls, namely: `prepare()`, `makeFullNode()`, and `startNode()`.\n\n### prepare\nThe implementation of the prepare() function is in the current main.go file. It is mainly used to set some configurations required for node initialization.\n\n### makeFullNode\nThe implementation of the `makeFullNode()` function is located in the `cmd/geth/config.go` file. It will load the context of the command and apply user given configuration; and generate instances of `stack` and `backend`. Among them, `stack` is an instance of `Node` type (Node is the top-level instance in the life cycle of geth. It is responsible for managing high-level abstractions such as P2P Server, Http Server, and Database in the node. The definition of the Node type is located in the `node/node.go` file), which is initialized by calling `makeConfigNode()` function through `makeFullNode()` function. inside `makeFullNode`, it calls `node.New(&cfg.Node)` to initiate a node. During instantiating of node, it invokes `rpc.NewServer()` to create a new rpc server and put in the field `inprocHandler`. it registers `rpc` api namespace by default.\n\nThe `backend` here is an interface of `ethapi.Backend` type, which provides the basic functions needed to obtain the runtime of the Ethereum execution layer. Its definition is located in `internal/ethapi/backend.go`. Since there are many functions in this interface, we have selected some of the key functions as below for a glimpse of its functionality. `backend` is created by calling `backend, eth := utils.RegisterEthService(stack, &cfg.Eth)`. Inside, it calls `eth.New(stack, cfg)` to create `backend` instance. During `backend` initiating, it opens database (`chainDb, err := stack.OpenDatabaseWithFreezer(\"chaindata\", config.DatabaseCache, config.DatabaseHandles, config.DatabaseFreezer, \"eth/db/chaindata/\", false)`). Further, it creates consensus engine, `engine := ethconfig.CreateConsensusEngine(stack, ðashConfig, cliqueConfig, config.Miner.Notify, config.Miner.Noverify, chainDb)`. goerli testnet use POA consensus (clique). \n```go\ntype Backend interface {\n\tSyncProgress() ethereum.SyncProgress\n\tSuggestGasTipCap(ctx context.Context) (*big.Int, error)\n\tChainDb() ethdb.Database\n\tAccountManager() *accounts.Manager\n\tExtRPCEnabled() bool\n\tRPCGasCap() uint64 // global gas cap for eth_call over rpc: DoS protection\n\tRPCEVMTimeout() time.Duration // global timeout for eth_call over rpc: DoS protection\n\tRPCTxFeeCap() float64 // global tx fee cap for all transaction related APIs\n\tUnprotectedAllowed() bool // allows only for EIP155 transactions.\n\tSetHead(number uint64)\n\tHeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Header, error)\n\tHeaderByHash(ctx context.Context, hash common.Hash) (*types.Header, error)\n\tHeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*types.Header, error)\n\tCurrentHeader() *types.Header\n\tCurrentBlock() *types.Header\n\tBlockByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Block, error)\n\tBlockByHash(ctx context.Context, hash common.Hash) (*types.Block, error)\n\tBlockByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*types.Block, error)\n\tStateAndHeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*state.StateDB, *types.Header, error)\n\tStateAndHeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*state.StateDB, *types.Header, error)\n\tPendingBlockAndReceipts() (*types.Block, types.Receipts)\n\tGetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error)\n\tGetTd(ctx context.Context, hash common.Hash) *big.Int\n\tGetEVM(ctx context.Context, msg *core.Message, state *state.StateDB, header *types.Header, vmConfig *vm.Config) (*vm.EVM, func() error, error)\n\tSubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription\n\tSubscribeChainHeadEvent(ch chan<- core.ChainHeadEvent) event.Subscription\n\tSubscribeChainSideEvent(ch chan<- core.ChainSideEvent) event.Subscription\n\tSendTx(ctx context.Context, signedTx *types.Transaction) error\n\tGetTransaction(ctx context.Context, txHash common.Hash) (*types.Transaction, common.Hash, uint64, uint64, error)\n\tGetPoolTransactions() (types.Transactions, error)\n\tGetPoolTransaction(txHash common.Hash) *types.Transaction\n\tGetPoolNonce(ctx context.Context, addr common.Address) (uint64, error)\n\tStats() (pending int, queued int)\n\tTxPoolContent() (map[common.Address]types.Transactions, map[common.Address]types.Transactions)\n\tTxPoolContentFrom(addr common.Address) (types.Transactions, types.Transactions)\n\tSubscribeNewTxsEvent(chan<- core.NewTxsEvent) event.Subscription\n\tChainConfig() *params.ChainConfig\n\tEngine() consensus.Engine\n\tGetBody(ctx context.Context, hash common.Hash, number rpc.BlockNumber) (*types.Body, error)\n\tGetLogs(ctx context.Context, blockHash common.Hash, number uint64) ([][]*types.Log, error)\n\tSubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription\n\tSubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription\n\tSubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription\n\tBloomStatus() (uint64, uint64)\n\tServiceFilter(ctx context.Context, session *bloombits.MatcherSession)\n}\n```\n\nIf readers want to customize some new RPC APIs, they can define functions in the /internal/ethapi.Backend interface and add specific implementations to EthAPIBackend\n\n### startNode\nThe last key function, `startNode()`, is to officially start an Ethereum execution layer node. It starts the Stack instance (Node) by calling the utils.StartNode() function which triggers the Node.Start() function. In the Node.Start() function, it traverses the backend instances registered in `Node.lifecycles` and starts them. In addition, in the startNode() function, the unlockAccounts() function is still called, and the unlocked wallet is registered in the stack, and the RPClient module that interacts with local Geth is created through the stack.Attach() function\n\nAt the end of the geth() function, the function executes `stack.Wait()`, so that the main thread enters the blocking state, and the services of other functional modules are distributed to other sub-coroutines for maintenance\n\n## Node\nAs we mentioned earlier, the Node type belongs to the top-level instance in the life cycle of geth, and it is responsible for being the administrator of the high-level abstract module communicating with the outside world, such as managing rpc server, http server, Web Socket, and P2P Server external interface . At the same time, Node maintains the back-end instances and services (lifecycles []Lifecycle) required for node operation, such as the Ethereum type we mentioned above that is responsible for the specific Service\n```go\ntype Node struct {\n\teventmux *event.TypeMux\n\tconfig *Config\n\taccman *accounts.Manager\n\tlog log.Logger\n\tkeyDir string // key store directory\n\tkeyDirTemp bool // If true, key directory will be removed by Stop\n\tdirLock *flock.Flock // prevents concurrent use of instance directory\n\tstop chan struct{} // Channel to wait for termination notifications\n\tserver *p2p.Server // Currently running P2P networking layer\n\tstartStopLock sync.Mutex // Start/Stop are protected by an additional lock\n\tstate int // Tracks state of node lifecycle\n\n\tlock sync.Mutex\n\tlifecycles []Lifecycle // All registered backends, services, and auxiliary services that have a lifecycle\n\trpcAPIs []rpc.API // List of APIs currently provided by the node\n\thttp *httpServer //\n\tws *httpServer //\n\thttpAuth *httpServer //\n\twsAuth *httpServer //\n\tipc *ipcServer // Stores information about the ipc http server\n\tinprocHandler *rpc.Server // In-process RPC request handler to process the API requests\n\n\tdatabases map[*closeTrackingDB]struct{} // All open databases\n}\n```\n\n### close node\nAs mentioned earlier, the main thread of the entire program is blocked because of calling `stack.Wait()`. We can see that a channel called `stop` is declared in the Node structure. Since this Channel has not been assigned a value, the main process of the entire geth enters the blocking state, and continues to execute other business coroutines concurrently\n```go\n// Wait blocks until the node is closed.\nfunc (n *Node) Wait() {\n <-n.stop\n}\n```\nWhen the Channel n.stop is assigned a value, the geth main function will stop the current blocking state and start to perform a series of corresponding resource release operations.\nIt is worth noting that in the current codebase of go-ethereum, the blocking state of the main process is not ended directly by assigning a value to the stop channel, but a more concise and rude way is used: call the close() function directly Close the Channel. We can find the related implementation in node.doClose(). close() is a native function of go language, used when closing Channel.\n```go\n// doClose releases resources acquired by New(), collecting errors.\nfunc (n *Node) doClose(errs []error) error {\n // Close databases. This needs the lock because it needs to\n // synchronize with OpenDatabase*.\n n.lock.Lock()\n n.state = closedState\n errs = append(errs, n.closeDatabases()...)\n n.lock.Unlock()\n\n if err := n.accman.Close(); err != nil {\n errs = append(errs, err)\n }\n if n.keyDirTemp {\n if err := os.RemoveAll(n.keyDir); err != nil {\n errs = append(errs, err)\n }\n }\n\n // Release instance directory lock.\n n.closeDataDir()\n\n // Unblock n.Wait.\n close(n.stop)\n\n // Report any errors that might have occurred.\n switch len(errs) {\n case 0:\n return nil\n case 1:\n return errs[0]\n default:\n return fmt.Errorf(\"%v\", errs)\n }\n}\n```\n\n## Ethereum Backend\nWe can find the definition of the Ethereum structure in eth/backend.go. The member variables and receiving methods contained in this structure implement all the functions and data structures required by an Ethereum full node. We can see in the following code definition that the Ethereum structure contains several core data structures such as TxPool, Blockchain, consensus.Engine, and miner as member variables.\n```go\ntype Ethereum struct {\n\tconfig *ethconfig.Config\n\n\t// Handlers\n\ttxPool *txpool.TxPool\n\tblockchain *core.BlockChain\n\thandler *handler\n\tethDialCandidates enode.Iterator\n\tsnapDialCandidates enode.Iterator\n\tmerger *consensus.Merger\n\n\t// DB interfaces\n\tchainDb ethdb.Database // Block chain database\n\n\teventMux *event.TypeMux\n\tengine consensus.Engine\n\taccountManager *accounts.Manager\n\n\tbloomRequests chan chan *bloombits.Retrieval // Channel receiving bloom data retrieval requests\n\tbloomIndexer *core.ChainIndexer // Bloom indexer operating during block imports\n\tcloseBloomHandler chan struct{}\n\n\tAPIBackend *EthAPIBackend\n\n\tminer *miner.Miner\n\tgasPrice *big.Int\n\tetherbase common.Address\n\n\tnetworkID uint64\n\tnetRPCService *ethapi.NetAPI\n\n\tp2pServer *p2p.Server\n\n\tlock sync.RWMutex // Protects the variadic fields (e.g. gas price and etherbase)\n\n\tshutdownTracker *shutdowncheck.ShutdownTracker // Tracks if and when the node has shutdown ungracefully\n}\n```\nNodes start and stop Mining by calling `Ethereum.StartMining()` and `Ethereum.StopMining()`. Setting the profit account of Mining is achieved by calling `Ethereum.SetEtherbase()`\nHere we pay extra attention to the member variable `handler`. The definition of `handler` is in `eth/handler.go`.\nFrom a macro point of view, the main workflow of a node needs to: 1. Obtain/synchronize Transaction and Block data from the network 2. Add the Block obtained from the network to the Blockchain. The handler is responsible for providing the function of synchronizing blocks and transaction data, for example, `downloader.Downloader` is responsible for synchronizing Block from the network, and `fetcher.TxFetcher` is responsible for synchronizing transactions from the network","slug":"geth/code_analysis/geth.0.get.start","published":1,"updated":"2023-11-05T04:21:13.733Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8dv0023qwsjbe4aeis9","content":"

build from source

1
2
3
git clone https://github.com/ethereum/go-ethereum.git
cd go-ethereum
make geth
\n\n

understanding geth config

geth config type is defined in /cmd/geth/config.go

\n
1
2
3
4
5
6
type gethConfig struct {
\tEth      ethconfig.Config
\tNode     node.Config
\tEthstats ethstatsConfig
\tMetrics  metrics.Config
}
\n
    \n
  • ethconfig (eth/ethconfig/config.go)
    contains configuration options for of the ETH and LES(light node) protocols, such as NetworkId, SyncMode, txpool.Config, database options
    • \n
  • nodeConfig (node/config.go)
    represents a small collection of configuration values to fine tune the P2P network layer of a protocol stack. These values can be further extended by all registered services. such as p2p.Config, DataDir, KeyStoreDir, HTTPHost, HTTPModules(eth,net,web3), WSHost
    • \n
  • metrics.Config (metrics/config.go)
    contains the configuration for the metric collection, such as InfluxDBEndpoint, etc
    • \n
  • ethstatsConfig
    only one URL entry
    • \n
\n

geth provides default config in the above files. user config file path is given by the below flag

\n
1
2
3
4
5
configFileFlag = &cli.StringFlag{
\t\tName:     "config",
\t\tUsage:    "TOML configuration file",
\t\tCategory: flags.EthCategory,
\t}
\n\n

The config file should be a .toml file. A convenient way to create a config file is to get Geth to create one for you and use it as a template. To do this, use the dumpconfig command, saving the result to a .toml file. Note that you also need to explicitly provide the network_id on the command line for the public testnets such as Sepolia or Geoerli:

\n
1
./geth --sepolia dumpconfig > geth-config.toml
\n

to specify path to config file

\n
1
geth --sepolia --config geth-config.toml
\n\n

key configs

    \n
  • [Eth].TxLookupLimit
    Number of recent blocks to maintain transactions index for (default = about one year, 0 = entire chain), default: 2350000
    • \n
  • [Node].BootstrapNodes
    used to establish connectivity with the rest of the network.
    geth provides default bootstrapNodes in file params/bootnodes.go
    • \n
  • [Metrics_AND_STATS].ethstats
    Reporting URL of a ethstats service (nodename:secret@host:port), more detail
    • \n
  • SyncMode
    • \n
  • TrieDirtyCache
    • \n
  • NoPruning
    • \n
  • TrieCleanCacheJournal e.g triecache
    • \n
\n

how geth starts

\"geth
the main func is in cmd/geth/main.go

\n
1
2
3
4
5
6
func main() {
\tif err := app.Run(os.Args); err != nil {
\t\tfmt.Fprintln(os.Stderr, err)
\t\tos.Exit(1)
\t}
}
\n

the main() function is very short, and its main function is to start a tool for parsing command line commands: gopkg.in/urfave/cli.v1. Going deeper, we will find that app.Action = geth will be called when the cli app is initialized to call the geth() function

\n
1
2
3
4
5
func init() {
\t// Initialize the CLI app and start Geth
\tapp.Action = geth
    // ....
}
\n

geth is the main entry point into the system if no special subcommand is run. It creates a default node based on the command line arguments and runs it in blocking mode, waiting for it to be shut down.

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
func geth(ctx *cli.Context) error {
\tif args := ctx.Args().Slice(); len(args) > 0 {
\t\treturn fmt.Errorf("invalid command: %q", args[0])
\t}

\tprepare(ctx)
\tstack, backend := makeFullNode(ctx)
\tdefer stack.Close()

\tstartNode(ctx, stack, backend, false)
\tstack.Wait()
\treturn nil
}
\n

In the geth() function, there are three important function calls, namely: prepare(), makeFullNode(), and startNode().

\n

prepare

The implementation of the prepare() function is in the current main.go file. It is mainly used to set some configurations required for node initialization.

\n

makeFullNode

The implementation of the makeFullNode() function is located in the cmd/geth/config.go file. It will load the context of the command and apply user given configuration; and generate instances of stack and backend. Among them, stack is an instance of Node type (Node is the top-level instance in the life cycle of geth. It is responsible for managing high-level abstractions such as P2P Server, Http Server, and Database in the node. The definition of the Node type is located in the node/node.go file), which is initialized by calling makeConfigNode() function through makeFullNode() function. inside makeFullNode, it calls node.New(&cfg.Node) to initiate a node. During instantiating of node, it invokes rpc.NewServer() to create a new rpc server and put in the field inprocHandler. it registers rpc api namespace by default.

\n

The backend here is an interface of ethapi.Backend type, which provides the basic functions needed to obtain the runtime of the Ethereum execution layer. Its definition is located in internal/ethapi/backend.go. Since there are many functions in this interface, we have selected some of the key functions as below for a glimpse of its functionality. backend is created by calling backend, eth := utils.RegisterEthService(stack, &cfg.Eth). Inside, it calls eth.New(stack, cfg) to create backend instance. During backend initiating, it opens database (chainDb, err := stack.OpenDatabaseWithFreezer("chaindata", config.DatabaseCache, config.DatabaseHandles, config.DatabaseFreezer, "eth/db/chaindata/", false)). Further, it creates consensus engine, engine := ethconfig.CreateConsensusEngine(stack, &ethashConfig, cliqueConfig, config.Miner.Notify, config.Miner.Noverify, chainDb). goerli testnet use POA consensus (clique).

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
type Backend interface {
\tSyncProgress() ethereum.SyncProgress
\tSuggestGasTipCap(ctx context.Context) (*big.Int, error)
\tChainDb() ethdb.Database
\tAccountManager() *accounts.Manager
\tExtRPCEnabled() bool
\tRPCGasCap() uint64            // global gas cap for eth_call over rpc: DoS protection
\tRPCEVMTimeout() time.Duration // global timeout for eth_call over rpc: DoS protection
\tRPCTxFeeCap() float64         // global tx fee cap for all transaction related APIs
\tUnprotectedAllowed() bool     // allows only for EIP155 transactions.
\tSetHead(number uint64)
\tHeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Header, error)
\tHeaderByHash(ctx context.Context, hash common.Hash) (*types.Header, error)
\tHeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*types.Header, error)
\tCurrentHeader() *types.Header
\tCurrentBlock() *types.Header
\tBlockByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Block, error)
\tBlockByHash(ctx context.Context, hash common.Hash) (*types.Block, error)
\tBlockByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*types.Block, error)
\tStateAndHeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*state.StateDB, *types.Header, error)
\tStateAndHeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*state.StateDB, *types.Header, error)
\tPendingBlockAndReceipts() (*types.Block, types.Receipts)
\tGetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error)
\tGetTd(ctx context.Context, hash common.Hash) *big.Int
\tGetEVM(ctx context.Context, msg *core.Message, state *state.StateDB, header *types.Header, vmConfig *vm.Config) (*vm.EVM, func() error, error)
\tSubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription
\tSubscribeChainHeadEvent(ch chan<- core.ChainHeadEvent) event.Subscription
\tSubscribeChainSideEvent(ch chan<- core.ChainSideEvent) event.Subscription
\tSendTx(ctx context.Context, signedTx *types.Transaction) error
\tGetTransaction(ctx context.Context, txHash common.Hash) (*types.Transaction, common.Hash, uint64, uint64, error)
\tGetPoolTransactions() (types.Transactions, error)
\tGetPoolTransaction(txHash common.Hash) *types.Transaction
\tGetPoolNonce(ctx context.Context, addr common.Address) (uint64, error)
\tStats() (pending int, queued int)
\tTxPoolContent() (map[common.Address]types.Transactions, map[common.Address]types.Transactions)
\tTxPoolContentFrom(addr common.Address) (types.Transactions, types.Transactions)
\tSubscribeNewTxsEvent(chan<- core.NewTxsEvent) event.Subscription
\tChainConfig() *params.ChainConfig
\tEngine() consensus.Engine
\tGetBody(ctx context.Context, hash common.Hash, number rpc.BlockNumber) (*types.Body, error)
\tGetLogs(ctx context.Context, blockHash common.Hash, number uint64) ([][]*types.Log, error)
\tSubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription
\tSubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription
\tSubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription
\tBloomStatus() (uint64, uint64)
\tServiceFilter(ctx context.Context, session *bloombits.MatcherSession)
}
\n\n

If readers want to customize some new RPC APIs, they can define functions in the /internal/ethapi.Backend interface and add specific implementations to EthAPIBackend

\n

startNode

The last key function, startNode(), is to officially start an Ethereum execution layer node. It starts the Stack instance (Node) by calling the utils.StartNode() function which triggers the Node.Start() function. In the Node.Start() function, it traverses the backend instances registered in Node.lifecycles and starts them. In addition, in the startNode() function, the unlockAccounts() function is still called, and the unlocked wallet is registered in the stack, and the RPClient module that interacts with local Geth is created through the stack.Attach() function

\n

At the end of the geth() function, the function executes stack.Wait(), so that the main thread enters the blocking state, and the services of other functional modules are distributed to other sub-coroutines for maintenance

\n

Node

As we mentioned earlier, the Node type belongs to the top-level instance in the life cycle of geth, and it is responsible for being the administrator of the high-level abstract module communicating with the outside world, such as managing rpc server, http server, Web Socket, and P2P Server external interface . At the same time, Node maintains the back-end instances and services (lifecycles []Lifecycle) required for node operation, such as the Ethereum type we mentioned above that is responsible for the specific Service

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
type Node struct {
\teventmux      *event.TypeMux
\tconfig        *Config
\taccman        *accounts.Manager
\tlog           log.Logger
\tkeyDir        string        // key store directory
\tkeyDirTemp    bool          // If true, key directory will be removed by Stop
\tdirLock       *flock.Flock  // prevents concurrent use of instance directory
\tstop          chan struct{} // Channel to wait for termination notifications
\tserver        *p2p.Server   // Currently running P2P networking layer
\tstartStopLock sync.Mutex    // Start/Stop are protected by an additional lock
\tstate         int           // Tracks state of node lifecycle

\tlock          sync.Mutex
\tlifecycles    []Lifecycle // All registered backends, services, and auxiliary services that have a lifecycle
\trpcAPIs       []rpc.API   // List of APIs currently provided by the node
\thttp          *httpServer //
\tws            *httpServer //
\thttpAuth      *httpServer //
\twsAuth        *httpServer //
\tipc           *ipcServer  // Stores information about the ipc http server
\tinprocHandler *rpc.Server // In-process RPC request handler to process the API requests

\tdatabases map[*closeTrackingDB]struct{} // All open databases
}
\n\n

close node

As mentioned earlier, the main thread of the entire program is blocked because of calling stack.Wait(). We can see that a channel called stop is declared in the Node structure. Since this Channel has not been assigned a value, the main process of the entire geth enters the blocking state, and continues to execute other business coroutines concurrently

\n
1
2
3
4
// Wait blocks until the node is closed.
func (n *Node) Wait() {
 <-n.stop
}
\n

When the Channel n.stop is assigned a value, the geth main function will stop the current blocking state and start to perform a series of corresponding resource release operations.
It is worth noting that in the current codebase of go-ethereum, the blocking state of the main process is not ended directly by assigning a value to the stop channel, but a more concise and rude way is used: call the close() function directly Close the Channel. We can find the related implementation in node.doClose(). close() is a native function of go language, used when closing Channel.

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
// doClose releases resources acquired by New(), collecting errors.
func (n *Node) doClose(errs []error) error {
 // Close databases. This needs the lock because it needs to
 // synchronize with OpenDatabase*.
 n.lock.Lock()
 n.state = closedState
 errs = append(errs, n.closeDatabases()...)
 n.lock.Unlock()

 if err := n.accman.Close(); err != nil {
  errs = append(errs, err)
 }
 if n.keyDirTemp {
  if err := os.RemoveAll(n.keyDir); err != nil {
   errs = append(errs, err)
  }
 }

 // Release instance directory lock.
 n.closeDataDir()

 // Unblock n.Wait.
 close(n.stop)

 // Report any errors that might have occurred.
 switch len(errs) {
 case 0:
  return nil
 case 1:
  return errs[0]
 default:
  return fmt.Errorf("%v", errs)
 }
}
\n\n

Ethereum Backend

We can find the definition of the Ethereum structure in eth/backend.go. The member variables and receiving methods contained in this structure implement all the functions and data structures required by an Ethereum full node. We can see in the following code definition that the Ethereum structure contains several core data structures such as TxPool, Blockchain, consensus.Engine, and miner as member variables.

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
type Ethereum struct {
\tconfig *ethconfig.Config

\t// Handlers
\ttxPool             *txpool.TxPool
\tblockchain         *core.BlockChain
\thandler            *handler
\tethDialCandidates  enode.Iterator
\tsnapDialCandidates enode.Iterator
\tmerger             *consensus.Merger

\t// DB interfaces
\tchainDb ethdb.Database // Block chain database

\teventMux       *event.TypeMux
\tengine         consensus.Engine
\taccountManager *accounts.Manager

\tbloomRequests     chan chan *bloombits.Retrieval // Channel receiving bloom data retrieval requests
\tbloomIndexer      *core.ChainIndexer             // Bloom indexer operating during block imports
\tcloseBloomHandler chan struct{}

\tAPIBackend *EthAPIBackend

\tminer     *miner.Miner
\tgasPrice  *big.Int
\tetherbase common.Address

\tnetworkID     uint64
\tnetRPCService *ethapi.NetAPI

\tp2pServer *p2p.Server

\tlock sync.RWMutex // Protects the variadic fields (e.g. gas price and etherbase)

\tshutdownTracker *shutdowncheck.ShutdownTracker // Tracks if and when the node has shutdown ungracefully
}
\n

Nodes start and stop Mining by calling Ethereum.StartMining() and Ethereum.StopMining(). Setting the profit account of Mining is achieved by calling Ethereum.SetEtherbase()
Here we pay extra attention to the member variable handler. The definition of handler is in eth/handler.go.
From a macro point of view, the main workflow of a node needs to: 1. Obtain/synchronize Transaction and Block data from the network 2. Add the Block obtained from the network to the Blockchain. The handler is responsible for providing the function of synchronizing blocks and transaction data, for example, downloader.Downloader is responsible for synchronizing Block from the network, and fetcher.TxFetcher is responsible for synchronizing transactions from the network

\n","site":{"data":{}},"excerpt":"","more":"

build from source

1
2
3
git clone https://github.com/ethereum/go-ethereum.git
cd go-ethereum
make geth
\n\n

understanding geth config

geth config type is defined in /cmd/geth/config.go

\n
1
2
3
4
5
6
type gethConfig struct {
\tEth      ethconfig.Config
\tNode     node.Config
\tEthstats ethstatsConfig
\tMetrics  metrics.Config
}
\n
    \n
  • ethconfig (eth/ethconfig/config.go)
    contains configuration options for of the ETH and LES(light node) protocols, such as NetworkId, SyncMode, txpool.Config, database options
    • \n
  • nodeConfig (node/config.go)
    represents a small collection of configuration values to fine tune the P2P network layer of a protocol stack. These values can be further extended by all registered services. such as p2p.Config, DataDir, KeyStoreDir, HTTPHost, HTTPModules(eth,net,web3), WSHost
    • \n
  • metrics.Config (metrics/config.go)
    contains the configuration for the metric collection, such as InfluxDBEndpoint, etc
    • \n
  • ethstatsConfig
    only one URL entry
    • \n
\n

geth provides default config in the above files. user config file path is given by the below flag

\n
1
2
3
4
5
configFileFlag = &cli.StringFlag{
\t\tName:     "config",
\t\tUsage:    "TOML configuration file",
\t\tCategory: flags.EthCategory,
\t}
\n\n

The config file should be a .toml file. A convenient way to create a config file is to get Geth to create one for you and use it as a template. To do this, use the dumpconfig command, saving the result to a .toml file. Note that you also need to explicitly provide the network_id on the command line for the public testnets such as Sepolia or Geoerli:

\n
1
./geth --sepolia dumpconfig > geth-config.toml
\n

to specify path to config file

\n
1
geth --sepolia --config geth-config.toml
\n\n

key configs

    \n
  • [Eth].TxLookupLimit
    Number of recent blocks to maintain transactions index for (default = about one year, 0 = entire chain), default: 2350000
    • \n
  • [Node].BootstrapNodes
    used to establish connectivity with the rest of the network.
    geth provides default bootstrapNodes in file params/bootnodes.go
    • \n
  • [Metrics_AND_STATS].ethstats
    Reporting URL of a ethstats service (nodename:secret@host:port), more detail
    • \n
  • SyncMode
    • \n
  • TrieDirtyCache
    • \n
  • NoPruning
    • \n
  • TrieCleanCacheJournal e.g triecache
    • \n
\n

how geth starts

\"geth
the main func is in cmd/geth/main.go

\n
1
2
3
4
5
6
func main() {
\tif err := app.Run(os.Args); err != nil {
\t\tfmt.Fprintln(os.Stderr, err)
\t\tos.Exit(1)
\t}
}
\n

the main() function is very short, and its main function is to start a tool for parsing command line commands: gopkg.in/urfave/cli.v1. Going deeper, we will find that app.Action = geth will be called when the cli app is initialized to call the geth() function

\n
1
2
3
4
5
func init() {
\t// Initialize the CLI app and start Geth
\tapp.Action = geth
    // ....
}
\n

geth is the main entry point into the system if no special subcommand is run. It creates a default node based on the command line arguments and runs it in blocking mode, waiting for it to be shut down.

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
func geth(ctx *cli.Context) error {
\tif args := ctx.Args().Slice(); len(args) > 0 {
\t\treturn fmt.Errorf("invalid command: %q", args[0])
\t}

\tprepare(ctx)
\tstack, backend := makeFullNode(ctx)
\tdefer stack.Close()

\tstartNode(ctx, stack, backend, false)
\tstack.Wait()
\treturn nil
}
\n

In the geth() function, there are three important function calls, namely: prepare(), makeFullNode(), and startNode().

\n

prepare

The implementation of the prepare() function is in the current main.go file. It is mainly used to set some configurations required for node initialization.

\n

makeFullNode

The implementation of the makeFullNode() function is located in the cmd/geth/config.go file. It will load the context of the command and apply user given configuration; and generate instances of stack and backend. Among them, stack is an instance of Node type (Node is the top-level instance in the life cycle of geth. It is responsible for managing high-level abstractions such as P2P Server, Http Server, and Database in the node. The definition of the Node type is located in the node/node.go file), which is initialized by calling makeConfigNode() function through makeFullNode() function. inside makeFullNode, it calls node.New(&cfg.Node) to initiate a node. During instantiating of node, it invokes rpc.NewServer() to create a new rpc server and put in the field inprocHandler. it registers rpc api namespace by default.

\n

The backend here is an interface of ethapi.Backend type, which provides the basic functions needed to obtain the runtime of the Ethereum execution layer. Its definition is located in internal/ethapi/backend.go. Since there are many functions in this interface, we have selected some of the key functions as below for a glimpse of its functionality. backend is created by calling backend, eth := utils.RegisterEthService(stack, &cfg.Eth). Inside, it calls eth.New(stack, cfg) to create backend instance. During backend initiating, it opens database (chainDb, err := stack.OpenDatabaseWithFreezer("chaindata", config.DatabaseCache, config.DatabaseHandles, config.DatabaseFreezer, "eth/db/chaindata/", false)). Further, it creates consensus engine, engine := ethconfig.CreateConsensusEngine(stack, &ethashConfig, cliqueConfig, config.Miner.Notify, config.Miner.Noverify, chainDb). goerli testnet use POA consensus (clique).

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
type Backend interface {
\tSyncProgress() ethereum.SyncProgress
\tSuggestGasTipCap(ctx context.Context) (*big.Int, error)
\tChainDb() ethdb.Database
\tAccountManager() *accounts.Manager
\tExtRPCEnabled() bool
\tRPCGasCap() uint64            // global gas cap for eth_call over rpc: DoS protection
\tRPCEVMTimeout() time.Duration // global timeout for eth_call over rpc: DoS protection
\tRPCTxFeeCap() float64         // global tx fee cap for all transaction related APIs
\tUnprotectedAllowed() bool     // allows only for EIP155 transactions.
\tSetHead(number uint64)
\tHeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Header, error)
\tHeaderByHash(ctx context.Context, hash common.Hash) (*types.Header, error)
\tHeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*types.Header, error)
\tCurrentHeader() *types.Header
\tCurrentBlock() *types.Header
\tBlockByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Block, error)
\tBlockByHash(ctx context.Context, hash common.Hash) (*types.Block, error)
\tBlockByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*types.Block, error)
\tStateAndHeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*state.StateDB, *types.Header, error)
\tStateAndHeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*state.StateDB, *types.Header, error)
\tPendingBlockAndReceipts() (*types.Block, types.Receipts)
\tGetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error)
\tGetTd(ctx context.Context, hash common.Hash) *big.Int
\tGetEVM(ctx context.Context, msg *core.Message, state *state.StateDB, header *types.Header, vmConfig *vm.Config) (*vm.EVM, func() error, error)
\tSubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription
\tSubscribeChainHeadEvent(ch chan<- core.ChainHeadEvent) event.Subscription
\tSubscribeChainSideEvent(ch chan<- core.ChainSideEvent) event.Subscription
\tSendTx(ctx context.Context, signedTx *types.Transaction) error
\tGetTransaction(ctx context.Context, txHash common.Hash) (*types.Transaction, common.Hash, uint64, uint64, error)
\tGetPoolTransactions() (types.Transactions, error)
\tGetPoolTransaction(txHash common.Hash) *types.Transaction
\tGetPoolNonce(ctx context.Context, addr common.Address) (uint64, error)
\tStats() (pending int, queued int)
\tTxPoolContent() (map[common.Address]types.Transactions, map[common.Address]types.Transactions)
\tTxPoolContentFrom(addr common.Address) (types.Transactions, types.Transactions)
\tSubscribeNewTxsEvent(chan<- core.NewTxsEvent) event.Subscription
\tChainConfig() *params.ChainConfig
\tEngine() consensus.Engine
\tGetBody(ctx context.Context, hash common.Hash, number rpc.BlockNumber) (*types.Body, error)
\tGetLogs(ctx context.Context, blockHash common.Hash, number uint64) ([][]*types.Log, error)
\tSubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription
\tSubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription
\tSubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription
\tBloomStatus() (uint64, uint64)
\tServiceFilter(ctx context.Context, session *bloombits.MatcherSession)
}
\n\n

If readers want to customize some new RPC APIs, they can define functions in the /internal/ethapi.Backend interface and add specific implementations to EthAPIBackend

\n

startNode

The last key function, startNode(), is to officially start an Ethereum execution layer node. It starts the Stack instance (Node) by calling the utils.StartNode() function which triggers the Node.Start() function. In the Node.Start() function, it traverses the backend instances registered in Node.lifecycles and starts them. In addition, in the startNode() function, the unlockAccounts() function is still called, and the unlocked wallet is registered in the stack, and the RPClient module that interacts with local Geth is created through the stack.Attach() function

\n

At the end of the geth() function, the function executes stack.Wait(), so that the main thread enters the blocking state, and the services of other functional modules are distributed to other sub-coroutines for maintenance

\n

Node

As we mentioned earlier, the Node type belongs to the top-level instance in the life cycle of geth, and it is responsible for being the administrator of the high-level abstract module communicating with the outside world, such as managing rpc server, http server, Web Socket, and P2P Server external interface . At the same time, Node maintains the back-end instances and services (lifecycles []Lifecycle) required for node operation, such as the Ethereum type we mentioned above that is responsible for the specific Service

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
type Node struct {
\teventmux      *event.TypeMux
\tconfig        *Config
\taccman        *accounts.Manager
\tlog           log.Logger
\tkeyDir        string        // key store directory
\tkeyDirTemp    bool          // If true, key directory will be removed by Stop
\tdirLock       *flock.Flock  // prevents concurrent use of instance directory
\tstop          chan struct{} // Channel to wait for termination notifications
\tserver        *p2p.Server   // Currently running P2P networking layer
\tstartStopLock sync.Mutex    // Start/Stop are protected by an additional lock
\tstate         int           // Tracks state of node lifecycle

\tlock          sync.Mutex
\tlifecycles    []Lifecycle // All registered backends, services, and auxiliary services that have a lifecycle
\trpcAPIs       []rpc.API   // List of APIs currently provided by the node
\thttp          *httpServer //
\tws            *httpServer //
\thttpAuth      *httpServer //
\twsAuth        *httpServer //
\tipc           *ipcServer  // Stores information about the ipc http server
\tinprocHandler *rpc.Server // In-process RPC request handler to process the API requests

\tdatabases map[*closeTrackingDB]struct{} // All open databases
}
\n\n

close node

As mentioned earlier, the main thread of the entire program is blocked because of calling stack.Wait(). We can see that a channel called stop is declared in the Node structure. Since this Channel has not been assigned a value, the main process of the entire geth enters the blocking state, and continues to execute other business coroutines concurrently

\n
1
2
3
4
// Wait blocks until the node is closed.
func (n *Node) Wait() {
 <-n.stop
}
\n

When the Channel n.stop is assigned a value, the geth main function will stop the current blocking state and start to perform a series of corresponding resource release operations.
It is worth noting that in the current codebase of go-ethereum, the blocking state of the main process is not ended directly by assigning a value to the stop channel, but a more concise and rude way is used: call the close() function directly Close the Channel. We can find the related implementation in node.doClose(). close() is a native function of go language, used when closing Channel.

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
// doClose releases resources acquired by New(), collecting errors.
func (n *Node) doClose(errs []error) error {
 // Close databases. This needs the lock because it needs to
 // synchronize with OpenDatabase*.
 n.lock.Lock()
 n.state = closedState
 errs = append(errs, n.closeDatabases()...)
 n.lock.Unlock()

 if err := n.accman.Close(); err != nil {
  errs = append(errs, err)
 }
 if n.keyDirTemp {
  if err := os.RemoveAll(n.keyDir); err != nil {
   errs = append(errs, err)
  }
 }

 // Release instance directory lock.
 n.closeDataDir()

 // Unblock n.Wait.
 close(n.stop)

 // Report any errors that might have occurred.
 switch len(errs) {
 case 0:
  return nil
 case 1:
  return errs[0]
 default:
  return fmt.Errorf("%v", errs)
 }
}
\n\n

Ethereum Backend

We can find the definition of the Ethereum structure in eth/backend.go. The member variables and receiving methods contained in this structure implement all the functions and data structures required by an Ethereum full node. We can see in the following code definition that the Ethereum structure contains several core data structures such as TxPool, Blockchain, consensus.Engine, and miner as member variables.

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
type Ethereum struct {
\tconfig *ethconfig.Config

\t// Handlers
\ttxPool             *txpool.TxPool
\tblockchain         *core.BlockChain
\thandler            *handler
\tethDialCandidates  enode.Iterator
\tsnapDialCandidates enode.Iterator
\tmerger             *consensus.Merger

\t// DB interfaces
\tchainDb ethdb.Database // Block chain database

\teventMux       *event.TypeMux
\tengine         consensus.Engine
\taccountManager *accounts.Manager

\tbloomRequests     chan chan *bloombits.Retrieval // Channel receiving bloom data retrieval requests
\tbloomIndexer      *core.ChainIndexer             // Bloom indexer operating during block imports
\tcloseBloomHandler chan struct{}

\tAPIBackend *EthAPIBackend

\tminer     *miner.Miner
\tgasPrice  *big.Int
\tetherbase common.Address

\tnetworkID     uint64
\tnetRPCService *ethapi.NetAPI

\tp2pServer *p2p.Server

\tlock sync.RWMutex // Protects the variadic fields (e.g. gas price and etherbase)

\tshutdownTracker *shutdowncheck.ShutdownTracker // Tracks if and when the node has shutdown ungracefully
}
\n

Nodes start and stop Mining by calling Ethereum.StartMining() and Ethereum.StopMining(). Setting the profit account of Mining is achieved by calling Ethereum.SetEtherbase()
Here we pay extra attention to the member variable handler. The definition of handler is in eth/handler.go.
From a macro point of view, the main workflow of a node needs to: 1. Obtain/synchronize Transaction and Block data from the network 2. Add the Block obtained from the network to the Blockchain. The handler is responsible for providing the function of synchronizing blocks and transaction data, for example, downloader.Downloader is responsible for synchronizing Block from the network, and fetcher.TxFetcher is responsible for synchronizing transactions from the network

\n"},{"title":"rpc","date":"2022-11-08T06:23:08.000Z","_content":"\n\n## overview\npackage rpc implements bi-directional JSON-RPC 2.0 on multiple transports (http, ws, ipc). After creating a server or client instance, objects can be registered to make them visible as 'services'. Exported methods that follow specific conventions can be called remotely. It also has support for the publish/subscribe pattern.\n\n## methods\n### rpc endpoints (callback)\nMethods that satisfy the following criteria are made available for remote access:\n - method must be exported\n - method returns 0, 1 (response or error) or 2 (response and error) values\n\nThe server offers the ServeCodec method which accepts a ServerCodec instance. It will read requests from the codec, process the request and sends the response back to the client using the codec. The server can execute requests concurrently. Responses can be sent back to the client out of order.\n\nAn example server which uses the JSON codec:\n```go\ntype CalculatorService struct {}\n\nfunc (s *CalculatorService) Add(a, b int) int {\n return a + b\n}\n\nfunc (s *CalculatorService) Div(a, b int) (int, error) {\n if b == 0 {\n return 0, errors.New(\"divide by zero\")\n }\n return a/b, nil\n}\n\ncalculator := new(CalculatorService)\nserver := NewServer()\nserver.RegisterName(\"calculator\", calculator)\nl, _ := net.ListenUnix(\"unix\", &net.UnixAddr{Net: \"unix\", Name: \"/tmp/calculator.sock\"})\nserver.ServeListener(l)\n```\n\n### subscriptions\nThe package also supports the publish subscribe pattern through the use of subscriptions.\nA method that is considered eligible for notifications must satisfy the following\ncriteria:\n - method must be exported\n - first method argument type must be context.Context\n - method must have return types (rpc.Subscription, error)\n\nAn example method:\n```go\nfunc (s *BlockChainService) NewBlocks(ctx context.Context) (rpc.Subscription, error) {\n\t\t...\n\t}\n```\n\n### Reverse Calls\nIn any method handler, an instance of rpc.Client can be accessed through the `ClientFromContext` method. Using this client instance, server-to-client method calls can be performed on the RPC connection.\n\n## server\nto start rpc service, the invoking chain is as below\n```\nnode/node.go[func (n *Node) Start()] -> node/node.go[func (n *Node) openEndpoints()] -> node/node.go[func (n *Node) startRPC()]\n```\n\n### API registration\n","source":"_posts/geth/code_analysis/geth.1.rpc.md","raw":"---\ntitle: rpc\ndate: 2022-11-08 14:23:08\ntags: [blockchain, geth]\n---\n\n\n## overview\npackage rpc implements bi-directional JSON-RPC 2.0 on multiple transports (http, ws, ipc). After creating a server or client instance, objects can be registered to make them visible as 'services'. Exported methods that follow specific conventions can be called remotely. It also has support for the publish/subscribe pattern.\n\n## methods\n### rpc endpoints (callback)\nMethods that satisfy the following criteria are made available for remote access:\n - method must be exported\n - method returns 0, 1 (response or error) or 2 (response and error) values\n\nThe server offers the ServeCodec method which accepts a ServerCodec instance. It will read requests from the codec, process the request and sends the response back to the client using the codec. The server can execute requests concurrently. Responses can be sent back to the client out of order.\n\nAn example server which uses the JSON codec:\n```go\ntype CalculatorService struct {}\n\nfunc (s *CalculatorService) Add(a, b int) int {\n return a + b\n}\n\nfunc (s *CalculatorService) Div(a, b int) (int, error) {\n if b == 0 {\n return 0, errors.New(\"divide by zero\")\n }\n return a/b, nil\n}\n\ncalculator := new(CalculatorService)\nserver := NewServer()\nserver.RegisterName(\"calculator\", calculator)\nl, _ := net.ListenUnix(\"unix\", &net.UnixAddr{Net: \"unix\", Name: \"/tmp/calculator.sock\"})\nserver.ServeListener(l)\n```\n\n### subscriptions\nThe package also supports the publish subscribe pattern through the use of subscriptions.\nA method that is considered eligible for notifications must satisfy the following\ncriteria:\n - method must be exported\n - first method argument type must be context.Context\n - method must have return types (rpc.Subscription, error)\n\nAn example method:\n```go\nfunc (s *BlockChainService) NewBlocks(ctx context.Context) (rpc.Subscription, error) {\n\t\t...\n\t}\n```\n\n### Reverse Calls\nIn any method handler, an instance of rpc.Client can be accessed through the `ClientFromContext` method. Using this client instance, server-to-client method calls can be performed on the RPC connection.\n\n## server\nto start rpc service, the invoking chain is as below\n```\nnode/node.go[func (n *Node) Start()] -> node/node.go[func (n *Node) openEndpoints()] -> node/node.go[func (n *Node) startRPC()]\n```\n\n### API registration\n","slug":"geth/code_analysis/geth.1.rpc","published":1,"updated":"2023-11-05T04:21:13.733Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8dv0025qwsjg5vzbwcu","content":"

overview

package rpc implements bi-directional JSON-RPC 2.0 on multiple transports (http, ws, ipc). After creating a server or client instance, objects can be registered to make them visible as ‘services’. Exported methods that follow specific conventions can be called remotely. It also has support for the publish/subscribe pattern.

\n

methods

rpc endpoints (callback)

Methods that satisfy the following criteria are made available for remote access:

\n
    \n
  • method must be exported
    • \n
  • method returns 0, 1 (response or error) or 2 (response and error) values
    • \n
\n

The server offers the ServeCodec method which accepts a ServerCodec instance. It will read requests from the codec, process the request and sends the response back to the client using the codec. The server can execute requests concurrently. Responses can be sent back to the client out of order.

\n

An example server which uses the JSON codec:

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
type CalculatorService struct {}

func (s *CalculatorService) Add(a, b int) int {
    return a + b
}

func (s *CalculatorService) Div(a, b int) (int, error) {
    if b == 0 {
        return 0, errors.New("divide by zero")
    }
    return a/b, nil
}

calculator := new(CalculatorService)
server := NewServer()
server.RegisterName("calculator", calculator)
l, _ := net.ListenUnix("unix", &net.UnixAddr{Net: "unix", Name: "/tmp/calculator.sock"})
server.ServeListener(l)
\n\n

subscriptions

The package also supports the publish subscribe pattern through the use of subscriptions.
A method that is considered eligible for notifications must satisfy the following
criteria:

\n
    \n
  • method must be exported
    • \n
  • first method argument type must be context.Context
    • \n
  • method must have return types (rpc.Subscription, error)
    • \n
\n

An example method:

\n
1
2
3
func (s *BlockChainService) NewBlocks(ctx context.Context) (rpc.Subscription, error) {
\t\t...
\t}
\n\n

Reverse Calls

In any method handler, an instance of rpc.Client can be accessed through the ClientFromContext method. Using this client instance, server-to-client method calls can be performed on the RPC connection.

\n

server

to start rpc service, the invoking chain is as below

\n
1
node/node.go[func (n *Node) Start()] -> node/node.go[func (n *Node) openEndpoints()] -> node/node.go[func (n *Node) startRPC()]
\n\n

API registration

","site":{"data":{}},"excerpt":"","more":"

overview

package rpc implements bi-directional JSON-RPC 2.0 on multiple transports (http, ws, ipc). After creating a server or client instance, objects can be registered to make them visible as ‘services’. Exported methods that follow specific conventions can be called remotely. It also has support for the publish/subscribe pattern.

\n

methods

rpc endpoints (callback)

Methods that satisfy the following criteria are made available for remote access:

\n
    \n
  • method must be exported
    • \n
  • method returns 0, 1 (response or error) or 2 (response and error) values
    • \n
\n

The server offers the ServeCodec method which accepts a ServerCodec instance. It will read requests from the codec, process the request and sends the response back to the client using the codec. The server can execute requests concurrently. Responses can be sent back to the client out of order.

\n

An example server which uses the JSON codec:

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
type CalculatorService struct {}

func (s *CalculatorService) Add(a, b int) int {
    return a + b
}

func (s *CalculatorService) Div(a, b int) (int, error) {
    if b == 0 {
        return 0, errors.New("divide by zero")
    }
    return a/b, nil
}

calculator := new(CalculatorService)
server := NewServer()
server.RegisterName("calculator", calculator)
l, _ := net.ListenUnix("unix", &net.UnixAddr{Net: "unix", Name: "/tmp/calculator.sock"})
server.ServeListener(l)
\n\n

subscriptions

The package also supports the publish subscribe pattern through the use of subscriptions.
A method that is considered eligible for notifications must satisfy the following
criteria:

\n
    \n
  • method must be exported
    • \n
  • first method argument type must be context.Context
    • \n
  • method must have return types (rpc.Subscription, error)
    • \n
\n

An example method:

\n
1
2
3
func (s *BlockChainService) NewBlocks(ctx context.Context) (rpc.Subscription, error) {
\t\t...
\t}
\n\n

Reverse Calls

In any method handler, an instance of rpc.Client can be accessed through the ClientFromContext method. Using this client instance, server-to-client method calls can be performed on the RPC connection.

\n

server

to start rpc service, the invoking chain is as below

\n
1
node/node.go[func (n *Node) Start()] -> node/node.go[func (n *Node) openEndpoints()] -> node/node.go[func (n *Node) startRPC()]
\n\n

API registration

"},{"title":"geth evm source analysis","date":"2023-01-08T08:24:54.000Z","_content":"\n# overall\nthe code is under path `core/vm`\noverview of the whole evm module ![evm](/images/evm.drawio.google.png)\n\nthe core is `EVM` struct (in evm.go), with main function in creating or call contract. a new `EVM` object is created every time when processing a transaction. inside the EVM struct, the main items are `Interpreter`, and `StateDB` (for state persistence). `Interpreter` loops through contract call instructions.Before each instruction is executed, some checks are performed to ensure sufficient gas and stack space. actual instruction execution code is recorded in `JumpTable` (256 sized array of `operation`)\n\ndepending on the version of Ethereum, JumpTable may point to four different instruction sets: constantinopleInstructionSet, byzantiumInstructionSet, homesteadInstructionSet, frontierInstructionSet. Most of the instructions of these four sets of instruction sets are the same, but as the version is updated, the new version supports more instruction sets than the old version.\n\n# evm\nThe `EVM` object is the most important object exported by the evm module, which represents an Ethereum virtual machine\n\n## creating evm\nEvery time a transaction is processed, an EVM is created to execute the transaction. This is reflected in the function `ApplyTransaction` (core/state_processor.go)\n\n## creating contract\nIf the `to` of the transaction is empty, it means that this transaction is to create a contract, so call `EVM.Create` to perform related functions\n- CREATE\n```\ncontractAddr = crypto.CreateAddress(caller.Address(), evm.StateDB.GetNonce(caller.Address()))\n```\n- CREATE2\n```\ncodeAndHash := &codeAndHash{code: code}\n\tcontractAddr = crypto.CreateAddress2(caller.Address(), salt.Bytes32(), codeAndHash.Hash().Bytes())\n```\nduring create contract, an object `Contract` is created. A Contract object contains and maintains the necessary information during the execution of the contract, such as the contract creator, the address of the contract itself, the remaining gas of the contract, the contract code and the `jumpdests` record of the code.\n\nthen, it invokes below method to create contract\n```\nret, err := evm.interpreter.Run(contract, nil, false)\nevm.StateDB.SetCode(address, ret)\n```\nIf the operation is successful and the contract code does not exceed the length limit, call StateDB.SetCode to store the contract code in the contract account of the Ethereum state database. Of course, the storage needs to consume a certain amount of gas.\n\nYou may wonder why the stored contract code is the return code after the contract runs, not the data in the original transaction (ie Transaction.data.Payload). This is because when the contract source code is compiled into binary data, in addition to the original code of the contract, the compiler also inserts some codes to perform related functions. For creation, the compiler inserts code that executes the contract's \"constructor\" (that is, the contract object's constructor method). Therefore, when the binary compiled by the compiler is submitted to the Ethereum node to create a contract, the EVM executes this binary code, in fact, it mainly executes the constructor method of the contract, and then returns other codes of the contract, so there is a `ret` variable here Stored in the state database as the actual code of the contract\n\n## call contract\nThe EVM object has three methods to implement the call of the contract, they are:\n\n- EVM. Call\n- EVM. CallCode\n- EVM. DelegateCall\n- EVM.StaticCall\nThe basic contract call function implemented by EVM.Call is nothing special. The following three calling methods are the differences compared with EVM.Call. So here we only introduce the particularity of the last three calling methods\n\n### EVM.CallCode & EVM.DelegateCall\nThe existence of EVM.CallCode and EVM.DelegateCall is to realize the characteristics of the \"library\" of the contract. If the code written by solidity is to be called as a library, it must be deployed on the blockchain to obtain a fixed address like a normal contract. , other contracts can call the method provided by this \"library contract\". But the contract also involves some unique attributes, such as the caller of the contract, contract address, the amount of ether it owns, etc. If we directly call the code of the \"library contract\", these properties must be the properties of the \"library contract\" itself, but this may not be what we want\n\nas an example\n```\nA -> contractB - delegateCall -> libC\n```\n`EVM.DelegateCall` sets the caller (msg.sender) of the \"library contract\" (libC) to A, rather than contractB; sets the address of the \"library contract\" (libC) to contractB. \n`EVM.CallCode` is similar to `EVM.DelegateCall`. the only difference is that `EVM.CallCode` only change the address of the \"library contract\" (libC) to contractB, without chanding the caller to A.\n`EVM.StaticCall` is similar to `EVM.Call`, the only difference is that EVM.StaticCall does not allow execution of instructions that modify permanently stored data\n\nduring contract call, it first check whether it is precompiled contract. some precompiled contracts are\n- common.BytesToAddress([]byte{1}): &ecrecover{},\n- common.BytesToAddress([]byte{2}): &sha256hash{},\n- common.BytesToAddress([]byte{3}): &ripemd160hash{},\n- common.BytesToAddress([]byte{4}): &dataCopy{},\n\n# EVMInterpreter\nThe interpreter object EVMInterpreter is used to interpret and execute specified contract instructions. However, note that the actual instruction interpretation and execution is not really completed by the interpreter object, but by the operation object JumpTable. The interpreter object is only responsible for parsing instruction codes one by one, and then obtains the corresponding operation object, and check objects such as the stack before calling the operation.execute function that actually executre the instruction. It can also be said that the interpreter object is only responsible for the scheduling of interpretation.\n\n## execution layout\n![layout](/images/evm.layout.png)\n\n## intrinsic gas\nThe intrinsic gas for a transaction is the amount of gas that the transaction uses before any code runs. It is a constant transaction fee (currently 21000 gas) plus a fee for every byte of data supplied with the transaction (4 gas for a zero byte, 68 gas for non-zeros). These constants are all currently defined for geth in params/protocol_params.go.\n\n## gas cost\nthe gas cost of each instruction is stored in `JumpTable.operation.dynamicGas` or `JumpTable.operation.constantGas`. constantGas means the operation gas cost is a fixed constant. dynamicGas is a function which will return gas during runtime.\n\nIn fact, not only the interpretation and execution of the instruction itself consumes gas, but also consumes gas when using memory storage and StateDB permanent storage. For most instructions, the latter two are not used (memory & storage), but for some instructions (such as CODECOPY or SSTORE), their gasCost function will take memory and StateDB usage into account.\n\na method `memoryGasCost`is used to calculate the gas consumption of memory usage. only when the required space size exceeds the current space size, the excess part needs to consume gas.\n\n# JumpTable\njumptable is 256 sized array of `operation`\n\n## jump instruction\nAmong the instructions of the contract, there are two jump instructions (excluding CALL): JUMP and JUMPI. Their special feature is that the first instruction of the target address after the jump must be JUMPDEST\n```\nfunc opJump(pc *uint64, interpreter *EVMInterpreter, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) {\n pos := stack.pop()\n if !contract.validJumpdest(pos) {\n nop := contract.GetOp(pos.Uint64())\n return nil, fmt.Errorf(\"invalid jump destination (%v) %v\", nop, pos)\n }\n *pc = pos.Uint64()\n\n interpreter.intPool.put(pos)\n return nil, nil\n}\n```\nA function interprets and executes the JUMP instruction. The code first fetches a value from the stack as the jump destination. This value is actually an offset relative to field 0 of the contract code. Then the code will call Contract.validJumpdest to determine whether the first instruction of this destination is JUMPDEST, if it is not, an error will occur.\n\nTo judge whether the first instruction of the destination is JUMPDEST, two points must be guaranteed: first, its value is the value of the opcode of the JUMPDEST instruction; second, it is an instruction, not ordinary data.\n\nLet's introduce how Contract.validJumpdest works. In addition to comparing opcode (this is very simple), Contract will also create a bit vector object (ie bitvec, bit vector). This object will analyze the contract instructions from the beginning to the end. If the byte at a certain offset of the contract belongs to ordinary data, the \"bit\" corresponding to the offset value in bitvec is set to 1, and if it is an instruction, it is set to 0. In Contract.validJumpdest, it is judged whether this is a normal instruction by checking whether the \"bit\" of the offset value of the jump destination in this bit vector object is 0\n\n# references\n- [yangzhe_blog](https://yangzhe.me/2019/08/12/ethereum-evm/#%E8%A7%A3%E9%87%8A%E5%99%A8%E5%AF%B9%E8%B1%A1evminterpreter)\n- [op code manual](https://www.evm.codes/?fork=shanghai)","source":"_posts/geth/code_analysis/geth.evm.md","raw":"---\ntitle: geth evm source analysis\ndate: 2023-01-08 16:24:54\ntags: [blockchain,geth]\n---\n\n# overall\nthe code is under path `core/vm`\noverview of the whole evm module ![evm](/images/evm.drawio.google.png)\n\nthe core is `EVM` struct (in evm.go), with main function in creating or call contract. a new `EVM` object is created every time when processing a transaction. inside the EVM struct, the main items are `Interpreter`, and `StateDB` (for state persistence). `Interpreter` loops through contract call instructions.Before each instruction is executed, some checks are performed to ensure sufficient gas and stack space. actual instruction execution code is recorded in `JumpTable` (256 sized array of `operation`)\n\ndepending on the version of Ethereum, JumpTable may point to four different instruction sets: constantinopleInstructionSet, byzantiumInstructionSet, homesteadInstructionSet, frontierInstructionSet. Most of the instructions of these four sets of instruction sets are the same, but as the version is updated, the new version supports more instruction sets than the old version.\n\n# evm\nThe `EVM` object is the most important object exported by the evm module, which represents an Ethereum virtual machine\n\n## creating evm\nEvery time a transaction is processed, an EVM is created to execute the transaction. This is reflected in the function `ApplyTransaction` (core/state_processor.go)\n\n## creating contract\nIf the `to` of the transaction is empty, it means that this transaction is to create a contract, so call `EVM.Create` to perform related functions\n- CREATE\n```\ncontractAddr = crypto.CreateAddress(caller.Address(), evm.StateDB.GetNonce(caller.Address()))\n```\n- CREATE2\n```\ncodeAndHash := &codeAndHash{code: code}\n\tcontractAddr = crypto.CreateAddress2(caller.Address(), salt.Bytes32(), codeAndHash.Hash().Bytes())\n```\nduring create contract, an object `Contract` is created. A Contract object contains and maintains the necessary information during the execution of the contract, such as the contract creator, the address of the contract itself, the remaining gas of the contract, the contract code and the `jumpdests` record of the code.\n\nthen, it invokes below method to create contract\n```\nret, err := evm.interpreter.Run(contract, nil, false)\nevm.StateDB.SetCode(address, ret)\n```\nIf the operation is successful and the contract code does not exceed the length limit, call StateDB.SetCode to store the contract code in the contract account of the Ethereum state database. Of course, the storage needs to consume a certain amount of gas.\n\nYou may wonder why the stored contract code is the return code after the contract runs, not the data in the original transaction (ie Transaction.data.Payload). This is because when the contract source code is compiled into binary data, in addition to the original code of the contract, the compiler also inserts some codes to perform related functions. For creation, the compiler inserts code that executes the contract's \"constructor\" (that is, the contract object's constructor method). Therefore, when the binary compiled by the compiler is submitted to the Ethereum node to create a contract, the EVM executes this binary code, in fact, it mainly executes the constructor method of the contract, and then returns other codes of the contract, so there is a `ret` variable here Stored in the state database as the actual code of the contract\n\n## call contract\nThe EVM object has three methods to implement the call of the contract, they are:\n\n- EVM. Call\n- EVM. CallCode\n- EVM. DelegateCall\n- EVM.StaticCall\nThe basic contract call function implemented by EVM.Call is nothing special. The following three calling methods are the differences compared with EVM.Call. So here we only introduce the particularity of the last three calling methods\n\n### EVM.CallCode & EVM.DelegateCall\nThe existence of EVM.CallCode and EVM.DelegateCall is to realize the characteristics of the \"library\" of the contract. If the code written by solidity is to be called as a library, it must be deployed on the blockchain to obtain a fixed address like a normal contract. , other contracts can call the method provided by this \"library contract\". But the contract also involves some unique attributes, such as the caller of the contract, contract address, the amount of ether it owns, etc. If we directly call the code of the \"library contract\", these properties must be the properties of the \"library contract\" itself, but this may not be what we want\n\nas an example\n```\nA -> contractB - delegateCall -> libC\n```\n`EVM.DelegateCall` sets the caller (msg.sender) of the \"library contract\" (libC) to A, rather than contractB; sets the address of the \"library contract\" (libC) to contractB. \n`EVM.CallCode` is similar to `EVM.DelegateCall`. the only difference is that `EVM.CallCode` only change the address of the \"library contract\" (libC) to contractB, without chanding the caller to A.\n`EVM.StaticCall` is similar to `EVM.Call`, the only difference is that EVM.StaticCall does not allow execution of instructions that modify permanently stored data\n\nduring contract call, it first check whether it is precompiled contract. some precompiled contracts are\n- common.BytesToAddress([]byte{1}): &ecrecover{},\n- common.BytesToAddress([]byte{2}): &sha256hash{},\n- common.BytesToAddress([]byte{3}): &ripemd160hash{},\n- common.BytesToAddress([]byte{4}): &dataCopy{},\n\n# EVMInterpreter\nThe interpreter object EVMInterpreter is used to interpret and execute specified contract instructions. However, note that the actual instruction interpretation and execution is not really completed by the interpreter object, but by the operation object JumpTable. The interpreter object is only responsible for parsing instruction codes one by one, and then obtains the corresponding operation object, and check objects such as the stack before calling the operation.execute function that actually executre the instruction. It can also be said that the interpreter object is only responsible for the scheduling of interpretation.\n\n## execution layout\n![layout](/images/evm.layout.png)\n\n## intrinsic gas\nThe intrinsic gas for a transaction is the amount of gas that the transaction uses before any code runs. It is a constant transaction fee (currently 21000 gas) plus a fee for every byte of data supplied with the transaction (4 gas for a zero byte, 68 gas for non-zeros). These constants are all currently defined for geth in params/protocol_params.go.\n\n## gas cost\nthe gas cost of each instruction is stored in `JumpTable.operation.dynamicGas` or `JumpTable.operation.constantGas`. constantGas means the operation gas cost is a fixed constant. dynamicGas is a function which will return gas during runtime.\n\nIn fact, not only the interpretation and execution of the instruction itself consumes gas, but also consumes gas when using memory storage and StateDB permanent storage. For most instructions, the latter two are not used (memory & storage), but for some instructions (such as CODECOPY or SSTORE), their gasCost function will take memory and StateDB usage into account.\n\na method `memoryGasCost`is used to calculate the gas consumption of memory usage. only when the required space size exceeds the current space size, the excess part needs to consume gas.\n\n# JumpTable\njumptable is 256 sized array of `operation`\n\n## jump instruction\nAmong the instructions of the contract, there are two jump instructions (excluding CALL): JUMP and JUMPI. Their special feature is that the first instruction of the target address after the jump must be JUMPDEST\n```\nfunc opJump(pc *uint64, interpreter *EVMInterpreter, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) {\n pos := stack.pop()\n if !contract.validJumpdest(pos) {\n nop := contract.GetOp(pos.Uint64())\n return nil, fmt.Errorf(\"invalid jump destination (%v) %v\", nop, pos)\n }\n *pc = pos.Uint64()\n\n interpreter.intPool.put(pos)\n return nil, nil\n}\n```\nA function interprets and executes the JUMP instruction. The code first fetches a value from the stack as the jump destination. This value is actually an offset relative to field 0 of the contract code. Then the code will call Contract.validJumpdest to determine whether the first instruction of this destination is JUMPDEST, if it is not, an error will occur.\n\nTo judge whether the first instruction of the destination is JUMPDEST, two points must be guaranteed: first, its value is the value of the opcode of the JUMPDEST instruction; second, it is an instruction, not ordinary data.\n\nLet's introduce how Contract.validJumpdest works. In addition to comparing opcode (this is very simple), Contract will also create a bit vector object (ie bitvec, bit vector). This object will analyze the contract instructions from the beginning to the end. If the byte at a certain offset of the contract belongs to ordinary data, the \"bit\" corresponding to the offset value in bitvec is set to 1, and if it is an instruction, it is set to 0. In Contract.validJumpdest, it is judged whether this is a normal instruction by checking whether the \"bit\" of the offset value of the jump destination in this bit vector object is 0\n\n# references\n- [yangzhe_blog](https://yangzhe.me/2019/08/12/ethereum-evm/#%E8%A7%A3%E9%87%8A%E5%99%A8%E5%AF%B9%E8%B1%A1evminterpreter)\n- [op code manual](https://www.evm.codes/?fork=shanghai)","slug":"geth/code_analysis/geth.evm","published":1,"updated":"2023-11-05T04:21:13.733Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8dw0028qwsj2vul70i7","content":"

overall

the code is under path core/vm
overview of the whole evm module \"evm\"

\n

the core is EVM struct (in evm.go), with main function in creating or call contract. a new EVM object is created every time when processing a transaction. inside the EVM struct, the main items are Interpreter, and StateDB (for state persistence). Interpreter loops through contract call instructions.Before each instruction is executed, some checks are performed to ensure sufficient gas and stack space. actual instruction execution code is recorded in JumpTable (256 sized array of operation)

\n

depending on the version of Ethereum, JumpTable may point to four different instruction sets: constantinopleInstructionSet, byzantiumInstructionSet, homesteadInstructionSet, frontierInstructionSet. Most of the instructions of these four sets of instruction sets are the same, but as the version is updated, the new version supports more instruction sets than the old version.

\n

evm

The EVM object is the most important object exported by the evm module, which represents an Ethereum virtual machine

\n

creating evm

Every time a transaction is processed, an EVM is created to execute the transaction. This is reflected in the function ApplyTransaction (core/state_processor.go)

\n

creating contract

If the to of the transaction is empty, it means that this transaction is to create a contract, so call EVM.Create to perform related functions

\n
    \n
  • CREATE
    1
    contractAddr = crypto.CreateAddress(caller.Address(), evm.StateDB.GetNonce(caller.Address()))
    • \n
  • CREATE2
    1
    2
    codeAndHash := &codeAndHash{code: code}
    \tcontractAddr = crypto.CreateAddress2(caller.Address(), salt.Bytes32(), codeAndHash.Hash().Bytes())
    \nduring create contract, an object Contract is created. A Contract object contains and maintains the necessary information during the execution of the contract, such as the contract creator, the address of the contract itself, the remaining gas of the contract, the contract code and the jumpdests record of the code.
    • \n
\n

then, it invokes below method to create contract

\n
1
2
ret, err := evm.interpreter.Run(contract, nil, false)
evm.StateDB.SetCode(address, ret)
\n

If the operation is successful and the contract code does not exceed the length limit, call StateDB.SetCode to store the contract code in the contract account of the Ethereum state database. Of course, the storage needs to consume a certain amount of gas.

\n

You may wonder why the stored contract code is the return code after the contract runs, not the data in the original transaction (ie Transaction.data.Payload). This is because when the contract source code is compiled into binary data, in addition to the original code of the contract, the compiler also inserts some codes to perform related functions. For creation, the compiler inserts code that executes the contract’s “constructor” (that is, the contract object’s constructor method). Therefore, when the binary compiled by the compiler is submitted to the Ethereum node to create a contract, the EVM executes this binary code, in fact, it mainly executes the constructor method of the contract, and then returns other codes of the contract, so there is a ret variable here Stored in the state database as the actual code of the contract

\n

call contract

The EVM object has three methods to implement the call of the contract, they are:

\n
    \n
  • EVM. Call
    • \n
  • EVM. CallCode
    • \n
  • EVM. DelegateCall
    • \n
  • EVM.StaticCall
    The basic contract call function implemented by EVM.Call is nothing special. The following three calling methods are the differences compared with EVM.Call. So here we only introduce the particularity of the last three calling methods
    • \n
\n

EVM.CallCode & EVM.DelegateCall

The existence of EVM.CallCode and EVM.DelegateCall is to realize the characteristics of the “library” of the contract. If the code written by solidity is to be called as a library, it must be deployed on the blockchain to obtain a fixed address like a normal contract. , other contracts can call the method provided by this “library contract”. But the contract also involves some unique attributes, such as the caller of the contract, contract address, the amount of ether it owns, etc. If we directly call the code of the “library contract”, these properties must be the properties of the “library contract” itself, but this may not be what we want

\n

as an example

\n
1
A -> contractB - delegateCall -> libC
\n

EVM.DelegateCall sets the caller (msg.sender) of the “library contract” (libC) to A, rather than contractB; sets the address of the “library contract” (libC) to contractB.
EVM.CallCode is similar to EVM.DelegateCall. the only difference is that EVM.CallCode only change the address of the “library contract” (libC) to contractB, without chanding the caller to A.
EVM.StaticCall is similar to EVM.Call, the only difference is that EVM.StaticCall does not allow execution of instructions that modify permanently stored data

\n

during contract call, it first check whether it is precompiled contract. some precompiled contracts are

\n
    \n
  • common.BytesToAddress([]byte{1}): &ecrecover{},
    • \n
  • common.BytesToAddress([]byte{2}): &sha256hash{},
    • \n
  • common.BytesToAddress([]byte{3}): &ripemd160hash{},
    • \n
  • common.BytesToAddress([]byte{4}): &dataCopy{},
    • \n
\n

EVMInterpreter

The interpreter object EVMInterpreter is used to interpret and execute specified contract instructions. However, note that the actual instruction interpretation and execution is not really completed by the interpreter object, but by the operation object JumpTable. The interpreter object is only responsible for parsing instruction codes one by one, and then obtains the corresponding operation object, and check objects such as the stack before calling the operation.execute function that actually executre the instruction. It can also be said that the interpreter object is only responsible for the scheduling of interpretation.

\n

execution layout

\"layout\"

\n

intrinsic gas

The intrinsic gas for a transaction is the amount of gas that the transaction uses before any code runs. It is a constant transaction fee (currently 21000 gas) plus a fee for every byte of data supplied with the transaction (4 gas for a zero byte, 68 gas for non-zeros). These constants are all currently defined for geth in params/protocol_params.go.

\n

gas cost

the gas cost of each instruction is stored in JumpTable.operation.dynamicGas or JumpTable.operation.constantGas. constantGas means the operation gas cost is a fixed constant. dynamicGas is a function which will return gas during runtime.

\n

In fact, not only the interpretation and execution of the instruction itself consumes gas, but also consumes gas when using memory storage and StateDB permanent storage. For most instructions, the latter two are not used (memory & storage), but for some instructions (such as CODECOPY or SSTORE), their gasCost function will take memory and StateDB usage into account.

\n

a method memoryGasCostis used to calculate the gas consumption of memory usage. only when the required space size exceeds the current space size, the excess part needs to consume gas.

\n

JumpTable

jumptable is 256 sized array of operation

\n

jump instruction

Among the instructions of the contract, there are two jump instructions (excluding CALL): JUMP and JUMPI. Their special feature is that the first instruction of the target address after the jump must be JUMPDEST

\n
1
2
3
4
5
6
7
8
9
10
11
func opJump(pc *uint64, interpreter *EVMInterpreter, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) {
    pos := stack.pop()
    if !contract.validJumpdest(pos) {
        nop := contract.GetOp(pos.Uint64())
        return nil, fmt.Errorf("invalid jump destination (%v) %v", nop, pos)
    }
    *pc = pos.Uint64()

    interpreter.intPool.put(pos)
    return nil, nil
}
\n

A function interprets and executes the JUMP instruction. The code first fetches a value from the stack as the jump destination. This value is actually an offset relative to field 0 of the contract code. Then the code will call Contract.validJumpdest to determine whether the first instruction of this destination is JUMPDEST, if it is not, an error will occur.

\n

To judge whether the first instruction of the destination is JUMPDEST, two points must be guaranteed: first, its value is the value of the opcode of the JUMPDEST instruction; second, it is an instruction, not ordinary data.

\n

Let’s introduce how Contract.validJumpdest works. In addition to comparing opcode (this is very simple), Contract will also create a bit vector object (ie bitvec, bit vector). This object will analyze the contract instructions from the beginning to the end. If the byte at a certain offset of the contract belongs to ordinary data, the “bit” corresponding to the offset value in bitvec is set to 1, and if it is an instruction, it is set to 0. In Contract.validJumpdest, it is judged whether this is a normal instruction by checking whether the “bit” of the offset value of the jump destination in this bit vector object is 0

\n

references

\n","site":{"data":{}},"excerpt":"","more":"

overall

the code is under path core/vm
overview of the whole evm module \"evm\"

\n

the core is EVM struct (in evm.go), with main function in creating or call contract. a new EVM object is created every time when processing a transaction. inside the EVM struct, the main items are Interpreter, and StateDB (for state persistence). Interpreter loops through contract call instructions.Before each instruction is executed, some checks are performed to ensure sufficient gas and stack space. actual instruction execution code is recorded in JumpTable (256 sized array of operation)

\n

depending on the version of Ethereum, JumpTable may point to four different instruction sets: constantinopleInstructionSet, byzantiumInstructionSet, homesteadInstructionSet, frontierInstructionSet. Most of the instructions of these four sets of instruction sets are the same, but as the version is updated, the new version supports more instruction sets than the old version.

\n

evm

The EVM object is the most important object exported by the evm module, which represents an Ethereum virtual machine

\n

creating evm

Every time a transaction is processed, an EVM is created to execute the transaction. This is reflected in the function ApplyTransaction (core/state_processor.go)

\n

creating contract

If the to of the transaction is empty, it means that this transaction is to create a contract, so call EVM.Create to perform related functions

\n
    \n
  • CREATE
    1
    contractAddr = crypto.CreateAddress(caller.Address(), evm.StateDB.GetNonce(caller.Address()))
    • \n
  • CREATE2
    1
    2
    codeAndHash := &codeAndHash{code: code}
    \tcontractAddr = crypto.CreateAddress2(caller.Address(), salt.Bytes32(), codeAndHash.Hash().Bytes())
    \nduring create contract, an object Contract is created. A Contract object contains and maintains the necessary information during the execution of the contract, such as the contract creator, the address of the contract itself, the remaining gas of the contract, the contract code and the jumpdests record of the code.
    • \n
\n

then, it invokes below method to create contract

\n
1
2
ret, err := evm.interpreter.Run(contract, nil, false)
evm.StateDB.SetCode(address, ret)
\n

If the operation is successful and the contract code does not exceed the length limit, call StateDB.SetCode to store the contract code in the contract account of the Ethereum state database. Of course, the storage needs to consume a certain amount of gas.

\n

You may wonder why the stored contract code is the return code after the contract runs, not the data in the original transaction (ie Transaction.data.Payload). This is because when the contract source code is compiled into binary data, in addition to the original code of the contract, the compiler also inserts some codes to perform related functions. For creation, the compiler inserts code that executes the contract’s “constructor” (that is, the contract object’s constructor method). Therefore, when the binary compiled by the compiler is submitted to the Ethereum node to create a contract, the EVM executes this binary code, in fact, it mainly executes the constructor method of the contract, and then returns other codes of the contract, so there is a ret variable here Stored in the state database as the actual code of the contract

\n

call contract

The EVM object has three methods to implement the call of the contract, they are:

\n
    \n
  • EVM. Call
    • \n
  • EVM. CallCode
    • \n
  • EVM. DelegateCall
    • \n
  • EVM.StaticCall
    The basic contract call function implemented by EVM.Call is nothing special. The following three calling methods are the differences compared with EVM.Call. So here we only introduce the particularity of the last three calling methods
    • \n
\n

EVM.CallCode & EVM.DelegateCall

The existence of EVM.CallCode and EVM.DelegateCall is to realize the characteristics of the “library” of the contract. If the code written by solidity is to be called as a library, it must be deployed on the blockchain to obtain a fixed address like a normal contract. , other contracts can call the method provided by this “library contract”. But the contract also involves some unique attributes, such as the caller of the contract, contract address, the amount of ether it owns, etc. If we directly call the code of the “library contract”, these properties must be the properties of the “library contract” itself, but this may not be what we want

\n

as an example

\n
1
A -> contractB - delegateCall -> libC
\n

EVM.DelegateCall sets the caller (msg.sender) of the “library contract” (libC) to A, rather than contractB; sets the address of the “library contract” (libC) to contractB.
EVM.CallCode is similar to EVM.DelegateCall. the only difference is that EVM.CallCode only change the address of the “library contract” (libC) to contractB, without chanding the caller to A.
EVM.StaticCall is similar to EVM.Call, the only difference is that EVM.StaticCall does not allow execution of instructions that modify permanently stored data

\n

during contract call, it first check whether it is precompiled contract. some precompiled contracts are

\n
    \n
  • common.BytesToAddress([]byte{1}): &ecrecover{},
    • \n
  • common.BytesToAddress([]byte{2}): &sha256hash{},
    • \n
  • common.BytesToAddress([]byte{3}): &ripemd160hash{},
    • \n
  • common.BytesToAddress([]byte{4}): &dataCopy{},
    • \n
\n

EVMInterpreter

The interpreter object EVMInterpreter is used to interpret and execute specified contract instructions. However, note that the actual instruction interpretation and execution is not really completed by the interpreter object, but by the operation object JumpTable. The interpreter object is only responsible for parsing instruction codes one by one, and then obtains the corresponding operation object, and check objects such as the stack before calling the operation.execute function that actually executre the instruction. It can also be said that the interpreter object is only responsible for the scheduling of interpretation.

\n

execution layout

\"layout\"

\n

intrinsic gas

The intrinsic gas for a transaction is the amount of gas that the transaction uses before any code runs. It is a constant transaction fee (currently 21000 gas) plus a fee for every byte of data supplied with the transaction (4 gas for a zero byte, 68 gas for non-zeros). These constants are all currently defined for geth in params/protocol_params.go.

\n

gas cost

the gas cost of each instruction is stored in JumpTable.operation.dynamicGas or JumpTable.operation.constantGas. constantGas means the operation gas cost is a fixed constant. dynamicGas is a function which will return gas during runtime.

\n

In fact, not only the interpretation and execution of the instruction itself consumes gas, but also consumes gas when using memory storage and StateDB permanent storage. For most instructions, the latter two are not used (memory & storage), but for some instructions (such as CODECOPY or SSTORE), their gasCost function will take memory and StateDB usage into account.

\n

a method memoryGasCostis used to calculate the gas consumption of memory usage. only when the required space size exceeds the current space size, the excess part needs to consume gas.

\n

JumpTable

jumptable is 256 sized array of operation

\n

jump instruction

Among the instructions of the contract, there are two jump instructions (excluding CALL): JUMP and JUMPI. Their special feature is that the first instruction of the target address after the jump must be JUMPDEST

\n
1
2
3
4
5
6
7
8
9
10
11
func opJump(pc *uint64, interpreter *EVMInterpreter, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) {
    pos := stack.pop()
    if !contract.validJumpdest(pos) {
        nop := contract.GetOp(pos.Uint64())
        return nil, fmt.Errorf("invalid jump destination (%v) %v", nop, pos)
    }
    *pc = pos.Uint64()

    interpreter.intPool.put(pos)
    return nil, nil
}
\n

A function interprets and executes the JUMP instruction. The code first fetches a value from the stack as the jump destination. This value is actually an offset relative to field 0 of the contract code. Then the code will call Contract.validJumpdest to determine whether the first instruction of this destination is JUMPDEST, if it is not, an error will occur.

\n

To judge whether the first instruction of the destination is JUMPDEST, two points must be guaranteed: first, its value is the value of the opcode of the JUMPDEST instruction; second, it is an instruction, not ordinary data.

\n

Let’s introduce how Contract.validJumpdest works. In addition to comparing opcode (this is very simple), Contract will also create a bit vector object (ie bitvec, bit vector). This object will analyze the contract instructions from the beginning to the end. If the byte at a certain offset of the contract belongs to ordinary data, the “bit” corresponding to the offset value in bitvec is set to 1, and if it is an instruction, it is set to 0. In Contract.validJumpdest, it is judged whether this is a normal instruction by checking whether the “bit” of the offset value of the jump destination in this bit vector object is 0

\n

references

\n"},{"title":"geth state prune","date":"2023-03-25T11:29:43.000Z","_content":"\n> **_NOTE:_** Offline pruning is only for the hash-based state scheme. In future release, geth will have a path-based state scheme which enables the pruning by default. Once the hash-based state scheme is no longer supported, offline pruning will be deprecated.\n\n## introduction\nA snap-sync'd Geth node currently requires more than 650 GB of disk space to store the historic blockchain data. With default cache size the database grows by about 14 GB/week. Since Geth v1.10, users have been able to trigger a snapshot offline prune to bring the total storage back down to the original ~650 GB in about 4-5 hours.\n\n## how pruning works\nPruning uses snapshots of the state database as an indicator to determine which nodes in the state trie can be kept and which ones are stale and can be discarded. Geth identifies the target state trie based on a stored snapshot layer which has at least 128 block confirmations on top (for surviving reorgs) data that isn't part of the target state trie or genesis state.\n\nGeth prunes the database in three stages:\n\n1. Iterating state snapshot: Geth iterates the bottom-most snapshot layer and constructs a bloom filter set for identifying the target trie nodes.\n2. Pruning state data: Geth deletes stale trie nodes from the database which are not in the bloom filter set.\n3. Compacting database: Geth tidies up the new database to reclaim free space.\n\n## Pruning command\n```\ngeth snapshot prune-state\n```\n\n## references\n- [geth doc](https://geth.ethereum.org/docs/fundamentals/pruning)","source":"_posts/geth/tech_docs/geth.prune.md","raw":"---\ntitle: geth state prune\ndate: 2023-03-25 19:29:43\ntags: [geth]\n---\n\n> **_NOTE:_** Offline pruning is only for the hash-based state scheme. In future release, geth will have a path-based state scheme which enables the pruning by default. Once the hash-based state scheme is no longer supported, offline pruning will be deprecated.\n\n## introduction\nA snap-sync'd Geth node currently requires more than 650 GB of disk space to store the historic blockchain data. With default cache size the database grows by about 14 GB/week. Since Geth v1.10, users have been able to trigger a snapshot offline prune to bring the total storage back down to the original ~650 GB in about 4-5 hours.\n\n## how pruning works\nPruning uses snapshots of the state database as an indicator to determine which nodes in the state trie can be kept and which ones are stale and can be discarded. Geth identifies the target state trie based on a stored snapshot layer which has at least 128 block confirmations on top (for surviving reorgs) data that isn't part of the target state trie or genesis state.\n\nGeth prunes the database in three stages:\n\n1. Iterating state snapshot: Geth iterates the bottom-most snapshot layer and constructs a bloom filter set for identifying the target trie nodes.\n2. Pruning state data: Geth deletes stale trie nodes from the database which are not in the bloom filter set.\n3. Compacting database: Geth tidies up the new database to reclaim free space.\n\n## Pruning command\n```\ngeth snapshot prune-state\n```\n\n## references\n- [geth doc](https://geth.ethereum.org/docs/fundamentals/pruning)","slug":"geth/tech_docs/geth.prune","published":1,"updated":"2023-11-05T04:21:13.733Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8dx002aqwsjacvpcivw","content":"
\n

NOTE: Offline pruning is only for the hash-based state scheme. In future release, geth will have a path-based state scheme which enables the pruning by default. Once the hash-based state scheme is no longer supported, offline pruning will be deprecated.

\n
\n

introduction

A snap-sync’d Geth node currently requires more than 650 GB of disk space to store the historic blockchain data. With default cache size the database grows by about 14 GB/week. Since Geth v1.10, users have been able to trigger a snapshot offline prune to bring the total storage back down to the original ~650 GB in about 4-5 hours.

\n

how pruning works

Pruning uses snapshots of the state database as an indicator to determine which nodes in the state trie can be kept and which ones are stale and can be discarded. Geth identifies the target state trie based on a stored snapshot layer which has at least 128 block confirmations on top (for surviving reorgs) data that isn’t part of the target state trie or genesis state.

\n

Geth prunes the database in three stages:

\n
    \n
  1. Iterating state snapshot: Geth iterates the bottom-most snapshot layer and constructs a bloom filter set for identifying the target trie nodes.
    2. \n
  2. Pruning state data: Geth deletes stale trie nodes from the database which are not in the bloom filter set.
    3. \n
  3. Compacting database: Geth tidies up the new database to reclaim free space.
    4. \n
\n

Pruning command

1
geth snapshot prune-state
\n\n

references

\n","site":{"data":{}},"excerpt":"","more":"
\n

NOTE: Offline pruning is only for the hash-based state scheme. In future release, geth will have a path-based state scheme which enables the pruning by default. Once the hash-based state scheme is no longer supported, offline pruning will be deprecated.

\n
\n

introduction

A snap-sync’d Geth node currently requires more than 650 GB of disk space to store the historic blockchain data. With default cache size the database grows by about 14 GB/week. Since Geth v1.10, users have been able to trigger a snapshot offline prune to bring the total storage back down to the original ~650 GB in about 4-5 hours.

\n

how pruning works

Pruning uses snapshots of the state database as an indicator to determine which nodes in the state trie can be kept and which ones are stale and can be discarded. Geth identifies the target state trie based on a stored snapshot layer which has at least 128 block confirmations on top (for surviving reorgs) data that isn’t part of the target state trie or genesis state.

\n

Geth prunes the database in three stages:

\n
    \n
  1. Iterating state snapshot: Geth iterates the bottom-most snapshot layer and constructs a bloom filter set for identifying the target trie nodes.
    2. \n
  2. Pruning state data: Geth deletes stale trie nodes from the database which are not in the bloom filter set.
    3. \n
  3. Compacting database: Geth tidies up the new database to reclaim free space.
    4. \n
\n

Pruning command

1
geth snapshot prune-state
\n\n

references

\n"},{"title":"geth sync","date":"2023-03-18T08:29:43.000Z","_content":"\n## state \nEthereum maintains two different types of state: the set of accounts; and a set of storage slots for each contract account. Naively, storing these key-value pairs as flat data would be very efficient, however, verifying their correctness becomes computationally intractable. Every time a modification would be made, we'd need to hash all that data from scratch (which is not efficient).\n\nInstead of hashing the entire dataset all the time, eth uses MPT. The original useful data would be in the leaves, and each internal node would be a hash of everything below it. This would allow us to only recalculate a logarithmic number of hashes when something is modified, inserted, deleted and verified. A tiny extra is that keys are hashed before insertion to balance the tries (secured trie).\n\n## state storage\nMPT make every read/write of O(lnN) complexity. the depth of the trie is continuously growing; LevelDB also organizes its data into a maximum of 7 levels, so there's an extra multiplier from there. The net result is that a single state access is expected to amplify into **25-50** random disk accesses. \n\nOf course all client implementations try their best to minimize this overhead. Geth uses large memory areas for caching trie nodes; and also uses in-memory pruning to avoid writing to disk nodes that get deleted anyway after a few blocks.\n\n## Not all accesses are created equal\nThe Merkle Patricia tree is essential for writes (matain the capability to verify data), but it's an overhead for reads. \nAn Ethereum node accesses state in a few different places:\n- When importing a new block, EVM code execution does a more-or-less balanced number of state reads and writes. \n- When a node operator retrieves state (e.g. eth_call and family), EVM code execution only does reads (it can write too, but those get discarded at the end and are not persisted).\n- When a node is synchronizing, it is requesting state from remote nodes that need to dig it up and serve it over the network.\n\nif we can short circuit reads not to hit the state trie, a slew of node operations will become significantly faster. \n\n## snapshot\nGeth introduced its snapshot acceleration structure (not enabled by default). A snapshot is a complete view of the Ethereum state at a given block. Abstract implementation wise, it is a dump of all accounts and storage slots, represented by a flat key-value store.\nsnapshot is maintained as an extra to MPT. The snapshot essentially reduces reads from O(log n) to O(1) at the cost of increasing writes from O(log n) to O(1 + log n). \n\n## devil's in the details\nto maintain a snapshot, the naitve approach is to apply changes to current snapshot upon new block. If there's a mini reorg however (even a single block), we're in trouble, because there's no undo. \nTo overcome this limitation, Geth's snapshot is composed of two entities: a persistent disk layer that is a complete snapshot of an older block (e.g. HEAD-128); and a tree of in-memory diff layers that gather the writes on top.\nWhenever a new block is processed, we do not merge the writes directly into the disk layer, rather just create a new in-memory diff layer with the changes. If enough in-memory diff layers are piled on top, the bottom ones start getting merged together and eventually pushed to disk. Whenever a state item is to be read, we start at the topmost diff layer and keep going backwards until we find it or reach the disk.\nOf course, there are lots and lots of gotchas and caveats.\n- On shutdown, the in-memory diff layers need to be persisted into a journal and loaded back up, otherwise the snapshot will become useless on restart.\n- Use the bottom-most diff layer as an accumulator and only flush to disk when it exceeds some memory usage.\n- Allocate a read cache for the disk layer so that contracts accessing the same ancient slot over and over don't cause disk hits.\n- Use cumulative bloom filters in the in-memory diff layers to quickly detect whether there's a chance for an item to be in the diffs, or if we can go to disk immediately.\n- The keys are not the raw data (account address, storage key), rather the hashes of these, ensuring the snapshot has the same iteration order as the Merkle Patricia tree.\n\nThe snapshot also enables blazing fast state iteration of the most recent blocks. This was actually the main reason for building snapshots, as it permitted the creation of the new snap [sync algorithm](https://github.com/ethereum/devp2p/pull/145).\n\n## Consensus layer syncing\nall consensus logic and block propagation is handled by consensus clients. Blocks are downloaded by the consensus client and verified by the execution client. **Geth cannot sync without being connected to a consensus client.** \nThere are two ways for the consensus client to find a block header that Geth can use as a sync target: optimistic syncing and checkpoint syncing:\n\n### optimistic sync\nOptimistic sync downloads blocks before the execution client has validated them. In optimistic sync the node assumes the data it receives from its peers is correct during the downloading phase but then retroactively verifies each downloaded block.\n[more details](https://github.com/ethereum/consensus-specs/blob/dev/sync/optimistic.md)\n\n### checkpoint sync\nAlternatively, the consensus client can grab a checkpoint from a trusted source which provides a target state to sync up to, before switching to full sync and verifying each block in turn. In this mode, the node trusts that the checkpoint is correct.\n\n## archive nodes\nAn archive node is a node that retains all historical data right back to genesis. There is no need to regenerate any data from checkpoints because all data is directly available in the node's own storage. \n\nIt is also possible to create a partial/recent archive node where the node was synced using snap but the state is never pruned. This creates an archive node that saves all state data from the point that the node first syncs. This is configured by starting Geth with `--syncmode snap --gcmode archive`.\n\n\n## light nodes\nA light node syncs very quickly and stores the bare minimum of blockchain data. Light nodes only process block headers, not entire blocks. they receive a proof from the full node and verify it against their local header chain. **Light nodes are not currently working on proof-of-stake Ethereum.**\n\n\n\n## full node\n### full\nA full block-by-block sync generates the current state by executing every block starting from the genesis block. A full sync independently verifies block provenance as well as all state transitions by re-executing the transactions in the entire historical sequence of blocks. Only the most recent 128 block states are stored in a full node - older block states are pruned periodically and represented as a series of checkpoints from which any previous state can be regenerated on request.\n\n### snap sync (default)\nSnap sync starts from a relatively recent block and syncs from there to the head of the chain, keeping only the most recent 128 block states in memory. The block header to sync up to is provided by the consensus client. Between the initial sync block and the 128 most recent blocks, the node stores occasional snapshots that can be used to rebuild any intermediate state \"on-the-fly\". The difference between the snap-synced node and a full block-by-block synced node is that a snap synced node started from an initial checkpoint that was more recent than the genesis block. Snap sync is much faster than a full block-by-block sync from genesis.\n![sync mode](/images/geth/sync.mode.jpg)\n\nSnap sync works by first downloading the headers for a chunk of blocks. Once the headers have been verified, the block bodies and receipts for those blocks are downloaded. In parallel, Geth also begins state-sync. In state-sync, Geth first downloads the leaves of the state trie for each block without the intermediate nodes along with a range proof. The state trie is then regenerated locally.\n\n\nThe state download is the part of the snap-sync that takes the most time to complete and the progress can be monitored using the ETA values in the log messages. However, the blockchain is also progressing at the same time and invalidating some of the regenerated state data. (don't really understand why regenrated state could be invalidated). This means it is also necessary to have a 'healing' phase where errors in the state are fixed. Geth regularly reports `Syncing, state heal in progress` during state healing - this informs the user that state heal has not finished.\n\nThe healing has to outpace the growth of the blockchain, otherwise the node will never catch up to the current state.\n\nTo summarize, snap sync progresses in the following sequence:\n\n- download and verify headers\n- download block bodies and receipts. In parallel, download raw state data and build state trie\n- heal state trie to account for newly arriving data\n\nA node that is started using snap will switch to block-by-block sync once it has caught up to the head of the chain.\n\n\n\n\n\n\n\n\n\n# references\n- [geth doc on sync mode](https://geth.ethereum.org/docs/fundamentals/sync-modes)\n- [eth.org blog on snapshot acceleration](https://blog.ethereum.org/2020/07/17/ask-about-geth-snapshot-acceleration)","source":"_posts/geth/tech_docs/geth.sync.mode.md","raw":"---\ntitle: geth sync\ndate: 2023-03-18 16:29:43\ntags: [geth]\n---\n\n## state \nEthereum maintains two different types of state: the set of accounts; and a set of storage slots for each contract account. Naively, storing these key-value pairs as flat data would be very efficient, however, verifying their correctness becomes computationally intractable. Every time a modification would be made, we'd need to hash all that data from scratch (which is not efficient).\n\nInstead of hashing the entire dataset all the time, eth uses MPT. The original useful data would be in the leaves, and each internal node would be a hash of everything below it. This would allow us to only recalculate a logarithmic number of hashes when something is modified, inserted, deleted and verified. A tiny extra is that keys are hashed before insertion to balance the tries (secured trie).\n\n## state storage\nMPT make every read/write of O(lnN) complexity. the depth of the trie is continuously growing; LevelDB also organizes its data into a maximum of 7 levels, so there's an extra multiplier from there. The net result is that a single state access is expected to amplify into **25-50** random disk accesses. \n\nOf course all client implementations try their best to minimize this overhead. Geth uses large memory areas for caching trie nodes; and also uses in-memory pruning to avoid writing to disk nodes that get deleted anyway after a few blocks.\n\n## Not all accesses are created equal\nThe Merkle Patricia tree is essential for writes (matain the capability to verify data), but it's an overhead for reads. \nAn Ethereum node accesses state in a few different places:\n- When importing a new block, EVM code execution does a more-or-less balanced number of state reads and writes. \n- When a node operator retrieves state (e.g. eth_call and family), EVM code execution only does reads (it can write too, but those get discarded at the end and are not persisted).\n- When a node is synchronizing, it is requesting state from remote nodes that need to dig it up and serve it over the network.\n\nif we can short circuit reads not to hit the state trie, a slew of node operations will become significantly faster. \n\n## snapshot\nGeth introduced its snapshot acceleration structure (not enabled by default). A snapshot is a complete view of the Ethereum state at a given block. Abstract implementation wise, it is a dump of all accounts and storage slots, represented by a flat key-value store.\nsnapshot is maintained as an extra to MPT. The snapshot essentially reduces reads from O(log n) to O(1) at the cost of increasing writes from O(log n) to O(1 + log n). \n\n## devil's in the details\nto maintain a snapshot, the naitve approach is to apply changes to current snapshot upon new block. If there's a mini reorg however (even a single block), we're in trouble, because there's no undo. \nTo overcome this limitation, Geth's snapshot is composed of two entities: a persistent disk layer that is a complete snapshot of an older block (e.g. HEAD-128); and a tree of in-memory diff layers that gather the writes on top.\nWhenever a new block is processed, we do not merge the writes directly into the disk layer, rather just create a new in-memory diff layer with the changes. If enough in-memory diff layers are piled on top, the bottom ones start getting merged together and eventually pushed to disk. Whenever a state item is to be read, we start at the topmost diff layer and keep going backwards until we find it or reach the disk.\nOf course, there are lots and lots of gotchas and caveats.\n- On shutdown, the in-memory diff layers need to be persisted into a journal and loaded back up, otherwise the snapshot will become useless on restart.\n- Use the bottom-most diff layer as an accumulator and only flush to disk when it exceeds some memory usage.\n- Allocate a read cache for the disk layer so that contracts accessing the same ancient slot over and over don't cause disk hits.\n- Use cumulative bloom filters in the in-memory diff layers to quickly detect whether there's a chance for an item to be in the diffs, or if we can go to disk immediately.\n- The keys are not the raw data (account address, storage key), rather the hashes of these, ensuring the snapshot has the same iteration order as the Merkle Patricia tree.\n\nThe snapshot also enables blazing fast state iteration of the most recent blocks. This was actually the main reason for building snapshots, as it permitted the creation of the new snap [sync algorithm](https://github.com/ethereum/devp2p/pull/145).\n\n## Consensus layer syncing\nall consensus logic and block propagation is handled by consensus clients. Blocks are downloaded by the consensus client and verified by the execution client. **Geth cannot sync without being connected to a consensus client.** \nThere are two ways for the consensus client to find a block header that Geth can use as a sync target: optimistic syncing and checkpoint syncing:\n\n### optimistic sync\nOptimistic sync downloads blocks before the execution client has validated them. In optimistic sync the node assumes the data it receives from its peers is correct during the downloading phase but then retroactively verifies each downloaded block.\n[more details](https://github.com/ethereum/consensus-specs/blob/dev/sync/optimistic.md)\n\n### checkpoint sync\nAlternatively, the consensus client can grab a checkpoint from a trusted source which provides a target state to sync up to, before switching to full sync and verifying each block in turn. In this mode, the node trusts that the checkpoint is correct.\n\n## archive nodes\nAn archive node is a node that retains all historical data right back to genesis. There is no need to regenerate any data from checkpoints because all data is directly available in the node's own storage. \n\nIt is also possible to create a partial/recent archive node where the node was synced using snap but the state is never pruned. This creates an archive node that saves all state data from the point that the node first syncs. This is configured by starting Geth with `--syncmode snap --gcmode archive`.\n\n\n## light nodes\nA light node syncs very quickly and stores the bare minimum of blockchain data. Light nodes only process block headers, not entire blocks. they receive a proof from the full node and verify it against their local header chain. **Light nodes are not currently working on proof-of-stake Ethereum.**\n\n\n\n## full node\n### full\nA full block-by-block sync generates the current state by executing every block starting from the genesis block. A full sync independently verifies block provenance as well as all state transitions by re-executing the transactions in the entire historical sequence of blocks. Only the most recent 128 block states are stored in a full node - older block states are pruned periodically and represented as a series of checkpoints from which any previous state can be regenerated on request.\n\n### snap sync (default)\nSnap sync starts from a relatively recent block and syncs from there to the head of the chain, keeping only the most recent 128 block states in memory. The block header to sync up to is provided by the consensus client. Between the initial sync block and the 128 most recent blocks, the node stores occasional snapshots that can be used to rebuild any intermediate state \"on-the-fly\". The difference between the snap-synced node and a full block-by-block synced node is that a snap synced node started from an initial checkpoint that was more recent than the genesis block. Snap sync is much faster than a full block-by-block sync from genesis.\n![sync mode](/images/geth/sync.mode.jpg)\n\nSnap sync works by first downloading the headers for a chunk of blocks. Once the headers have been verified, the block bodies and receipts for those blocks are downloaded. In parallel, Geth also begins state-sync. In state-sync, Geth first downloads the leaves of the state trie for each block without the intermediate nodes along with a range proof. The state trie is then regenerated locally.\n\n\nThe state download is the part of the snap-sync that takes the most time to complete and the progress can be monitored using the ETA values in the log messages. However, the blockchain is also progressing at the same time and invalidating some of the regenerated state data. (don't really understand why regenrated state could be invalidated). This means it is also necessary to have a 'healing' phase where errors in the state are fixed. Geth regularly reports `Syncing, state heal in progress` during state healing - this informs the user that state heal has not finished.\n\nThe healing has to outpace the growth of the blockchain, otherwise the node will never catch up to the current state.\n\nTo summarize, snap sync progresses in the following sequence:\n\n- download and verify headers\n- download block bodies and receipts. In parallel, download raw state data and build state trie\n- heal state trie to account for newly arriving data\n\nA node that is started using snap will switch to block-by-block sync once it has caught up to the head of the chain.\n\n\n\n\n\n\n\n\n\n# references\n- [geth doc on sync mode](https://geth.ethereum.org/docs/fundamentals/sync-modes)\n- [eth.org blog on snapshot acceleration](https://blog.ethereum.org/2020/07/17/ask-about-geth-snapshot-acceleration)","slug":"geth/tech_docs/geth.sync.mode","published":1,"updated":"2023-11-05T04:21:13.733Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8dx002dqwsj8w7zb4o4","content":"

state

Ethereum maintains two different types of state: the set of accounts; and a set of storage slots for each contract account. Naively, storing these key-value pairs as flat data would be very efficient, however, verifying their correctness becomes computationally intractable. Every time a modification would be made, we’d need to hash all that data from scratch (which is not efficient).

\n

Instead of hashing the entire dataset all the time, eth uses MPT. The original useful data would be in the leaves, and each internal node would be a hash of everything below it. This would allow us to only recalculate a logarithmic number of hashes when something is modified, inserted, deleted and verified. A tiny extra is that keys are hashed before insertion to balance the tries (secured trie).

\n

state storage

MPT make every read/write of O(lnN) complexity. the depth of the trie is continuously growing; LevelDB also organizes its data into a maximum of 7 levels, so there’s an extra multiplier from there. The net result is that a single state access is expected to amplify into 25-50 random disk accesses.

\n

Of course all client implementations try their best to minimize this overhead. Geth uses large memory areas for caching trie nodes; and also uses in-memory pruning to avoid writing to disk nodes that get deleted anyway after a few blocks.

\n

Not all accesses are created equal

The Merkle Patricia tree is essential for writes (matain the capability to verify data), but it’s an overhead for reads.
An Ethereum node accesses state in a few different places:

\n
    \n
  • When importing a new block, EVM code execution does a more-or-less balanced number of state reads and writes.
    • \n
  • When a node operator retrieves state (e.g. eth_call and family), EVM code execution only does reads (it can write too, but those get discarded at the end and are not persisted).
    • \n
  • When a node is synchronizing, it is requesting state from remote nodes that need to dig it up and serve it over the network.
    • \n
\n

if we can short circuit reads not to hit the state trie, a slew of node operations will become significantly faster.

\n

snapshot

Geth introduced its snapshot acceleration structure (not enabled by default). A snapshot is a complete view of the Ethereum state at a given block. Abstract implementation wise, it is a dump of all accounts and storage slots, represented by a flat key-value store.
snapshot is maintained as an extra to MPT. The snapshot essentially reduces reads from O(log n) to O(1) at the cost of increasing writes from O(log n) to O(1 + log n).

\n

devil’s in the details

to maintain a snapshot, the naitve approach is to apply changes to current snapshot upon new block. If there’s a mini reorg however (even a single block), we’re in trouble, because there’s no undo.
To overcome this limitation, Geth’s snapshot is composed of two entities: a persistent disk layer that is a complete snapshot of an older block (e.g. HEAD-128); and a tree of in-memory diff layers that gather the writes on top.
Whenever a new block is processed, we do not merge the writes directly into the disk layer, rather just create a new in-memory diff layer with the changes. If enough in-memory diff layers are piled on top, the bottom ones start getting merged together and eventually pushed to disk. Whenever a state item is to be read, we start at the topmost diff layer and keep going backwards until we find it or reach the disk.
Of course, there are lots and lots of gotchas and caveats.

\n
    \n
  • On shutdown, the in-memory diff layers need to be persisted into a journal and loaded back up, otherwise the snapshot will become useless on restart.
    • \n
  • Use the bottom-most diff layer as an accumulator and only flush to disk when it exceeds some memory usage.
    • \n
  • Allocate a read cache for the disk layer so that contracts accessing the same ancient slot over and over don’t cause disk hits.
    • \n
  • Use cumulative bloom filters in the in-memory diff layers to quickly detect whether there’s a chance for an item to be in the diffs, or if we can go to disk immediately.
    • \n
  • The keys are not the raw data (account address, storage key), rather the hashes of these, ensuring the snapshot has the same iteration order as the Merkle Patricia tree.
    • \n
\n

The snapshot also enables blazing fast state iteration of the most recent blocks. This was actually the main reason for building snapshots, as it permitted the creation of the new snap sync algorithm.

\n

Consensus layer syncing

all consensus logic and block propagation is handled by consensus clients. Blocks are downloaded by the consensus client and verified by the execution client. Geth cannot sync without being connected to a consensus client.
There are two ways for the consensus client to find a block header that Geth can use as a sync target: optimistic syncing and checkpoint syncing:

\n

optimistic sync

Optimistic sync downloads blocks before the execution client has validated them. In optimistic sync the node assumes the data it receives from its peers is correct during the downloading phase but then retroactively verifies each downloaded block.
more details

\n

checkpoint sync

Alternatively, the consensus client can grab a checkpoint from a trusted source which provides a target state to sync up to, before switching to full sync and verifying each block in turn. In this mode, the node trusts that the checkpoint is correct.

\n

archive nodes

An archive node is a node that retains all historical data right back to genesis. There is no need to regenerate any data from checkpoints because all data is directly available in the node’s own storage.

\n

It is also possible to create a partial/recent archive node where the node was synced using snap but the state is never pruned. This creates an archive node that saves all state data from the point that the node first syncs. This is configured by starting Geth with --syncmode snap --gcmode archive.

\n

light nodes

A light node syncs very quickly and stores the bare minimum of blockchain data. Light nodes only process block headers, not entire blocks. they receive a proof from the full node and verify it against their local header chain. Light nodes are not currently working on proof-of-stake Ethereum.

\n

full node

full

A full block-by-block sync generates the current state by executing every block starting from the genesis block. A full sync independently verifies block provenance as well as all state transitions by re-executing the transactions in the entire historical sequence of blocks. Only the most recent 128 block states are stored in a full node - older block states are pruned periodically and represented as a series of checkpoints from which any previous state can be regenerated on request.

\n

snap sync (default)

Snap sync starts from a relatively recent block and syncs from there to the head of the chain, keeping only the most recent 128 block states in memory. The block header to sync up to is provided by the consensus client. Between the initial sync block and the 128 most recent blocks, the node stores occasional snapshots that can be used to rebuild any intermediate state “on-the-fly”. The difference between the snap-synced node and a full block-by-block synced node is that a snap synced node started from an initial checkpoint that was more recent than the genesis block. Snap sync is much faster than a full block-by-block sync from genesis.
\"sync

\n

Snap sync works by first downloading the headers for a chunk of blocks. Once the headers have been verified, the block bodies and receipts for those blocks are downloaded. In parallel, Geth also begins state-sync. In state-sync, Geth first downloads the leaves of the state trie for each block without the intermediate nodes along with a range proof. The state trie is then regenerated locally.

\n

The state download is the part of the snap-sync that takes the most time to complete and the progress can be monitored using the ETA values in the log messages. However, the blockchain is also progressing at the same time and invalidating some of the regenerated state data. (don’t really understand why regenrated state could be invalidated). This means it is also necessary to have a ‘healing’ phase where errors in the state are fixed. Geth regularly reports Syncing, state heal in progress during state healing - this informs the user that state heal has not finished.

\n

The healing has to outpace the growth of the blockchain, otherwise the node will never catch up to the current state.

\n

To summarize, snap sync progresses in the following sequence:

\n
    \n
  • download and verify headers
    • \n
  • download block bodies and receipts. In parallel, download raw state data and build state trie
    • \n
  • heal state trie to account for newly arriving data
    • \n
\n

A node that is started using snap will switch to block-by-block sync once it has caught up to the head of the chain.

\n

references

\n","site":{"data":{}},"excerpt":"","more":"

state

Ethereum maintains two different types of state: the set of accounts; and a set of storage slots for each contract account. Naively, storing these key-value pairs as flat data would be very efficient, however, verifying their correctness becomes computationally intractable. Every time a modification would be made, we’d need to hash all that data from scratch (which is not efficient).

\n

Instead of hashing the entire dataset all the time, eth uses MPT. The original useful data would be in the leaves, and each internal node would be a hash of everything below it. This would allow us to only recalculate a logarithmic number of hashes when something is modified, inserted, deleted and verified. A tiny extra is that keys are hashed before insertion to balance the tries (secured trie).

\n

state storage

MPT make every read/write of O(lnN) complexity. the depth of the trie is continuously growing; LevelDB also organizes its data into a maximum of 7 levels, so there’s an extra multiplier from there. The net result is that a single state access is expected to amplify into 25-50 random disk accesses.

\n

Of course all client implementations try their best to minimize this overhead. Geth uses large memory areas for caching trie nodes; and also uses in-memory pruning to avoid writing to disk nodes that get deleted anyway after a few blocks.

\n

Not all accesses are created equal

The Merkle Patricia tree is essential for writes (matain the capability to verify data), but it’s an overhead for reads.
An Ethereum node accesses state in a few different places:

\n
    \n
  • When importing a new block, EVM code execution does a more-or-less balanced number of state reads and writes.
    • \n
  • When a node operator retrieves state (e.g. eth_call and family), EVM code execution only does reads (it can write too, but those get discarded at the end and are not persisted).
    • \n
  • When a node is synchronizing, it is requesting state from remote nodes that need to dig it up and serve it over the network.
    • \n
\n

if we can short circuit reads not to hit the state trie, a slew of node operations will become significantly faster.

\n

snapshot

Geth introduced its snapshot acceleration structure (not enabled by default). A snapshot is a complete view of the Ethereum state at a given block. Abstract implementation wise, it is a dump of all accounts and storage slots, represented by a flat key-value store.
snapshot is maintained as an extra to MPT. The snapshot essentially reduces reads from O(log n) to O(1) at the cost of increasing writes from O(log n) to O(1 + log n).

\n

devil’s in the details

to maintain a snapshot, the naitve approach is to apply changes to current snapshot upon new block. If there’s a mini reorg however (even a single block), we’re in trouble, because there’s no undo.
To overcome this limitation, Geth’s snapshot is composed of two entities: a persistent disk layer that is a complete snapshot of an older block (e.g. HEAD-128); and a tree of in-memory diff layers that gather the writes on top.
Whenever a new block is processed, we do not merge the writes directly into the disk layer, rather just create a new in-memory diff layer with the changes. If enough in-memory diff layers are piled on top, the bottom ones start getting merged together and eventually pushed to disk. Whenever a state item is to be read, we start at the topmost diff layer and keep going backwards until we find it or reach the disk.
Of course, there are lots and lots of gotchas and caveats.

\n
    \n
  • On shutdown, the in-memory diff layers need to be persisted into a journal and loaded back up, otherwise the snapshot will become useless on restart.
    • \n
  • Use the bottom-most diff layer as an accumulator and only flush to disk when it exceeds some memory usage.
    • \n
  • Allocate a read cache for the disk layer so that contracts accessing the same ancient slot over and over don’t cause disk hits.
    • \n
  • Use cumulative bloom filters in the in-memory diff layers to quickly detect whether there’s a chance for an item to be in the diffs, or if we can go to disk immediately.
    • \n
  • The keys are not the raw data (account address, storage key), rather the hashes of these, ensuring the snapshot has the same iteration order as the Merkle Patricia tree.
    • \n
\n

The snapshot also enables blazing fast state iteration of the most recent blocks. This was actually the main reason for building snapshots, as it permitted the creation of the new snap sync algorithm.

\n

Consensus layer syncing

all consensus logic and block propagation is handled by consensus clients. Blocks are downloaded by the consensus client and verified by the execution client. Geth cannot sync without being connected to a consensus client.
There are two ways for the consensus client to find a block header that Geth can use as a sync target: optimistic syncing and checkpoint syncing:

\n

optimistic sync

Optimistic sync downloads blocks before the execution client has validated them. In optimistic sync the node assumes the data it receives from its peers is correct during the downloading phase but then retroactively verifies each downloaded block.
more details

\n

checkpoint sync

Alternatively, the consensus client can grab a checkpoint from a trusted source which provides a target state to sync up to, before switching to full sync and verifying each block in turn. In this mode, the node trusts that the checkpoint is correct.

\n

archive nodes

An archive node is a node that retains all historical data right back to genesis. There is no need to regenerate any data from checkpoints because all data is directly available in the node’s own storage.

\n

It is also possible to create a partial/recent archive node where the node was synced using snap but the state is never pruned. This creates an archive node that saves all state data from the point that the node first syncs. This is configured by starting Geth with --syncmode snap --gcmode archive.

\n

light nodes

A light node syncs very quickly and stores the bare minimum of blockchain data. Light nodes only process block headers, not entire blocks. they receive a proof from the full node and verify it against their local header chain. Light nodes are not currently working on proof-of-stake Ethereum.

\n

full node

full

A full block-by-block sync generates the current state by executing every block starting from the genesis block. A full sync independently verifies block provenance as well as all state transitions by re-executing the transactions in the entire historical sequence of blocks. Only the most recent 128 block states are stored in a full node - older block states are pruned periodically and represented as a series of checkpoints from which any previous state can be regenerated on request.

\n

snap sync (default)

Snap sync starts from a relatively recent block and syncs from there to the head of the chain, keeping only the most recent 128 block states in memory. The block header to sync up to is provided by the consensus client. Between the initial sync block and the 128 most recent blocks, the node stores occasional snapshots that can be used to rebuild any intermediate state “on-the-fly”. The difference between the snap-synced node and a full block-by-block synced node is that a snap synced node started from an initial checkpoint that was more recent than the genesis block. Snap sync is much faster than a full block-by-block sync from genesis.
\"sync

\n

Snap sync works by first downloading the headers for a chunk of blocks. Once the headers have been verified, the block bodies and receipts for those blocks are downloaded. In parallel, Geth also begins state-sync. In state-sync, Geth first downloads the leaves of the state trie for each block without the intermediate nodes along with a range proof. The state trie is then regenerated locally.

\n

The state download is the part of the snap-sync that takes the most time to complete and the progress can be monitored using the ETA values in the log messages. However, the blockchain is also progressing at the same time and invalidating some of the regenerated state data. (don’t really understand why regenrated state could be invalidated). This means it is also necessary to have a ‘healing’ phase where errors in the state are fixed. Geth regularly reports Syncing, state heal in progress during state healing - this informs the user that state heal has not finished.

\n

The healing has to outpace the growth of the blockchain, otherwise the node will never catch up to the current state.

\n

To summarize, snap sync progresses in the following sequence:

\n
    \n
  • download and verify headers
    • \n
  • download block bodies and receipts. In parallel, download raw state data and build state trie
    • \n
  • heal state trie to account for newly arriving data
    • \n
\n

A node that is started using snap will switch to block-by-block sync once it has caught up to the head of the chain.

\n

references

\n"},{"title":"zkp demystify zokrates","date":"2023-07-14T06:29:26.000Z","_content":"\n\n\n## pipeline\n### source file\nan example, `square.zok`\n```\ndef main(private field a, field b) {\n assert(a * a == b);\n return;\n}\n```\n- The keyword private signals that we do not want to reveal this input, but still prove that we know its value.\n### compile\n```\nzokrates compile -i root.zok\n```\nafter compile, it generates below files\n- out\n- out.r1cs\n- abi.json\n### setup\nPerforms a trusted setup for a given constraint system\n```\nzokrates setup\n```\noptions \n- -i, --input Path of the binary [default: out]\nit generates below two files\n- proving.key\n- verification.key","source":"_posts/cryptography/zkp/zkp-demystify-zokrates.md","raw":"---\ntitle: zkp demystify zokrates\ndate: 2023-07-14 14:29:26\ntags: [cryptography,zkp]\n---\n\n\n\n## pipeline\n### source file\nan example, `square.zok`\n```\ndef main(private field a, field b) {\n assert(a * a == b);\n return;\n}\n```\n- The keyword private signals that we do not want to reveal this input, but still prove that we know its value.\n### compile\n```\nzokrates compile -i root.zok\n```\nafter compile, it generates below files\n- out\n- out.r1cs\n- abi.json\n### setup\nPerforms a trusted setup for a given constraint system\n```\nzokrates setup\n```\noptions \n- -i, --input Path of the binary [default: out]\nit generates below two files\n- proving.key\n- verification.key","slug":"cryptography/zkp/zkp-demystify-zokrates","published":1,"updated":"2023-11-05T04:21:17.035Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8e00039qwsjfxurcxan","content":"\n\n\n

pipeline

source file

an example, square.zok

\n
1
2
3
4
def main(private field a, field b) {
    assert(a * a == b);
    return;
}
\n
    \n
  • The keyword private signals that we do not want to reveal this input, but still prove that we know its value.
    • \n
\n

compile

1
zokrates compile -i root.zok
\n

after compile, it generates below files

\n
    \n
  • out
    • \n
  • out.r1cs
    • \n
  • abi.json
    • \n
\n

setup

Performs a trusted setup for a given constraint system

\n
1
zokrates setup
\n

options

\n
    \n
  • -i, –input Path of the binary [default: out]
    it generates below two files
    • \n
  • proving.key
    • \n
  • verification.key
    • \n
\n","site":{"data":{}},"excerpt":"","more":"\n\n\n

pipeline

source file

an example, square.zok

\n
1
2
3
4
def main(private field a, field b) {
    assert(a * a == b);
    return;
}
\n
    \n
  • The keyword private signals that we do not want to reveal this input, but still prove that we know its value.
    • \n
\n

compile

1
zokrates compile -i root.zok
\n

after compile, it generates below files

\n
    \n
  • out
    • \n
  • out.r1cs
    • \n
  • abi.json
    • \n
\n

setup

Performs a trusted setup for a given constraint system

\n
1
zokrates setup
\n

options

\n
    \n
  • -i, –input Path of the binary [default: out]
    it generates below two files
    • \n
  • proving.key
    • \n
  • verification.key
    • \n
\n"},{"title":"zkp a brief understanding (1)","date":"2023-06-27T06:29:26.000Z","_content":"\n\n\n## introduction\nzk-SNARKs cannot be applied to any computational problem directly; rather, you have to convert the problem into the right “form” for the problem to operate on. The form is called a “quadratic arithmetic program” (QAP), and transforming the code of a function into one of these is itself highly nontrivial.\n\nThe example that we will choose is a simple one: proving that you know the solution to a cubic equation: \\\\(x^3 + x + 5 == 35\\\\)\n\nNote that modulo (%) and comparison operators (<, >, ≤, ≥) are NOT supported, as there is no efficient way to do modulo or comparison directly in finite cyclic group arithmetic (be thankful for this; if there was a way to do either one, then elliptic curve cryptography would be broken faster)\n\nYou can extend the language to modulo and comparisons by providing bit decompositions (eg. \\\\(13 = 2^3 + 2^2 + 1\\\\)) as auxiliary inputs, proving correctness of those decompositions and doing the math in binary circuits; in finite field arithmetic, doing equality `(==)` checks is also doable and in fact a bit easier, but these are both details we won’t get into right now. We can extend the language to support conditionals (eg. `if x < 5: y = 7; else: y = 9`) by converting them to an arithmetic form: `y = 7 * (x < 5) + 9 * (x >= 5)`;though note that both “paths” of the conditional would need to be executed.\n\n## Flattening\nThe first step is a “flattening” procedure, where we convert the original code into a sequence of statements that are of two forms\n```\nsym1 = x * x\ny = sym1 * x\nsym2 = y + x\n~out = sym2 + 5\n```\n\n## Gates to R1CS\nNow, we convert this into something called a rank-1 constraint system (R1CS). An R1CS is a sequence of groups of three vectors `(a, b, c)`, and the solution to an R1CS is a vector s, where s must satisfy the equation `s . a * s . b - s . c = 0`, where `.` represents the dot product. For example, this is a satisfied R1CS:\n![r1cs](/images/cryptography/zkp/r1cs.png)\n\\\\[s \\cdot a = (1,3,35,9,27,30) \\cdot (5,0,0,0,0,1) = 35\\\\]\n\\\\[s \\cdot b = (1,3,35,9,27,30) \\cdot (1,0,0,0,0,0) = 1\\\\]\n\\\\[s \\cdot c = (1,3,35,9,27,30) \\cdot (0,0,1,0,0,0) = 35\\\\]\nHence \n\\\\[ s \\cdot a * s \\cdot b - s \\cdot c = 35 * 1 - 35 = 0\\\\]\n\nBut instead of having just one constraint, we are going to have many constraints: one for each logic gate. There is a standard way of converting a logic gate into a `(a, b, c)` triple depending on what the operation is `(+, -, * or /)` and whether the arguments are variables or numbers. The length of each vector is equal to the total number of variables in the system, including a dummy variable `~one` at the first index representing the number 1, the input variables `x`, a dummy variable `~out` representing the output, and then all of the intermediate variables (`sym1` and `sym2` above);\nFirst, we’ll provide the variable mapping that we’ll use:\n`'~one', 'x', '~out', 'sym1', 'y', 'sym2'`\n\nNow, we’ll give the (a, b, c) triple for the first gate:\n```\na = [0, 1, 0, 0, 0, 0]\nb = [0, 1, 0, 0, 0, 0]\nc = [0, 0, 0, 1, 0, 0]\n```\nwhich is \\\\(x*x -sym1 = 0\\\\)\n\nNow, let’s go on to the second gate:\n```\na = [0, 0, 0, 1, 0, 0]\nb = [0, 1, 0, 0, 0, 0]\nc = [0, 0, 0, 0, 1, 0]\n```\nwhich is \\\\(sym1 * x = y\\\\)\nNow, the third gate:\n```\na = [0, 1, 0, 0, 1, 0]\nb = [1, 0, 0, 0, 0, 0]\nc = [0, 0, 0, 0, 0, 1]\n```\nwhich is \\\\( (x+y) * \\sim one = sym2\\\\)\n\nFinally, the fourth gate:\n```\na = [5, 0, 0, 0, 0, 1]\nb = [1, 0, 0, 0, 0, 0]\nc = [0, 0, 1, 0, 0, 0]\n```\nwhich is \\\\((5 + sym2) * \\sim one = \\sim out\\\\)\nAnd there we have our R1CS with four constraints. The witness is simply the assignment to all the variables, including input, output and internal variables:\n`[1, 3, 35, 9, 27, 30]`\n\n\nThe complete R1CS put together is:\n```\nA\n[0, 1, 0, 0, 0, 0]\n[0, 0, 0, 1, 0, 0]\n[0, 1, 0, 0, 1, 0]\n[5, 0, 0, 0, 0, 1]\nB\n[0, 1, 0, 0, 0, 0]\n[0, 1, 0, 0, 0, 0]\n[1, 0, 0, 0, 0, 0]\n[1, 0, 0, 0, 0, 0]\nC\n[0, 0, 0, 1, 0, 0]\n[0, 0, 0, 0, 1, 0]\n[0, 0, 0, 0, 0, 1]\n[0, 0, 1, 0, 0, 0]\n```\n\n## R1CS to QAP\nThe next step is taking this R1CS and converting it into QAP form, which implements the exact same logic except using polynomials instead of dot products. We do this as follows. We go from four groups of three vectors of length six to six groups of three degree-3 polynomials, where evaluating the polynomials at each x coordinate represents one of the constraints. That is, if we evaluate the polynomials at x=1, then we get our first set of vectors, if we evaluate the polynomials at x=2, then we get our second set of vectors, and so on.\n\nWe can make this transformation using something called a **Lagrange interpolation**. \n![lagrange interpolating](/images/cryptography/zkp/lagrange_interpolating.png)\n\nNow, let’s use Lagrange interpolation to transform our R1CS. What we are going to do is take the first value out of every a vector, use Lagrange interpolation to make a polynomial out of that (where evaluating the polynomial at i gets you the first value of the ith a vector), repeat the process for the first value of every b and c vector, and then repeat that process for the second values, the third, values, and so on. For convenience I'll provide the answers right now:\n\n> Note: the intuition here is to think the R1CS A,B,C matrix vertically (column). For example, for the first column of A, the polynomial should pass (1,0), (2,0), (3,0), (4,5); the second polynomial shoudd pass (1,1), (2,0), (3,1), (4,)\n```\nA polynomials\n[-5.0, 9.166, -5.0, 0.833]\n[8.0, -11.333, 5.0, -0.666]\n[0.0, 0.0, 0.0, 0.0]\n[-6.0, 9.5, -4.0, 0.5]\n[4.0, -7.0, 3.5, -0.5]\n[-1.0, 1.833, -1.0, 0.166]\nB polynomials\n[3.0, -5.166, 2.5, -0.333]\n[-2.0, 5.166, -2.5, 0.333]\n[0.0, 0.0, 0.0, 0.0]\n[0.0, 0.0, 0.0, 0.0]\n[0.0, 0.0, 0.0, 0.0]\n[0.0, 0.0, 0.0, 0.0]\nC polynomials\n[0.0, 0.0, 0.0, 0.0]\n[0.0, 0.0, 0.0, 0.0]\n[-1.0, 1.833, -1.0, 0.166]\n[4.0, -4.333, 1.5, -0.166]\n[-6.0, 9.5, -4.0, 0.5]\n[4.0, -7.0, 3.5, -0.5]\n```\nCoefficients are in ascending order, so the first polynomial above is actually \\\\(0.833 x^3 — 5 x^2 + 9.166 x - 5\\\\)\nLet’s try evaluating all of these polynomials at x=1. \n```\nA results at x=1\n0 = 0.833 * 1^3 - 5 * 1^2 + 9.166 * 1^1 -5 * 1^0 = 0.833 -5 + 9.166 -5 = 0\n1 = -0.666 * 1^3 - 5.0 * 1^2 + -11.333 * 1^1 + 8.0 * 1^0 = -0.666 + 5.0 -11.333 +8.0 = 1\n0\n0\n0\n0\nB results at x=1\n0 = -0.333 * 1^3 +2.5 * 1^2 -5.166 * 1^1 +3.0 * 1^0 = -0.333 +2.5 -5.166 +3.0 = 0\n1\n0\n0\n0\n0\nC results at x=1\n0\n0\n0\n1\n0\n0\n```\n\n## Checking the QAP\nNow what’s the point of this crazy transformation? The answer is that instead of checking the constraints in the R1CS individually, we can now check all of the constraints at the same time by doing the dot product check on the polynomials.\n![checking qap](/images/cryptography/zkp/checking_qap.png)\nBecause in this case the dot product check is a series of additions and multiplications of polynomials, the result is itself going to be a polynomial. If the resulting polynomial, evaluated at every x coordinate that we used above to represent a logic gate, is equal to zero, then that means that all of the checks pass; if the resulting polynomial evaluated at at least one of the x coordinate representing a logic gate gives a nonzero value, then that means that the values going into and out of that logic gate are inconsistent\n\nTo check correctness, we don’t actually evaluate the polynomial `t = A . s * B . s - C . s` at every point corresponding to a gate; instead, we divide `t` by another polynomial, `Z`, and check that `Z` evenly divides `t` - that is, **the division `t / Z` leaves no remainder**.\n\n`Z` is defined as `(x - 1) * (x - 2) * (x - 3) ...` - the simplest polynomial that is equal to zero at all points that correspond to logic gates. It is an elementary fact of algebra that any polynomial that is equal to zero at all of these points has to be a multiple of this minimal polynomial, and if a polynomial is a multiple of `Z` then its evaluation at any of those points will be zero;\n\nNote that the above is a simplification; “in the real world”, the addition, multiplication, subtraction and division will happen not with regular numbers, but rather with finite field elements — a spooky kind of arithmetic which is self-consistent, so all the algebraic laws we know and love still hold true, but where all answers are elements of some finite-sized set\n\n## KEA (Knowledge of Exponent Assumption)\nLet \\\\(q\\\\) be a prime such that \\\\(2q+1\\\\) is also prime, and let \\\\(g\\\\) be a generator\nof the order \\\\(q\\\\) subgroup of \\\\(Z_{2q+1}^{\\ast}\\\\). Suppose we are given input \\\\(q, g, g^a\\\\) and want to output a pair \\\\((C, Y )\\\\) such that \\\\(Y = C^a\\\\). One way to do this is to pick some \\\\(c \\in Z_{q}\\\\), let \\\\(C = g^c\\\\), and let \\\\(Y = (g^a)^c\\\\). Intuitively, `KEA1`` can be viewed as saying that this is the 'only' way to produce such a pair. The assumption captures this by saying that any adversary outputting such a pair must “know” an exponent \\\\(c\\\\) such that \\\\(g^c = C\\\\). The formalization asks that there be an “extractor” that can return \\\\(c\\\\). Roughly:\n***\n**KEA1**\nFor any adversary \\\\(A\\\\) that takes input \\\\(q, g,g^a\\\\) and returns \\\\((C,Y)\\\\) with \\\\(Y = C^a\\\\), there exists an 'extractor' \\\\(\\bar{A}\\\\), which given the same inputs as \\\\(A\\\\) returns \\\\(c\\\\) such that \\\\(g^c = C\\\\)\n***\n## reference\n- [vitalik's blog: qap zero to hero](https://medium.com/@VitalikButerin/quadratic-arithmetic-programs-from-zero-to-hero-f6d558cea649)\n- [lagrange interpolating](https://mathworld.wolfram.com/LagrangeInterpolatingPolynomial.html)\n- [zkSNARKs in a nutshell](https://blog.ethereum.org/2016/12/05/zksnarks-in-a-nutshell)\n- [Pinocchio protocol by Parno, Gentry, Howell](https://eprint.iacr.org/2013/279.pdf)\n- [KEA](https://eprint.iacr.org/2004/008.pdf)","source":"_posts/cryptography/zkp/zkp-a-brief-understanding.md","raw":"---\ntitle: zkp a brief understanding (1)\ndate: 2023-06-27 14:29:26\ntags: [cryptography,zkp]\n---\n\n\n\n## introduction\nzk-SNARKs cannot be applied to any computational problem directly; rather, you have to convert the problem into the right “form” for the problem to operate on. The form is called a “quadratic arithmetic program” (QAP), and transforming the code of a function into one of these is itself highly nontrivial.\n\nThe example that we will choose is a simple one: proving that you know the solution to a cubic equation: \\\\(x^3 + x + 5 == 35\\\\)\n\nNote that modulo (%) and comparison operators (<, >, ≤, ≥) are NOT supported, as there is no efficient way to do modulo or comparison directly in finite cyclic group arithmetic (be thankful for this; if there was a way to do either one, then elliptic curve cryptography would be broken faster)\n\nYou can extend the language to modulo and comparisons by providing bit decompositions (eg. \\\\(13 = 2^3 + 2^2 + 1\\\\)) as auxiliary inputs, proving correctness of those decompositions and doing the math in binary circuits; in finite field arithmetic, doing equality `(==)` checks is also doable and in fact a bit easier, but these are both details we won’t get into right now. We can extend the language to support conditionals (eg. `if x < 5: y = 7; else: y = 9`) by converting them to an arithmetic form: `y = 7 * (x < 5) + 9 * (x >= 5)`;though note that both “paths” of the conditional would need to be executed.\n\n## Flattening\nThe first step is a “flattening” procedure, where we convert the original code into a sequence of statements that are of two forms\n```\nsym1 = x * x\ny = sym1 * x\nsym2 = y + x\n~out = sym2 + 5\n```\n\n## Gates to R1CS\nNow, we convert this into something called a rank-1 constraint system (R1CS). An R1CS is a sequence of groups of three vectors `(a, b, c)`, and the solution to an R1CS is a vector s, where s must satisfy the equation `s . a * s . b - s . c = 0`, where `.` represents the dot product. For example, this is a satisfied R1CS:\n![r1cs](/images/cryptography/zkp/r1cs.png)\n\\\\[s \\cdot a = (1,3,35,9,27,30) \\cdot (5,0,0,0,0,1) = 35\\\\]\n\\\\[s \\cdot b = (1,3,35,9,27,30) \\cdot (1,0,0,0,0,0) = 1\\\\]\n\\\\[s \\cdot c = (1,3,35,9,27,30) \\cdot (0,0,1,0,0,0) = 35\\\\]\nHence \n\\\\[ s \\cdot a * s \\cdot b - s \\cdot c = 35 * 1 - 35 = 0\\\\]\n\nBut instead of having just one constraint, we are going to have many constraints: one for each logic gate. There is a standard way of converting a logic gate into a `(a, b, c)` triple depending on what the operation is `(+, -, * or /)` and whether the arguments are variables or numbers. The length of each vector is equal to the total number of variables in the system, including a dummy variable `~one` at the first index representing the number 1, the input variables `x`, a dummy variable `~out` representing the output, and then all of the intermediate variables (`sym1` and `sym2` above);\nFirst, we’ll provide the variable mapping that we’ll use:\n`'~one', 'x', '~out', 'sym1', 'y', 'sym2'`\n\nNow, we’ll give the (a, b, c) triple for the first gate:\n```\na = [0, 1, 0, 0, 0, 0]\nb = [0, 1, 0, 0, 0, 0]\nc = [0, 0, 0, 1, 0, 0]\n```\nwhich is \\\\(x*x -sym1 = 0\\\\)\n\nNow, let’s go on to the second gate:\n```\na = [0, 0, 0, 1, 0, 0]\nb = [0, 1, 0, 0, 0, 0]\nc = [0, 0, 0, 0, 1, 0]\n```\nwhich is \\\\(sym1 * x = y\\\\)\nNow, the third gate:\n```\na = [0, 1, 0, 0, 1, 0]\nb = [1, 0, 0, 0, 0, 0]\nc = [0, 0, 0, 0, 0, 1]\n```\nwhich is \\\\( (x+y) * \\sim one = sym2\\\\)\n\nFinally, the fourth gate:\n```\na = [5, 0, 0, 0, 0, 1]\nb = [1, 0, 0, 0, 0, 0]\nc = [0, 0, 1, 0, 0, 0]\n```\nwhich is \\\\((5 + sym2) * \\sim one = \\sim out\\\\)\nAnd there we have our R1CS with four constraints. The witness is simply the assignment to all the variables, including input, output and internal variables:\n`[1, 3, 35, 9, 27, 30]`\n\n\nThe complete R1CS put together is:\n```\nA\n[0, 1, 0, 0, 0, 0]\n[0, 0, 0, 1, 0, 0]\n[0, 1, 0, 0, 1, 0]\n[5, 0, 0, 0, 0, 1]\nB\n[0, 1, 0, 0, 0, 0]\n[0, 1, 0, 0, 0, 0]\n[1, 0, 0, 0, 0, 0]\n[1, 0, 0, 0, 0, 0]\nC\n[0, 0, 0, 1, 0, 0]\n[0, 0, 0, 0, 1, 0]\n[0, 0, 0, 0, 0, 1]\n[0, 0, 1, 0, 0, 0]\n```\n\n## R1CS to QAP\nThe next step is taking this R1CS and converting it into QAP form, which implements the exact same logic except using polynomials instead of dot products. We do this as follows. We go from four groups of three vectors of length six to six groups of three degree-3 polynomials, where evaluating the polynomials at each x coordinate represents one of the constraints. That is, if we evaluate the polynomials at x=1, then we get our first set of vectors, if we evaluate the polynomials at x=2, then we get our second set of vectors, and so on.\n\nWe can make this transformation using something called a **Lagrange interpolation**. \n![lagrange interpolating](/images/cryptography/zkp/lagrange_interpolating.png)\n\nNow, let’s use Lagrange interpolation to transform our R1CS. What we are going to do is take the first value out of every a vector, use Lagrange interpolation to make a polynomial out of that (where evaluating the polynomial at i gets you the first value of the ith a vector), repeat the process for the first value of every b and c vector, and then repeat that process for the second values, the third, values, and so on. For convenience I'll provide the answers right now:\n\n> Note: the intuition here is to think the R1CS A,B,C matrix vertically (column). For example, for the first column of A, the polynomial should pass (1,0), (2,0), (3,0), (4,5); the second polynomial shoudd pass (1,1), (2,0), (3,1), (4,)\n```\nA polynomials\n[-5.0, 9.166, -5.0, 0.833]\n[8.0, -11.333, 5.0, -0.666]\n[0.0, 0.0, 0.0, 0.0]\n[-6.0, 9.5, -4.0, 0.5]\n[4.0, -7.0, 3.5, -0.5]\n[-1.0, 1.833, -1.0, 0.166]\nB polynomials\n[3.0, -5.166, 2.5, -0.333]\n[-2.0, 5.166, -2.5, 0.333]\n[0.0, 0.0, 0.0, 0.0]\n[0.0, 0.0, 0.0, 0.0]\n[0.0, 0.0, 0.0, 0.0]\n[0.0, 0.0, 0.0, 0.0]\nC polynomials\n[0.0, 0.0, 0.0, 0.0]\n[0.0, 0.0, 0.0, 0.0]\n[-1.0, 1.833, -1.0, 0.166]\n[4.0, -4.333, 1.5, -0.166]\n[-6.0, 9.5, -4.0, 0.5]\n[4.0, -7.0, 3.5, -0.5]\n```\nCoefficients are in ascending order, so the first polynomial above is actually \\\\(0.833 x^3 — 5 x^2 + 9.166 x - 5\\\\)\nLet’s try evaluating all of these polynomials at x=1. \n```\nA results at x=1\n0 = 0.833 * 1^3 - 5 * 1^2 + 9.166 * 1^1 -5 * 1^0 = 0.833 -5 + 9.166 -5 = 0\n1 = -0.666 * 1^3 - 5.0 * 1^2 + -11.333 * 1^1 + 8.0 * 1^0 = -0.666 + 5.0 -11.333 +8.0 = 1\n0\n0\n0\n0\nB results at x=1\n0 = -0.333 * 1^3 +2.5 * 1^2 -5.166 * 1^1 +3.0 * 1^0 = -0.333 +2.5 -5.166 +3.0 = 0\n1\n0\n0\n0\n0\nC results at x=1\n0\n0\n0\n1\n0\n0\n```\n\n## Checking the QAP\nNow what’s the point of this crazy transformation? The answer is that instead of checking the constraints in the R1CS individually, we can now check all of the constraints at the same time by doing the dot product check on the polynomials.\n![checking qap](/images/cryptography/zkp/checking_qap.png)\nBecause in this case the dot product check is a series of additions and multiplications of polynomials, the result is itself going to be a polynomial. If the resulting polynomial, evaluated at every x coordinate that we used above to represent a logic gate, is equal to zero, then that means that all of the checks pass; if the resulting polynomial evaluated at at least one of the x coordinate representing a logic gate gives a nonzero value, then that means that the values going into and out of that logic gate are inconsistent\n\nTo check correctness, we don’t actually evaluate the polynomial `t = A . s * B . s - C . s` at every point corresponding to a gate; instead, we divide `t` by another polynomial, `Z`, and check that `Z` evenly divides `t` - that is, **the division `t / Z` leaves no remainder**.\n\n`Z` is defined as `(x - 1) * (x - 2) * (x - 3) ...` - the simplest polynomial that is equal to zero at all points that correspond to logic gates. It is an elementary fact of algebra that any polynomial that is equal to zero at all of these points has to be a multiple of this minimal polynomial, and if a polynomial is a multiple of `Z` then its evaluation at any of those points will be zero;\n\nNote that the above is a simplification; “in the real world”, the addition, multiplication, subtraction and division will happen not with regular numbers, but rather with finite field elements — a spooky kind of arithmetic which is self-consistent, so all the algebraic laws we know and love still hold true, but where all answers are elements of some finite-sized set\n\n## KEA (Knowledge of Exponent Assumption)\nLet \\\\(q\\\\) be a prime such that \\\\(2q+1\\\\) is also prime, and let \\\\(g\\\\) be a generator\nof the order \\\\(q\\\\) subgroup of \\\\(Z_{2q+1}^{\\ast}\\\\). Suppose we are given input \\\\(q, g, g^a\\\\) and want to output a pair \\\\((C, Y )\\\\) such that \\\\(Y = C^a\\\\). One way to do this is to pick some \\\\(c \\in Z_{q}\\\\), let \\\\(C = g^c\\\\), and let \\\\(Y = (g^a)^c\\\\). Intuitively, `KEA1`` can be viewed as saying that this is the 'only' way to produce such a pair. The assumption captures this by saying that any adversary outputting such a pair must “know” an exponent \\\\(c\\\\) such that \\\\(g^c = C\\\\). The formalization asks that there be an “extractor” that can return \\\\(c\\\\). Roughly:\n***\n**KEA1**\nFor any adversary \\\\(A\\\\) that takes input \\\\(q, g,g^a\\\\) and returns \\\\((C,Y)\\\\) with \\\\(Y = C^a\\\\), there exists an 'extractor' \\\\(\\bar{A}\\\\), which given the same inputs as \\\\(A\\\\) returns \\\\(c\\\\) such that \\\\(g^c = C\\\\)\n***\n## reference\n- [vitalik's blog: qap zero to hero](https://medium.com/@VitalikButerin/quadratic-arithmetic-programs-from-zero-to-hero-f6d558cea649)\n- [lagrange interpolating](https://mathworld.wolfram.com/LagrangeInterpolatingPolynomial.html)\n- [zkSNARKs in a nutshell](https://blog.ethereum.org/2016/12/05/zksnarks-in-a-nutshell)\n- [Pinocchio protocol by Parno, Gentry, Howell](https://eprint.iacr.org/2013/279.pdf)\n- [KEA](https://eprint.iacr.org/2004/008.pdf)","slug":"cryptography/zkp/zkp-a-brief-understanding","published":1,"updated":"2023-11-05T04:21:17.034Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8e1003aqwsj8bnledgy","content":"\n\n\n

introduction

zk-SNARKs cannot be applied to any computational problem directly; rather, you have to convert the problem into the right “form” for the problem to operate on. The form is called a “quadratic arithmetic program” (QAP), and transforming the code of a function into one of these is itself highly nontrivial.

\n

The example that we will choose is a simple one: proving that you know the solution to a cubic equation: \\(x^3 + x + 5 == 35\\)

\n

Note that modulo (%) and comparison operators (<, >, ≤, ≥) are NOT supported, as there is no efficient way to do modulo or comparison directly in finite cyclic group arithmetic (be thankful for this; if there was a way to do either one, then elliptic curve cryptography would be broken faster)

\n

You can extend the language to modulo and comparisons by providing bit decompositions (eg. \\(13 = 2^3 + 2^2 + 1\\)) as auxiliary inputs, proving correctness of those decompositions and doing the math in binary circuits; in finite field arithmetic, doing equality (==) checks is also doable and in fact a bit easier, but these are both details we won’t get into right now. We can extend the language to support conditionals (eg. if x < 5: y = 7; else: y = 9) by converting them to an arithmetic form: y = 7 * (x < 5) + 9 * (x >= 5);though note that both “paths” of the conditional would need to be executed.

\n

Flattening

The first step is a “flattening” procedure, where we convert the original code into a sequence of statements that are of two forms

\n
1
2
3
4
sym1 = x * x
y = sym1 * x
sym2 = y + x
~out = sym2 + 5
\n\n

Gates to R1CS

Now, we convert this into something called a rank-1 constraint system (R1CS). An R1CS is a sequence of groups of three vectors (a, b, c), and the solution to an R1CS is a vector s, where s must satisfy the equation s . a * s . b - s . c = 0, where . represents the dot product. For example, this is a satisfied R1CS:
\"r1cs\"
\\[s \\cdot a = (1,3,35,9,27,30) \\cdot (5,0,0,0,0,1) = 35\\]
\\[s \\cdot b = (1,3,35,9,27,30) \\cdot (1,0,0,0,0,0) = 1\\]
\\[s \\cdot c = (1,3,35,9,27,30) \\cdot (0,0,1,0,0,0) = 35\\]
Hence
\\[ s \\cdot a * s \\cdot b - s \\cdot c = 35 * 1 - 35 = 0\\]

\n

But instead of having just one constraint, we are going to have many constraints: one for each logic gate. There is a standard way of converting a logic gate into a (a, b, c) triple depending on what the operation is (+, -, * or /) and whether the arguments are variables or numbers. The length of each vector is equal to the total number of variables in the system, including a dummy variable ~one at the first index representing the number 1, the input variables x, a dummy variable ~out representing the output, and then all of the intermediate variables (sym1 and sym2 above);
First, we’ll provide the variable mapping that we’ll use:
'~one', 'x', '~out', 'sym1', 'y', 'sym2'

\n

Now, we’ll give the (a, b, c) triple for the first gate:

\n
1
2
3
a = [0, 1, 0, 0, 0, 0]
b = [0, 1, 0, 0, 0, 0]
c = [0, 0, 0, 1, 0, 0]
\n

which is \\(x*x -sym1 = 0\\)

\n

Now, let’s go on to the second gate:

\n
1
2
3
a = [0, 0, 0, 1, 0, 0]
b = [0, 1, 0, 0, 0, 0]
c = [0, 0, 0, 0, 1, 0]
\n

which is \\(sym1 * x = y\\)
Now, the third gate:

\n
1
2
3
a = [0, 1, 0, 0, 1, 0]
b = [1, 0, 0, 0, 0, 0]
c = [0, 0, 0, 0, 0, 1]
\n

which is \\( (x+y) * \\sim one = sym2\\)

\n

Finally, the fourth gate:

\n
1
2
3
a = [5, 0, 0, 0, 0, 1]
b = [1, 0, 0, 0, 0, 0]
c = [0, 0, 1, 0, 0, 0]
\n

which is \\((5 + sym2) * \\sim one = \\sim out\\)
And there we have our R1CS with four constraints. The witness is simply the assignment to all the variables, including input, output and internal variables:
[1, 3, 35, 9, 27, 30]

\n

The complete R1CS put together is:

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
A
[0, 1, 0, 0, 0, 0]
[0, 0, 0, 1, 0, 0]
[0, 1, 0, 0, 1, 0]
[5, 0, 0, 0, 0, 1]
B
[0, 1, 0, 0, 0, 0]
[0, 1, 0, 0, 0, 0]
[1, 0, 0, 0, 0, 0]
[1, 0, 0, 0, 0, 0]
C
[0, 0, 0, 1, 0, 0]
[0, 0, 0, 0, 1, 0]
[0, 0, 0, 0, 0, 1]
[0, 0, 1, 0, 0, 0]
\n\n

R1CS to QAP

The next step is taking this R1CS and converting it into QAP form, which implements the exact same logic except using polynomials instead of dot products. We do this as follows. We go from four groups of three vectors of length six to six groups of three degree-3 polynomials, where evaluating the polynomials at each x coordinate represents one of the constraints. That is, if we evaluate the polynomials at x=1, then we get our first set of vectors, if we evaluate the polynomials at x=2, then we get our second set of vectors, and so on.

\n

We can make this transformation using something called a Lagrange interpolation.
\"lagrange

\n

Now, let’s use Lagrange interpolation to transform our R1CS. What we are going to do is take the first value out of every a vector, use Lagrange interpolation to make a polynomial out of that (where evaluating the polynomial at i gets you the first value of the ith a vector), repeat the process for the first value of every b and c vector, and then repeat that process for the second values, the third, values, and so on. For convenience I’ll provide the answers right now:

\n
\n

Note: the intuition here is to think the R1CS A,B,C matrix vertically (column). For example, for the first column of A, the polynomial should pass (1,0), (2,0), (3,0), (4,5); the second polynomial shoudd pass (1,1), (2,0), (3,1), (4,)

\n
\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
A polynomials
[-5.0, 9.166, -5.0, 0.833]
[8.0, -11.333, 5.0, -0.666]
[0.0, 0.0, 0.0, 0.0]
[-6.0, 9.5, -4.0, 0.5]
[4.0, -7.0, 3.5, -0.5]
[-1.0, 1.833, -1.0, 0.166]
B polynomials
[3.0, -5.166, 2.5, -0.333]
[-2.0, 5.166, -2.5, 0.333]
[0.0, 0.0, 0.0, 0.0]
[0.0, 0.0, 0.0, 0.0]
[0.0, 0.0, 0.0, 0.0]
[0.0, 0.0, 0.0, 0.0]
C polynomials
[0.0, 0.0, 0.0, 0.0]
[0.0, 0.0, 0.0, 0.0]
[-1.0, 1.833, -1.0, 0.166]
[4.0, -4.333, 1.5, -0.166]
[-6.0, 9.5, -4.0, 0.5]
[4.0, -7.0, 3.5, -0.5]
\n

Coefficients are in ascending order, so the first polynomial above is actually \\(0.833 x^3 — 5 x^2 + 9.166 x - 5\\)
Let’s try evaluating all of these polynomials at x=1.

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
A results at x=1
0 = 0.833 * 1^3 - 5 * 1^2 + 9.166 * 1^1 -5 * 1^0 = 0.833 -5 + 9.166 -5 = 0
1 = -0.666 * 1^3 - 5.0 * 1^2 + -11.333 * 1^1 + 8.0 * 1^0 = -0.666 + 5.0 -11.333 +8.0 = 1
0
0
0
0
B results at x=1
0 = -0.333 * 1^3 +2.5 * 1^2 -5.166 * 1^1 +3.0 * 1^0 = -0.333 +2.5 -5.166 +3.0 = 0
1
0
0
0
0
C results at x=1
0
0
0
1
0
0
\n\n

Checking the QAP

Now what’s the point of this crazy transformation? The answer is that instead of checking the constraints in the R1CS individually, we can now check all of the constraints at the same time by doing the dot product check on the polynomials.
\"checking
Because in this case the dot product check is a series of additions and multiplications of polynomials, the result is itself going to be a polynomial. If the resulting polynomial, evaluated at every x coordinate that we used above to represent a logic gate, is equal to zero, then that means that all of the checks pass; if the resulting polynomial evaluated at at least one of the x coordinate representing a logic gate gives a nonzero value, then that means that the values going into and out of that logic gate are inconsistent

\n

To check correctness, we don’t actually evaluate the polynomial t = A . s * B . s - C . s at every point corresponding to a gate; instead, we divide t by another polynomial, Z, and check that Z evenly divides t - that is, the division t / Z leaves no remainder.

\n

Z is defined as (x - 1) * (x - 2) * (x - 3) ... - the simplest polynomial that is equal to zero at all points that correspond to logic gates. It is an elementary fact of algebra that any polynomial that is equal to zero at all of these points has to be a multiple of this minimal polynomial, and if a polynomial is a multiple of Z then its evaluation at any of those points will be zero;

\n

Note that the above is a simplification; “in the real world”, the addition, multiplication, subtraction and division will happen not with regular numbers, but rather with finite field elements — a spooky kind of arithmetic which is self-consistent, so all the algebraic laws we know and love still hold true, but where all answers are elements of some finite-sized set

\n

KEA (Knowledge of Exponent Assumption)

Let \\(q\\) be a prime such that \\(2q+1\\) is also prime, and let \\(g\\) be a generator
of the order \\(q\\) subgroup of \\(Z_{2q+1}^{\\ast}\\). Suppose we are given input \\(q, g, g^a\\) and want to output a pair \\((C, Y )\\) such that \\(Y = C^a\\). One way to do this is to pick some \\(c \\in Z_{q}\\), let \\(C = g^c\\), and let \\(Y = (g^a)^c\\). Intuitively, `KEA1`` can be viewed as saying that this is the ‘only’ way to produce such a pair. The assumption captures this by saying that any adversary outputting such a pair must “know” an exponent \\(c\\) such that \\(g^c = C\\). The formalization asks that there be an “extractor” that can return \\(c\\). Roughly:

\n
\n

KEA1
For any adversary \\(A\\) that takes input \\(q, g,g^a\\) and returns \\((C,Y)\\) with \\(Y = C^a\\), there exists an ‘extractor’ \\(\\bar{A}\\), which given the same inputs as \\(A\\) returns \\(c\\) such that \\(g^c = C\\)

\n
\n

reference

\n","site":{"data":{}},"excerpt":"","more":"\n\n\n

introduction

zk-SNARKs cannot be applied to any computational problem directly; rather, you have to convert the problem into the right “form” for the problem to operate on. The form is called a “quadratic arithmetic program” (QAP), and transforming the code of a function into one of these is itself highly nontrivial.

\n

The example that we will choose is a simple one: proving that you know the solution to a cubic equation: \\(x^3 + x + 5 == 35\\)

\n

Note that modulo (%) and comparison operators (<, >, ≤, ≥) are NOT supported, as there is no efficient way to do modulo or comparison directly in finite cyclic group arithmetic (be thankful for this; if there was a way to do either one, then elliptic curve cryptography would be broken faster)

\n

You can extend the language to modulo and comparisons by providing bit decompositions (eg. \\(13 = 2^3 + 2^2 + 1\\)) as auxiliary inputs, proving correctness of those decompositions and doing the math in binary circuits; in finite field arithmetic, doing equality (==) checks is also doable and in fact a bit easier, but these are both details we won’t get into right now. We can extend the language to support conditionals (eg. if x < 5: y = 7; else: y = 9) by converting them to an arithmetic form: y = 7 * (x < 5) + 9 * (x >= 5);though note that both “paths” of the conditional would need to be executed.

\n

Flattening

The first step is a “flattening” procedure, where we convert the original code into a sequence of statements that are of two forms

\n
1
2
3
4
sym1 = x * x
y = sym1 * x
sym2 = y + x
~out = sym2 + 5
\n\n

Gates to R1CS

Now, we convert this into something called a rank-1 constraint system (R1CS). An R1CS is a sequence of groups of three vectors (a, b, c), and the solution to an R1CS is a vector s, where s must satisfy the equation s . a * s . b - s . c = 0, where . represents the dot product. For example, this is a satisfied R1CS:
\"r1cs\"
\\[s \\cdot a = (1,3,35,9,27,30) \\cdot (5,0,0,0,0,1) = 35\\]
\\[s \\cdot b = (1,3,35,9,27,30) \\cdot (1,0,0,0,0,0) = 1\\]
\\[s \\cdot c = (1,3,35,9,27,30) \\cdot (0,0,1,0,0,0) = 35\\]
Hence
\\[ s \\cdot a * s \\cdot b - s \\cdot c = 35 * 1 - 35 = 0\\]

\n

But instead of having just one constraint, we are going to have many constraints: one for each logic gate. There is a standard way of converting a logic gate into a (a, b, c) triple depending on what the operation is (+, -, * or /) and whether the arguments are variables or numbers. The length of each vector is equal to the total number of variables in the system, including a dummy variable ~one at the first index representing the number 1, the input variables x, a dummy variable ~out representing the output, and then all of the intermediate variables (sym1 and sym2 above);
First, we’ll provide the variable mapping that we’ll use:
'~one', 'x', '~out', 'sym1', 'y', 'sym2'

\n

Now, we’ll give the (a, b, c) triple for the first gate:

\n
1
2
3
a = [0, 1, 0, 0, 0, 0]
b = [0, 1, 0, 0, 0, 0]
c = [0, 0, 0, 1, 0, 0]
\n

which is \\(x*x -sym1 = 0\\)

\n

Now, let’s go on to the second gate:

\n
1
2
3
a = [0, 0, 0, 1, 0, 0]
b = [0, 1, 0, 0, 0, 0]
c = [0, 0, 0, 0, 1, 0]
\n

which is \\(sym1 * x = y\\)
Now, the third gate:

\n
1
2
3
a = [0, 1, 0, 0, 1, 0]
b = [1, 0, 0, 0, 0, 0]
c = [0, 0, 0, 0, 0, 1]
\n

which is \\( (x+y) * \\sim one = sym2\\)

\n

Finally, the fourth gate:

\n
1
2
3
a = [5, 0, 0, 0, 0, 1]
b = [1, 0, 0, 0, 0, 0]
c = [0, 0, 1, 0, 0, 0]
\n

which is \\((5 + sym2) * \\sim one = \\sim out\\)
And there we have our R1CS with four constraints. The witness is simply the assignment to all the variables, including input, output and internal variables:
[1, 3, 35, 9, 27, 30]

\n

The complete R1CS put together is:

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
A
[0, 1, 0, 0, 0, 0]
[0, 0, 0, 1, 0, 0]
[0, 1, 0, 0, 1, 0]
[5, 0, 0, 0, 0, 1]
B
[0, 1, 0, 0, 0, 0]
[0, 1, 0, 0, 0, 0]
[1, 0, 0, 0, 0, 0]
[1, 0, 0, 0, 0, 0]
C
[0, 0, 0, 1, 0, 0]
[0, 0, 0, 0, 1, 0]
[0, 0, 0, 0, 0, 1]
[0, 0, 1, 0, 0, 0]
\n\n

R1CS to QAP

The next step is taking this R1CS and converting it into QAP form, which implements the exact same logic except using polynomials instead of dot products. We do this as follows. We go from four groups of three vectors of length six to six groups of three degree-3 polynomials, where evaluating the polynomials at each x coordinate represents one of the constraints. That is, if we evaluate the polynomials at x=1, then we get our first set of vectors, if we evaluate the polynomials at x=2, then we get our second set of vectors, and so on.

\n

We can make this transformation using something called a Lagrange interpolation.
\"lagrange

\n

Now, let’s use Lagrange interpolation to transform our R1CS. What we are going to do is take the first value out of every a vector, use Lagrange interpolation to make a polynomial out of that (where evaluating the polynomial at i gets you the first value of the ith a vector), repeat the process for the first value of every b and c vector, and then repeat that process for the second values, the third, values, and so on. For convenience I’ll provide the answers right now:

\n
\n

Note: the intuition here is to think the R1CS A,B,C matrix vertically (column). For example, for the first column of A, the polynomial should pass (1,0), (2,0), (3,0), (4,5); the second polynomial shoudd pass (1,1), (2,0), (3,1), (4,)

\n
\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
A polynomials
[-5.0, 9.166, -5.0, 0.833]
[8.0, -11.333, 5.0, -0.666]
[0.0, 0.0, 0.0, 0.0]
[-6.0, 9.5, -4.0, 0.5]
[4.0, -7.0, 3.5, -0.5]
[-1.0, 1.833, -1.0, 0.166]
B polynomials
[3.0, -5.166, 2.5, -0.333]
[-2.0, 5.166, -2.5, 0.333]
[0.0, 0.0, 0.0, 0.0]
[0.0, 0.0, 0.0, 0.0]
[0.0, 0.0, 0.0, 0.0]
[0.0, 0.0, 0.0, 0.0]
C polynomials
[0.0, 0.0, 0.0, 0.0]
[0.0, 0.0, 0.0, 0.0]
[-1.0, 1.833, -1.0, 0.166]
[4.0, -4.333, 1.5, -0.166]
[-6.0, 9.5, -4.0, 0.5]
[4.0, -7.0, 3.5, -0.5]
\n

Coefficients are in ascending order, so the first polynomial above is actually \\(0.833 x^3 — 5 x^2 + 9.166 x - 5\\)
Let’s try evaluating all of these polynomials at x=1.

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
A results at x=1
0 = 0.833 * 1^3 - 5 * 1^2 + 9.166 * 1^1 -5 * 1^0 = 0.833 -5 + 9.166 -5 = 0
1 = -0.666 * 1^3 - 5.0 * 1^2 + -11.333 * 1^1 + 8.0 * 1^0 = -0.666 + 5.0 -11.333 +8.0 = 1
0
0
0
0
B results at x=1
0 = -0.333 * 1^3 +2.5 * 1^2 -5.166 * 1^1 +3.0 * 1^0 = -0.333 +2.5 -5.166 +3.0 = 0
1
0
0
0
0
C results at x=1
0
0
0
1
0
0
\n\n

Checking the QAP

Now what’s the point of this crazy transformation? The answer is that instead of checking the constraints in the R1CS individually, we can now check all of the constraints at the same time by doing the dot product check on the polynomials.
\"checking
Because in this case the dot product check is a series of additions and multiplications of polynomials, the result is itself going to be a polynomial. If the resulting polynomial, evaluated at every x coordinate that we used above to represent a logic gate, is equal to zero, then that means that all of the checks pass; if the resulting polynomial evaluated at at least one of the x coordinate representing a logic gate gives a nonzero value, then that means that the values going into and out of that logic gate are inconsistent

\n

To check correctness, we don’t actually evaluate the polynomial t = A . s * B . s - C . s at every point corresponding to a gate; instead, we divide t by another polynomial, Z, and check that Z evenly divides t - that is, the division t / Z leaves no remainder.

\n

Z is defined as (x - 1) * (x - 2) * (x - 3) ... - the simplest polynomial that is equal to zero at all points that correspond to logic gates. It is an elementary fact of algebra that any polynomial that is equal to zero at all of these points has to be a multiple of this minimal polynomial, and if a polynomial is a multiple of Z then its evaluation at any of those points will be zero;

\n

Note that the above is a simplification; “in the real world”, the addition, multiplication, subtraction and division will happen not with regular numbers, but rather with finite field elements — a spooky kind of arithmetic which is self-consistent, so all the algebraic laws we know and love still hold true, but where all answers are elements of some finite-sized set

\n

KEA (Knowledge of Exponent Assumption)

Let \\(q\\) be a prime such that \\(2q+1\\) is also prime, and let \\(g\\) be a generator
of the order \\(q\\) subgroup of \\(Z_{2q+1}^{\\ast}\\). Suppose we are given input \\(q, g, g^a\\) and want to output a pair \\((C, Y )\\) such that \\(Y = C^a\\). One way to do this is to pick some \\(c \\in Z_{q}\\), let \\(C = g^c\\), and let \\(Y = (g^a)^c\\). Intuitively, `KEA1`` can be viewed as saying that this is the ‘only’ way to produce such a pair. The assumption captures this by saying that any adversary outputting such a pair must “know” an exponent \\(c\\) such that \\(g^c = C\\). The formalization asks that there be an “extractor” that can return \\(c\\). Roughly:

\n
\n

KEA1
For any adversary \\(A\\) that takes input \\(q, g,g^a\\) and returns \\((C,Y)\\) with \\(Y = C^a\\), there exists an ‘extractor’ \\(\\bar{A}\\), which given the same inputs as \\(A\\) returns \\(c\\) such that \\(g^c = C\\)

\n
\n

reference

\n"},{"title":"zkp groth16 paper review","date":"2023-07-07T06:29:26.000Z","_content":"\n\n## 1. Introduction\nGoldwasser, Micali and Rackoff [GMR89] introduced zero-knowledge proofs that enable a prover to convince a verifier that a statement is true without revealing anything else. They have three core properties:\n- **Completeness**: Given a statement and a witness, the prover can convince the verifier. \n- **Soundness**: A malicious prover cannot convince the verifier of a false statement. \n- **Zero-knowledge**: The proof does not reveal anything but the truth of the statement\n\n### 1.1. Contribution\n**Succinct NIZK** We construct a NIZK argument for arithmetic circuit satisfiability where a proof consists of only 3 group elements. In addition to being small, the proof is also easy to verify. The verifier just needs to compute a number of exponentiations proportional to the statement size and check a single pairing product equation, which only has 3 pairings.\n\n\n## 2. Preliminaries\n### 2.1 Bilinear Groups\nWe will work over bilinear groups \\\\((p, \\mathbb{G_{1}}, \\mathbb{G_{2}}, \\mathbb{G_{T}} , e, g, h)\\\\) with the following properties:\n- \\\\(\\mathbb{G_{1}}, \\mathbb{G_{2}}, \\mathbb{G_{T}} \\\\)are groups of prime order \\\\(p\\\\)\n- The pairing \\\\( e:\\mathbb{G_{1}} \\times \\mathbb{G_{2}} \\rightarrow \\mathbb{G_{T}}\\\\) is a bilinear map\n- \\\\(g\\\\) is a generator for \\\\(\\mathbb{G_{1}}\\\\), \\\\(h\\\\) is a generator for \\\\(\\mathbb{G_{2}}\\\\), and \\\\(e(g,h)\\\\) is a generator for \\\\(\\mathbb{G_{T}}\\\\)\n\nThere are many ways to set up bilinear groups both as symmetric bilinear groups where \\\\(\\mathbb{G_{1}} = \\mathbb{G_{1}}\\\\) and as asymmetric bilinear groups where \\\\(\\mathbb{G_{1}} \\neq \\mathbb{G_{1}}\\\\). Galbraith, Paterson and Smart [GPS08] classify bilinear groups as **Type I** where \\\\(\\mathbb{G_{1}} = \\mathbb{G_{2}}\\\\), **Type II** where there is an efficiently computable non-trivial homomorphism \\\\(\\Phi : \\mathbb{G_{2}} \\rightarrow \\mathbb{G_{1}}\\\\), and **Type III** where no such efficiently computable homomorphism exists in either direction between \\\\(\\mathbb{G_{1}}\\\\) and \\\\(\\mathbb{G_{2}}\\\\). Type III bilinear groups are the most efficient type of bilinear groups and hence the most relevant for practical applications.\nAs a notation for group elements, we write \\\\( \\lbrack a \\rbrack_{1} \\\\) for \\\\( g^a\\\\), \\\\( \\lbrack b \\rbrack_{2}\\\\) for \\\\( h^b\\\\) and \\\\( \\lbrack c \\rbrack_{T}\\\\) for \\\\(e(g,h)^{c} \\\\). A vector of group elements will be represented as \\\\( \\lbrack \\mathbf{a} \\rbrack_{i} \\\\). Given two vectors of \\\\(n\\\\) group elements \\\\( \\lbrack \\mathbf{a} \\rbrack_{1} \\\\) and \\\\( \\lbrack \\mathbf{b} \\rbrack_{2} \\\\), we define their dot product as \\\\( \\lbrack \\mathbf{a} \\rbrack_{1} \\cdot \\lbrack \\mathbf{b} \\rbrack_{2} = \\lbrack \\mathbf{a} \\cdot \\mathbf{b} \\rbrack_{T} \\\\), which can be efficiently computed using the pairing \\\\(e\\\\).\n\n\n### 2.2 Non-interactive zero-knowledge arguments of knowledge\n\n\n## references\n- [1] groth 16 paper, On the Size of Pairing-based Non-interactive Arguments by Jens Groth","source":"_posts/cryptography/zkp/zkp-groth16-paper-review.md","raw":"---\ntitle: zkp groth16 paper review\ndate: 2023-07-07 14:29:26\ntags: [cryptography,zkp]\n---\n\n\n## 1. Introduction\nGoldwasser, Micali and Rackoff [GMR89] introduced zero-knowledge proofs that enable a prover to convince a verifier that a statement is true without revealing anything else. They have three core properties:\n- **Completeness**: Given a statement and a witness, the prover can convince the verifier. \n- **Soundness**: A malicious prover cannot convince the verifier of a false statement. \n- **Zero-knowledge**: The proof does not reveal anything but the truth of the statement\n\n### 1.1. Contribution\n**Succinct NIZK** We construct a NIZK argument for arithmetic circuit satisfiability where a proof consists of only 3 group elements. In addition to being small, the proof is also easy to verify. The verifier just needs to compute a number of exponentiations proportional to the statement size and check a single pairing product equation, which only has 3 pairings.\n\n\n## 2. Preliminaries\n### 2.1 Bilinear Groups\nWe will work over bilinear groups \\\\((p, \\mathbb{G_{1}}, \\mathbb{G_{2}}, \\mathbb{G_{T}} , e, g, h)\\\\) with the following properties:\n- \\\\(\\mathbb{G_{1}}, \\mathbb{G_{2}}, \\mathbb{G_{T}} \\\\)are groups of prime order \\\\(p\\\\)\n- The pairing \\\\( e:\\mathbb{G_{1}} \\times \\mathbb{G_{2}} \\rightarrow \\mathbb{G_{T}}\\\\) is a bilinear map\n- \\\\(g\\\\) is a generator for \\\\(\\mathbb{G_{1}}\\\\), \\\\(h\\\\) is a generator for \\\\(\\mathbb{G_{2}}\\\\), and \\\\(e(g,h)\\\\) is a generator for \\\\(\\mathbb{G_{T}}\\\\)\n\nThere are many ways to set up bilinear groups both as symmetric bilinear groups where \\\\(\\mathbb{G_{1}} = \\mathbb{G_{1}}\\\\) and as asymmetric bilinear groups where \\\\(\\mathbb{G_{1}} \\neq \\mathbb{G_{1}}\\\\). Galbraith, Paterson and Smart [GPS08] classify bilinear groups as **Type I** where \\\\(\\mathbb{G_{1}} = \\mathbb{G_{2}}\\\\), **Type II** where there is an efficiently computable non-trivial homomorphism \\\\(\\Phi : \\mathbb{G_{2}} \\rightarrow \\mathbb{G_{1}}\\\\), and **Type III** where no such efficiently computable homomorphism exists in either direction between \\\\(\\mathbb{G_{1}}\\\\) and \\\\(\\mathbb{G_{2}}\\\\). Type III bilinear groups are the most efficient type of bilinear groups and hence the most relevant for practical applications.\nAs a notation for group elements, we write \\\\( \\lbrack a \\rbrack_{1} \\\\) for \\\\( g^a\\\\), \\\\( \\lbrack b \\rbrack_{2}\\\\) for \\\\( h^b\\\\) and \\\\( \\lbrack c \\rbrack_{T}\\\\) for \\\\(e(g,h)^{c} \\\\). A vector of group elements will be represented as \\\\( \\lbrack \\mathbf{a} \\rbrack_{i} \\\\). Given two vectors of \\\\(n\\\\) group elements \\\\( \\lbrack \\mathbf{a} \\rbrack_{1} \\\\) and \\\\( \\lbrack \\mathbf{b} \\rbrack_{2} \\\\), we define their dot product as \\\\( \\lbrack \\mathbf{a} \\rbrack_{1} \\cdot \\lbrack \\mathbf{b} \\rbrack_{2} = \\lbrack \\mathbf{a} \\cdot \\mathbf{b} \\rbrack_{T} \\\\), which can be efficiently computed using the pairing \\\\(e\\\\).\n\n\n### 2.2 Non-interactive zero-knowledge arguments of knowledge\n\n\n## references\n- [1] groth 16 paper, On the Size of Pairing-based Non-interactive Arguments by Jens Groth","slug":"cryptography/zkp/zkp-groth16-paper-review","published":1,"updated":"2023-11-05T04:21:17.035Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8e1003cqwsjc3p416e4","content":"\n\n

1. Introduction

Goldwasser, Micali and Rackoff [GMR89] introduced zero-knowledge proofs that enable a prover to convince a verifier that a statement is true without revealing anything else. They have three core properties:

\n
    \n
  • Completeness: Given a statement and a witness, the prover can convince the verifier.
    • \n
  • Soundness: A malicious prover cannot convince the verifier of a false statement.
    • \n
  • Zero-knowledge: The proof does not reveal anything but the truth of the statement
    • \n
\n

1.1. Contribution

Succinct NIZK We construct a NIZK argument for arithmetic circuit satisfiability where a proof consists of only 3 group elements. In addition to being small, the proof is also easy to verify. The verifier just needs to compute a number of exponentiations proportional to the statement size and check a single pairing product equation, which only has 3 pairings.

\n

2. Preliminaries

2.1 Bilinear Groups

We will work over bilinear groups \\((p, \\mathbb{G_{1}}, \\mathbb{G_{2}}, \\mathbb{G_{T}} , e, g, h)\\) with the following properties:

\n
    \n
  • \\(\\mathbb{G_{1}}, \\mathbb{G_{2}}, \\mathbb{G_{T}} \\)are groups of prime order \\(p\\)
    • \n
  • The pairing \\( e:\\mathbb{G_{1}} \\times \\mathbb{G_{2}} \\rightarrow \\mathbb{G_{T}}\\) is a bilinear map
    • \n
  • \\(g\\) is a generator for \\(\\mathbb{G_{1}}\\), \\(h\\) is a generator for \\(\\mathbb{G_{2}}\\), and \\(e(g,h)\\) is a generator for \\(\\mathbb{G_{T}}\\)
    • \n
\n

There are many ways to set up bilinear groups both as symmetric bilinear groups where \\(\\mathbb{G_{1}} = \\mathbb{G_{1}}\\) and as asymmetric bilinear groups where \\(\\mathbb{G_{1}} \\neq \\mathbb{G_{1}}\\). Galbraith, Paterson and Smart [GPS08] classify bilinear groups as Type I where \\(\\mathbb{G_{1}} = \\mathbb{G_{2}}\\), Type II where there is an efficiently computable non-trivial homomorphism \\(\\Phi : \\mathbb{G_{2}} \\rightarrow \\mathbb{G_{1}}\\), and Type III where no such efficiently computable homomorphism exists in either direction between \\(\\mathbb{G_{1}}\\) and \\(\\mathbb{G_{2}}\\). Type III bilinear groups are the most efficient type of bilinear groups and hence the most relevant for practical applications.
As a notation for group elements, we write \\( \\lbrack a \\rbrack_{1} \\) for \\( g^a\\), \\( \\lbrack b \\rbrack_{2}\\) for \\( h^b\\) and \\( \\lbrack c \\rbrack_{T}\\) for \\(e(g,h)^{c} \\). A vector of group elements will be represented as \\( \\lbrack \\mathbf{a} \\rbrack_{i} \\). Given two vectors of \\(n\\) group elements \\( \\lbrack \\mathbf{a} \\rbrack_{1} \\) and \\( \\lbrack \\mathbf{b} \\rbrack_{2} \\), we define their dot product as \\( \\lbrack \\mathbf{a} \\rbrack_{1} \\cdot \\lbrack \\mathbf{b} \\rbrack_{2} = \\lbrack \\mathbf{a} \\cdot \\mathbf{b} \\rbrack_{T} \\), which can be efficiently computed using the pairing \\(e\\).

\n

2.2 Non-interactive zero-knowledge arguments of knowledge

references

    \n
  • [1] groth 16 paper, On the Size of Pairing-based Non-interactive Arguments by Jens Groth
    • \n
\n","site":{"data":{}},"excerpt":"","more":"\n\n

1. Introduction

Goldwasser, Micali and Rackoff [GMR89] introduced zero-knowledge proofs that enable a prover to convince a verifier that a statement is true without revealing anything else. They have three core properties:

\n
    \n
  • Completeness: Given a statement and a witness, the prover can convince the verifier.
    • \n
  • Soundness: A malicious prover cannot convince the verifier of a false statement.
    • \n
  • Zero-knowledge: The proof does not reveal anything but the truth of the statement
    • \n
\n

1.1. Contribution

Succinct NIZK We construct a NIZK argument for arithmetic circuit satisfiability where a proof consists of only 3 group elements. In addition to being small, the proof is also easy to verify. The verifier just needs to compute a number of exponentiations proportional to the statement size and check a single pairing product equation, which only has 3 pairings.

\n

2. Preliminaries

2.1 Bilinear Groups

We will work over bilinear groups \\((p, \\mathbb{G_{1}}, \\mathbb{G_{2}}, \\mathbb{G_{T}} , e, g, h)\\) with the following properties:

\n
    \n
  • \\(\\mathbb{G_{1}}, \\mathbb{G_{2}}, \\mathbb{G_{T}} \\)are groups of prime order \\(p\\)
    • \n
  • The pairing \\( e:\\mathbb{G_{1}} \\times \\mathbb{G_{2}} \\rightarrow \\mathbb{G_{T}}\\) is a bilinear map
    • \n
  • \\(g\\) is a generator for \\(\\mathbb{G_{1}}\\), \\(h\\) is a generator for \\(\\mathbb{G_{2}}\\), and \\(e(g,h)\\) is a generator for \\(\\mathbb{G_{T}}\\)
    • \n
\n

There are many ways to set up bilinear groups both as symmetric bilinear groups where \\(\\mathbb{G_{1}} = \\mathbb{G_{1}}\\) and as asymmetric bilinear groups where \\(\\mathbb{G_{1}} \\neq \\mathbb{G_{1}}\\). Galbraith, Paterson and Smart [GPS08] classify bilinear groups as Type I where \\(\\mathbb{G_{1}} = \\mathbb{G_{2}}\\), Type II where there is an efficiently computable non-trivial homomorphism \\(\\Phi : \\mathbb{G_{2}} \\rightarrow \\mathbb{G_{1}}\\), and Type III where no such efficiently computable homomorphism exists in either direction between \\(\\mathbb{G_{1}}\\) and \\(\\mathbb{G_{2}}\\). Type III bilinear groups are the most efficient type of bilinear groups and hence the most relevant for practical applications.
As a notation for group elements, we write \\( \\lbrack a \\rbrack_{1} \\) for \\( g^a\\), \\( \\lbrack b \\rbrack_{2}\\) for \\( h^b\\) and \\( \\lbrack c \\rbrack_{T}\\) for \\(e(g,h)^{c} \\). A vector of group elements will be represented as \\( \\lbrack \\mathbf{a} \\rbrack_{i} \\). Given two vectors of \\(n\\) group elements \\( \\lbrack \\mathbf{a} \\rbrack_{1} \\) and \\( \\lbrack \\mathbf{b} \\rbrack_{2} \\), we define their dot product as \\( \\lbrack \\mathbf{a} \\rbrack_{1} \\cdot \\lbrack \\mathbf{b} \\rbrack_{2} = \\lbrack \\mathbf{a} \\cdot \\mathbf{b} \\rbrack_{T} \\), which can be efficiently computed using the pairing \\(e\\).

\n

2.2 Non-interactive zero-knowledge arguments of knowledge

references

    \n
  • [1] groth 16 paper, On the Size of Pairing-based Non-interactive Arguments by Jens Groth
    • \n
\n"},{"title":"geth v1.10.0 summary","date":"2023-03-15T08:29:43.000Z","_content":"\n## introduction\ngeth v1.10.0 has been [released](https://github.com/ethereum/go-ethereum/releases/tag/v1.10.0) on Mar 4 2021. this is a late summary of v1.10.0.\n\n## snapshots\nthe snapshot feature reduces the cost of accessing an account from `O(logN)` to `O(1)`. Whilst snapshots do grant us a 10x read performance, EVM execution also writes data, and these writes need to be Merkle proven. The Merkle proof requirement retains the necessity for `O(logN)` disk access on writes. \nProblems it solves\n- **DoS** In 2016, Ethereum sustained its worse DoS attack ever - The [Shanghai Attacks](https://2017.edcon.io/ppt/one/Martin%20Holst%20Swende_The%20%27Shanghai%20%27Attacks_EDCON.pdf) - that lasted about 2-3 months. The attack revolved around bloating Ethereum's state and abusing various underpriced opcodes to grind the network to a halt. After numerous client optimizations and repricing hard forks, the attack was repelled. The root cause still lingers: state access opcodes have a fixed EVM gas cost O(1), but an ever slowly increasing execution cost O(logN). Snapshots on the other hand reduce execution cost of state reads to O(1) - in line with EVM costs - thus solves the read-based DoS issues long term.\n- **Call** Checking a smart contract's state in Ethereum entails a mini EVM execution. Part of that is running bytecode and part of it is reading state slots from disk. snap makes the state access faster.\n- **Sync** There are two major ways you can synchronize an Ethereum node. You can download the blocks and execute all the transactions within; or you can download the blocks, verify the PoWs and download the state associated a recent block. The latter is much faster, but it relies on benefactors serving you a copy of the recent state. With the current Merkle-Patricia state model, these benefactors read 16TB of data off disk to serve a syncing node. Snapshots enable serving nodes to read only **96GB** of data off disk to get a new node joined into the network.\n\ndrawbacks of snapshot\n- A snapshot is a redundant copy of the raw Ethereum state already contained in the leaves of the Merkle Patricia trie. \nuser can disable snapshot via `--snapshot=false`\n\n## snap sync\nWhen Ethereum launched, you could choose from two different ways to synchronize the network: full sync and fast sync。 Full sync operated by downloading the entire chain and executing all transactions; vs. fast sync placed an initial trust in a recent-ish block, and directly downloaded the state associated with it (after which it switched to block execution like full sync). \n- **full sync** minimized trust, choosing to execute all transactions from genesis to head. \n- **fast sync** chose to rely on the security of the PoWs.it assumed that a block with 64 valid PoWs on top would be prohibitively expensive for someone to construct, as such it's ok to download the state associated with `HEAD-64`\n\n### delays of fast sync\n- network latency (download node)\n- io latency (level db random disk access)\n- upload latency (requst with node `hash` to remote servers)\n\nThe core idea of `snap sync` is fairly simple: instead of downloading the trie node-by-node, snap sync downloads the contiguous chunks of useful state data, and reconstructs the Merkle trie locally:\n- Without downloading intermediate Merkle trie nodes, state data can be fetched in large batches, removing the delay caused by network latency.\n- Without downloading Merkle nodes, downstream data drops to half; and without addressing each piece of data individually, upstream data gets insignificant, removing the delay caused by bandwidth.\n- Without requesting randomly keyed data, peers do only a couple contiguous disk reads to serve the responses, removing the delay of disk IO\n\n## offline pruning\nWhen processing a new block, a node takes the current state of the network as input data and mutates it according to the transactions in the block. only state diff is kept. Pushing these new pieces of state data, block-by-block, to the database is a problem. They keep accumulating. In theory we could \"just delete\" state data that's old enough to not run the risk of a reorg. it's exceedingly costly to figure out if a node deep within an old state is still referenced by anything newer or not.\nIf you have snapshots enabled and fully generated, Geth can use those as an acceleration structure to relatively quickly determine which trie nodes should be kept and which should be deleted. Pruning trie nodes based on snapshots does have the drawback that the chain may not progress during pruning. This means, that you need to stop Geth, prune its database and then restart it. To prune your database, please run `geth snapshot prune-state`.\n\n## transaction unindexing\nNode operators always took it for granted that they can look up an arbitrary transaction from the past, given only its hash. To make transactions searchable, we need to - at minimum - map the entire range of transaction hashes to the blocks they are in. It's also important to note that transaction indices are not part of consensus and are not part of the network protocol. They are purely a locally generated acceleration structure.\nGeth v1.10.0 switches on transaction unindexing by default and sets it to 2,350,000 blocks (about 1 year). The transaction unindexer will linger in the background, and every time a new block arrives, it ensures that only transactions from the most recent N blocks are indexed, deleting older ones. user can use `--txlookuplimit` to control the indexing block range\n\n## preimage discarding\nEthereum stores all its data in a Merkle Patricia trie. The values in the leaves are the raw data being stored (e.g. storage slot content, account content), and the path to the leaf is the key at which the data is stored. The keys however are not the account addresses or storage addresses, rather the Keccak256 hashes of those. This helps balance the branch depths of the state tries.\nthe preimage is the actual key related to the hash. The preimages aren't particularly heavy. If you do a full sync from genesis - reexecuting all the transactions - you'll only end up with 5GB extra load. Still, there is no reason to keep that data around for users not using it, as it only increases the load on LevelDB compactions. As such, Geth v1.10.0 disables preimage collection by default, but there's no mechanism to actively delete already stored preimages.\nIf you are using your Geth instance to debug transactions, you can retain the original behavior via `--cache.preimages`. \n\n## ETH/66 protocol\nThe eth/66 protocol is a fairly small change, yet has quite a number of beneficial implications. In short, the protocol introduces request and reply IDs for all bidirectional packets. The goal behind these IDs is to more easily match up responses to requests, specifically, to more easily deliver a response to a subsystem that made the original request.\n\n## chainid enforcement\nGeth v1.10.0 supports reverting to the old behavior and accepting non-EIP155 transactions via --rpc.allow-unprotected-txs. Be advised that this is a temporary mechanism that will be removed long term.\n\n## Database introspection\nEvery now and again we receive an issue report about a corrupted database, with no real way to debug it. Geth v1.10.0 ships a built-in database introspection tool to try and alleviate the situation a bit. It is a very low level accessor to LevelDB, but it allows arbitrary data retrievals, insertions and deletions. We are unsure how useful these will turn out to be, but they at least give a fighting chance to restore a broken node without having to resync.\n\n## Unclean shutdown tracking\nFairly often we receive bug reports that Geth started importing old blocks on startup. This phenomenon is generally caused by the node operator terminating Geth abruptly (power outage, OOM killer, too short shutdown timeout). Since Geth keeps a lot of dirty state in memory - to avoid writing to disk things that get stale a few blocks later - an abrupt shutdown can cause these to not be flushed. With recent state missing on startup, Geth has no choice but to rewind it's local chain to the point where it last saved the progress.\n\nGeth v1.10.0 will start tracking and reporting node crashes. We're hopeful that this will allow operatos to detect that their infra is misconfigured or has issue before those turn into irreversible data loss.\n```\nWARN [03-03|06:36:38.734] Unclean shutdown detected booted=2021-02-03T06:47:28+0000 age=3w6d23h\n```\n\n## references\n- [eth foundation blog]()","source":"_posts/geth/tech_docs/geth.v1.10.0.md","raw":"---\ntitle: geth v1.10.0 summary\ndate: 2023-03-15 16:29:43\ntags: [geth]\n---\n\n## introduction\ngeth v1.10.0 has been [released](https://github.com/ethereum/go-ethereum/releases/tag/v1.10.0) on Mar 4 2021. this is a late summary of v1.10.0.\n\n## snapshots\nthe snapshot feature reduces the cost of accessing an account from `O(logN)` to `O(1)`. Whilst snapshots do grant us a 10x read performance, EVM execution also writes data, and these writes need to be Merkle proven. The Merkle proof requirement retains the necessity for `O(logN)` disk access on writes. \nProblems it solves\n- **DoS** In 2016, Ethereum sustained its worse DoS attack ever - The [Shanghai Attacks](https://2017.edcon.io/ppt/one/Martin%20Holst%20Swende_The%20%27Shanghai%20%27Attacks_EDCON.pdf) - that lasted about 2-3 months. The attack revolved around bloating Ethereum's state and abusing various underpriced opcodes to grind the network to a halt. After numerous client optimizations and repricing hard forks, the attack was repelled. The root cause still lingers: state access opcodes have a fixed EVM gas cost O(1), but an ever slowly increasing execution cost O(logN). Snapshots on the other hand reduce execution cost of state reads to O(1) - in line with EVM costs - thus solves the read-based DoS issues long term.\n- **Call** Checking a smart contract's state in Ethereum entails a mini EVM execution. Part of that is running bytecode and part of it is reading state slots from disk. snap makes the state access faster.\n- **Sync** There are two major ways you can synchronize an Ethereum node. You can download the blocks and execute all the transactions within; or you can download the blocks, verify the PoWs and download the state associated a recent block. The latter is much faster, but it relies on benefactors serving you a copy of the recent state. With the current Merkle-Patricia state model, these benefactors read 16TB of data off disk to serve a syncing node. Snapshots enable serving nodes to read only **96GB** of data off disk to get a new node joined into the network.\n\ndrawbacks of snapshot\n- A snapshot is a redundant copy of the raw Ethereum state already contained in the leaves of the Merkle Patricia trie. \nuser can disable snapshot via `--snapshot=false`\n\n## snap sync\nWhen Ethereum launched, you could choose from two different ways to synchronize the network: full sync and fast sync。 Full sync operated by downloading the entire chain and executing all transactions; vs. fast sync placed an initial trust in a recent-ish block, and directly downloaded the state associated with it (after which it switched to block execution like full sync). \n- **full sync** minimized trust, choosing to execute all transactions from genesis to head. \n- **fast sync** chose to rely on the security of the PoWs.it assumed that a block with 64 valid PoWs on top would be prohibitively expensive for someone to construct, as such it's ok to download the state associated with `HEAD-64`\n\n### delays of fast sync\n- network latency (download node)\n- io latency (level db random disk access)\n- upload latency (requst with node `hash` to remote servers)\n\nThe core idea of `snap sync` is fairly simple: instead of downloading the trie node-by-node, snap sync downloads the contiguous chunks of useful state data, and reconstructs the Merkle trie locally:\n- Without downloading intermediate Merkle trie nodes, state data can be fetched in large batches, removing the delay caused by network latency.\n- Without downloading Merkle nodes, downstream data drops to half; and without addressing each piece of data individually, upstream data gets insignificant, removing the delay caused by bandwidth.\n- Without requesting randomly keyed data, peers do only a couple contiguous disk reads to serve the responses, removing the delay of disk IO\n\n## offline pruning\nWhen processing a new block, a node takes the current state of the network as input data and mutates it according to the transactions in the block. only state diff is kept. Pushing these new pieces of state data, block-by-block, to the database is a problem. They keep accumulating. In theory we could \"just delete\" state data that's old enough to not run the risk of a reorg. it's exceedingly costly to figure out if a node deep within an old state is still referenced by anything newer or not.\nIf you have snapshots enabled and fully generated, Geth can use those as an acceleration structure to relatively quickly determine which trie nodes should be kept and which should be deleted. Pruning trie nodes based on snapshots does have the drawback that the chain may not progress during pruning. This means, that you need to stop Geth, prune its database and then restart it. To prune your database, please run `geth snapshot prune-state`.\n\n## transaction unindexing\nNode operators always took it for granted that they can look up an arbitrary transaction from the past, given only its hash. To make transactions searchable, we need to - at minimum - map the entire range of transaction hashes to the blocks they are in. It's also important to note that transaction indices are not part of consensus and are not part of the network protocol. They are purely a locally generated acceleration structure.\nGeth v1.10.0 switches on transaction unindexing by default and sets it to 2,350,000 blocks (about 1 year). The transaction unindexer will linger in the background, and every time a new block arrives, it ensures that only transactions from the most recent N blocks are indexed, deleting older ones. user can use `--txlookuplimit` to control the indexing block range\n\n## preimage discarding\nEthereum stores all its data in a Merkle Patricia trie. The values in the leaves are the raw data being stored (e.g. storage slot content, account content), and the path to the leaf is the key at which the data is stored. The keys however are not the account addresses or storage addresses, rather the Keccak256 hashes of those. This helps balance the branch depths of the state tries.\nthe preimage is the actual key related to the hash. The preimages aren't particularly heavy. If you do a full sync from genesis - reexecuting all the transactions - you'll only end up with 5GB extra load. Still, there is no reason to keep that data around for users not using it, as it only increases the load on LevelDB compactions. As such, Geth v1.10.0 disables preimage collection by default, but there's no mechanism to actively delete already stored preimages.\nIf you are using your Geth instance to debug transactions, you can retain the original behavior via `--cache.preimages`. \n\n## ETH/66 protocol\nThe eth/66 protocol is a fairly small change, yet has quite a number of beneficial implications. In short, the protocol introduces request and reply IDs for all bidirectional packets. The goal behind these IDs is to more easily match up responses to requests, specifically, to more easily deliver a response to a subsystem that made the original request.\n\n## chainid enforcement\nGeth v1.10.0 supports reverting to the old behavior and accepting non-EIP155 transactions via --rpc.allow-unprotected-txs. Be advised that this is a temporary mechanism that will be removed long term.\n\n## Database introspection\nEvery now and again we receive an issue report about a corrupted database, with no real way to debug it. Geth v1.10.0 ships a built-in database introspection tool to try and alleviate the situation a bit. It is a very low level accessor to LevelDB, but it allows arbitrary data retrievals, insertions and deletions. We are unsure how useful these will turn out to be, but they at least give a fighting chance to restore a broken node without having to resync.\n\n## Unclean shutdown tracking\nFairly often we receive bug reports that Geth started importing old blocks on startup. This phenomenon is generally caused by the node operator terminating Geth abruptly (power outage, OOM killer, too short shutdown timeout). Since Geth keeps a lot of dirty state in memory - to avoid writing to disk things that get stale a few blocks later - an abrupt shutdown can cause these to not be flushed. With recent state missing on startup, Geth has no choice but to rewind it's local chain to the point where it last saved the progress.\n\nGeth v1.10.0 will start tracking and reporting node crashes. We're hopeful that this will allow operatos to detect that their infra is misconfigured or has issue before those turn into irreversible data loss.\n```\nWARN [03-03|06:36:38.734] Unclean shutdown detected booted=2021-02-03T06:47:28+0000 age=3w6d23h\n```\n\n## references\n- [eth foundation blog]()","slug":"geth/tech_docs/geth.v1.10.0","published":1,"updated":"2023-11-05T04:21:13.733Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8e1003eqwsj3oda50v5","content":"

introduction

geth v1.10.0 has been released on Mar 4 2021. this is a late summary of v1.10.0.

\n

snapshots

the snapshot feature reduces the cost of accessing an account from O(logN) to O(1). Whilst snapshots do grant us a 10x read performance, EVM execution also writes data, and these writes need to be Merkle proven. The Merkle proof requirement retains the necessity for O(logN) disk access on writes.
Problems it solves

\n
    \n
  • DoS In 2016, Ethereum sustained its worse DoS attack ever - The Shanghai Attacks - that lasted about 2-3 months. The attack revolved around bloating Ethereum’s state and abusing various underpriced opcodes to grind the network to a halt. After numerous client optimizations and repricing hard forks, the attack was repelled. The root cause still lingers: state access opcodes have a fixed EVM gas cost O(1), but an ever slowly increasing execution cost O(logN). Snapshots on the other hand reduce execution cost of state reads to O(1) - in line with EVM costs - thus solves the read-based DoS issues long term.
    • \n
  • Call Checking a smart contract’s state in Ethereum entails a mini EVM execution. Part of that is running bytecode and part of it is reading state slots from disk. snap makes the state access faster.
    • \n
  • Sync There are two major ways you can synchronize an Ethereum node. You can download the blocks and execute all the transactions within; or you can download the blocks, verify the PoWs and download the state associated a recent block. The latter is much faster, but it relies on benefactors serving you a copy of the recent state. With the current Merkle-Patricia state model, these benefactors read 16TB of data off disk to serve a syncing node. Snapshots enable serving nodes to read only 96GB of data off disk to get a new node joined into the network.
    • \n
\n

drawbacks of snapshot

\n
    \n
  • A snapshot is a redundant copy of the raw Ethereum state already contained in the leaves of the Merkle Patricia trie.
    user can disable snapshot via --snapshot=false
    • \n
\n

snap sync

When Ethereum launched, you could choose from two different ways to synchronize the network: full sync and fast sync。 Full sync operated by downloading the entire chain and executing all transactions; vs. fast sync placed an initial trust in a recent-ish block, and directly downloaded the state associated with it (after which it switched to block execution like full sync).

\n
    \n
  • full sync minimized trust, choosing to execute all transactions from genesis to head.
    • \n
  • fast sync chose to rely on the security of the PoWs.it assumed that a block with 64 valid PoWs on top would be prohibitively expensive for someone to construct, as such it’s ok to download the state associated with HEAD-64
    • \n
\n

delays of fast sync

    \n
  • network latency (download node)
    • \n
  • io latency (level db random disk access)
    • \n
  • upload latency (requst with node hash to remote servers)
    • \n
\n

The core idea of snap sync is fairly simple: instead of downloading the trie node-by-node, snap sync downloads the contiguous chunks of useful state data, and reconstructs the Merkle trie locally:

\n
    \n
  • Without downloading intermediate Merkle trie nodes, state data can be fetched in large batches, removing the delay caused by network latency.
    • \n
  • Without downloading Merkle nodes, downstream data drops to half; and without addressing each piece of data individually, upstream data gets insignificant, removing the delay caused by bandwidth.
    • \n
  • Without requesting randomly keyed data, peers do only a couple contiguous disk reads to serve the responses, removing the delay of disk IO
    • \n
\n

offline pruning

When processing a new block, a node takes the current state of the network as input data and mutates it according to the transactions in the block. only state diff is kept. Pushing these new pieces of state data, block-by-block, to the database is a problem. They keep accumulating. In theory we could “just delete” state data that’s old enough to not run the risk of a reorg. it’s exceedingly costly to figure out if a node deep within an old state is still referenced by anything newer or not.
If you have snapshots enabled and fully generated, Geth can use those as an acceleration structure to relatively quickly determine which trie nodes should be kept and which should be deleted. Pruning trie nodes based on snapshots does have the drawback that the chain may not progress during pruning. This means, that you need to stop Geth, prune its database and then restart it. To prune your database, please run geth snapshot prune-state.

\n

transaction unindexing

Node operators always took it for granted that they can look up an arbitrary transaction from the past, given only its hash. To make transactions searchable, we need to - at minimum - map the entire range of transaction hashes to the blocks they are in. It’s also important to note that transaction indices are not part of consensus and are not part of the network protocol. They are purely a locally generated acceleration structure.
Geth v1.10.0 switches on transaction unindexing by default and sets it to 2,350,000 blocks (about 1 year). The transaction unindexer will linger in the background, and every time a new block arrives, it ensures that only transactions from the most recent N blocks are indexed, deleting older ones. user can use --txlookuplimit to control the indexing block range

\n

preimage discarding

Ethereum stores all its data in a Merkle Patricia trie. The values in the leaves are the raw data being stored (e.g. storage slot content, account content), and the path to the leaf is the key at which the data is stored. The keys however are not the account addresses or storage addresses, rather the Keccak256 hashes of those. This helps balance the branch depths of the state tries.
the preimage is the actual key related to the hash. The preimages aren’t particularly heavy. If you do a full sync from genesis - reexecuting all the transactions - you’ll only end up with 5GB extra load. Still, there is no reason to keep that data around for users not using it, as it only increases the load on LevelDB compactions. As such, Geth v1.10.0 disables preimage collection by default, but there’s no mechanism to actively delete already stored preimages.
If you are using your Geth instance to debug transactions, you can retain the original behavior via --cache.preimages.

\n

ETH/66 protocol

The eth/66 protocol is a fairly small change, yet has quite a number of beneficial implications. In short, the protocol introduces request and reply IDs for all bidirectional packets. The goal behind these IDs is to more easily match up responses to requests, specifically, to more easily deliver a response to a subsystem that made the original request.

\n

chainid enforcement

Geth v1.10.0 supports reverting to the old behavior and accepting non-EIP155 transactions via –rpc.allow-unprotected-txs. Be advised that this is a temporary mechanism that will be removed long term.

\n

Database introspection

Every now and again we receive an issue report about a corrupted database, with no real way to debug it. Geth v1.10.0 ships a built-in database introspection tool to try and alleviate the situation a bit. It is a very low level accessor to LevelDB, but it allows arbitrary data retrievals, insertions and deletions. We are unsure how useful these will turn out to be, but they at least give a fighting chance to restore a broken node without having to resync.

\n

Unclean shutdown tracking

Fairly often we receive bug reports that Geth started importing old blocks on startup. This phenomenon is generally caused by the node operator terminating Geth abruptly (power outage, OOM killer, too short shutdown timeout). Since Geth keeps a lot of dirty state in memory - to avoid writing to disk things that get stale a few blocks later - an abrupt shutdown can cause these to not be flushed. With recent state missing on startup, Geth has no choice but to rewind it’s local chain to the point where it last saved the progress.

\n

Geth v1.10.0 will start tracking and reporting node crashes. We’re hopeful that this will allow operatos to detect that their infra is misconfigured or has issue before those turn into irreversible data loss.

\n
1
WARN [03-03|06:36:38.734] Unclean shutdown detected        booted=2021-02-03T06:47:28+0000 age=3w6d23h
\n\n

references

\n","site":{"data":{}},"excerpt":"","more":"

introduction

geth v1.10.0 has been released on Mar 4 2021. this is a late summary of v1.10.0.

\n

snapshots

the snapshot feature reduces the cost of accessing an account from O(logN) to O(1). Whilst snapshots do grant us a 10x read performance, EVM execution also writes data, and these writes need to be Merkle proven. The Merkle proof requirement retains the necessity for O(logN) disk access on writes.
Problems it solves

\n
    \n
  • DoS In 2016, Ethereum sustained its worse DoS attack ever - The Shanghai Attacks - that lasted about 2-3 months. The attack revolved around bloating Ethereum’s state and abusing various underpriced opcodes to grind the network to a halt. After numerous client optimizations and repricing hard forks, the attack was repelled. The root cause still lingers: state access opcodes have a fixed EVM gas cost O(1), but an ever slowly increasing execution cost O(logN). Snapshots on the other hand reduce execution cost of state reads to O(1) - in line with EVM costs - thus solves the read-based DoS issues long term.
    • \n
  • Call Checking a smart contract’s state in Ethereum entails a mini EVM execution. Part of that is running bytecode and part of it is reading state slots from disk. snap makes the state access faster.
    • \n
  • Sync There are two major ways you can synchronize an Ethereum node. You can download the blocks and execute all the transactions within; or you can download the blocks, verify the PoWs and download the state associated a recent block. The latter is much faster, but it relies on benefactors serving you a copy of the recent state. With the current Merkle-Patricia state model, these benefactors read 16TB of data off disk to serve a syncing node. Snapshots enable serving nodes to read only 96GB of data off disk to get a new node joined into the network.
    • \n
\n

drawbacks of snapshot

\n
    \n
  • A snapshot is a redundant copy of the raw Ethereum state already contained in the leaves of the Merkle Patricia trie.
    user can disable snapshot via --snapshot=false
    • \n
\n

snap sync

When Ethereum launched, you could choose from two different ways to synchronize the network: full sync and fast sync。 Full sync operated by downloading the entire chain and executing all transactions; vs. fast sync placed an initial trust in a recent-ish block, and directly downloaded the state associated with it (after which it switched to block execution like full sync).

\n
    \n
  • full sync minimized trust, choosing to execute all transactions from genesis to head.
    • \n
  • fast sync chose to rely on the security of the PoWs.it assumed that a block with 64 valid PoWs on top would be prohibitively expensive for someone to construct, as such it’s ok to download the state associated with HEAD-64
    • \n
\n

delays of fast sync

    \n
  • network latency (download node)
    • \n
  • io latency (level db random disk access)
    • \n
  • upload latency (requst with node hash to remote servers)
    • \n
\n

The core idea of snap sync is fairly simple: instead of downloading the trie node-by-node, snap sync downloads the contiguous chunks of useful state data, and reconstructs the Merkle trie locally:

\n
    \n
  • Without downloading intermediate Merkle trie nodes, state data can be fetched in large batches, removing the delay caused by network latency.
    • \n
  • Without downloading Merkle nodes, downstream data drops to half; and without addressing each piece of data individually, upstream data gets insignificant, removing the delay caused by bandwidth.
    • \n
  • Without requesting randomly keyed data, peers do only a couple contiguous disk reads to serve the responses, removing the delay of disk IO
    • \n
\n

offline pruning

When processing a new block, a node takes the current state of the network as input data and mutates it according to the transactions in the block. only state diff is kept. Pushing these new pieces of state data, block-by-block, to the database is a problem. They keep accumulating. In theory we could “just delete” state data that’s old enough to not run the risk of a reorg. it’s exceedingly costly to figure out if a node deep within an old state is still referenced by anything newer or not.
If you have snapshots enabled and fully generated, Geth can use those as an acceleration structure to relatively quickly determine which trie nodes should be kept and which should be deleted. Pruning trie nodes based on snapshots does have the drawback that the chain may not progress during pruning. This means, that you need to stop Geth, prune its database and then restart it. To prune your database, please run geth snapshot prune-state.

\n

transaction unindexing

Node operators always took it for granted that they can look up an arbitrary transaction from the past, given only its hash. To make transactions searchable, we need to - at minimum - map the entire range of transaction hashes to the blocks they are in. It’s also important to note that transaction indices are not part of consensus and are not part of the network protocol. They are purely a locally generated acceleration structure.
Geth v1.10.0 switches on transaction unindexing by default and sets it to 2,350,000 blocks (about 1 year). The transaction unindexer will linger in the background, and every time a new block arrives, it ensures that only transactions from the most recent N blocks are indexed, deleting older ones. user can use --txlookuplimit to control the indexing block range

\n

preimage discarding

Ethereum stores all its data in a Merkle Patricia trie. The values in the leaves are the raw data being stored (e.g. storage slot content, account content), and the path to the leaf is the key at which the data is stored. The keys however are not the account addresses or storage addresses, rather the Keccak256 hashes of those. This helps balance the branch depths of the state tries.
the preimage is the actual key related to the hash. The preimages aren’t particularly heavy. If you do a full sync from genesis - reexecuting all the transactions - you’ll only end up with 5GB extra load. Still, there is no reason to keep that data around for users not using it, as it only increases the load on LevelDB compactions. As such, Geth v1.10.0 disables preimage collection by default, but there’s no mechanism to actively delete already stored preimages.
If you are using your Geth instance to debug transactions, you can retain the original behavior via --cache.preimages.

\n

ETH/66 protocol

The eth/66 protocol is a fairly small change, yet has quite a number of beneficial implications. In short, the protocol introduces request and reply IDs for all bidirectional packets. The goal behind these IDs is to more easily match up responses to requests, specifically, to more easily deliver a response to a subsystem that made the original request.

\n

chainid enforcement

Geth v1.10.0 supports reverting to the old behavior and accepting non-EIP155 transactions via –rpc.allow-unprotected-txs. Be advised that this is a temporary mechanism that will be removed long term.

\n

Database introspection

Every now and again we receive an issue report about a corrupted database, with no real way to debug it. Geth v1.10.0 ships a built-in database introspection tool to try and alleviate the situation a bit. It is a very low level accessor to LevelDB, but it allows arbitrary data retrievals, insertions and deletions. We are unsure how useful these will turn out to be, but they at least give a fighting chance to restore a broken node without having to resync.

\n

Unclean shutdown tracking

Fairly often we receive bug reports that Geth started importing old blocks on startup. This phenomenon is generally caused by the node operator terminating Geth abruptly (power outage, OOM killer, too short shutdown timeout). Since Geth keeps a lot of dirty state in memory - to avoid writing to disk things that get stale a few blocks later - an abrupt shutdown can cause these to not be flushed. With recent state missing on startup, Geth has no choice but to rewind it’s local chain to the point where it last saved the progress.

\n

Geth v1.10.0 will start tracking and reporting node crashes. We’re hopeful that this will allow operatos to detect that their infra is misconfigured or has issue before those turn into irreversible data loss.

\n
1
WARN [03-03|06:36:38.734] Unclean shutdown detected        booted=2021-02-03T06:47:28+0000 age=3w6d23h
\n\n

references

\n"},{"title":"elliptic curve paring","date":"2023-06-27T06:29:26.000Z","_content":"\n\n\n## introduction\n\nPairings allow you to check certain kinds of more complicated equations on elliptic curve points — for example, if \\\\(P = G * p\\\\), \\\\(Q = G * q\\\\) and \\\\(R = G * r\\\\), you can check whether or not \\\\(p * q = r\\\\), having just P, Q and R as inputs. This might seem like the fundamental security guarantees of elliptic curves are being broken, as information about p is leaking from just knowing P, but it turns out that the leakage is highly contained — specifically, the ***decisional Diffie Hellman problem*** is easy, but the computational Diffie Hellman problem (knowing P and Q in the above example, computing R = G * p * q) and the discrete logarithm problem (recovering p from P) remain computationally infeasible.\n\n whereas traditional elliptic curve math lets you check linear constraints on the numbers (eg. if P = G * p, Q = G * q and R = G * r, checking 5 * P + 7 * Q = 11 * R is really checking that 5 * p + 7 * q = 11 * r), pairings let you check quadratic constraints (eg. checking e(P, Q) * e(G, G * 5) = 1 is really checking that p * q + 5 = 0).\n\n Now, what is this funny e(P, Q) operator that we introduced above? This is the pairing. Mathematicians also sometimes call it a bilinear map; the word “bilinear” here basically means that it satisfies the constraints:\n\n\\\\[ e(P, Q + R) = e(P, Q) * e(P, R) \\\\]\n\\\\[ e(P + S, Q) = e(P, Q) * e(S, Q) \\\\]\nNote that + and * can be arbitrary operators; when you’re creating fancy new kinds of mathematical objects, abstract algebra doesn’t care how + and * are defined, as long as they are consistent in the usual ways.\n\nIf P, Q, R and S were simple numbers, then making a simple pairing is easy: we can do e(x, y) = 2^xy. Then, we can see:\n\\\\[e(3, 4+ 5) = 2^(3 * 9) = 2^{27}\\\\]\n\\\\[e(3, 4) * e(3, 5) = 2^(3 * 4) * 2^(3 * 5) = 2^{12} * 2^{15} = 2^{27} \\\\]\nHowever, such simple pairings are not suitable for cryptography because the objects that they work on are simple integers and are too easy to analyze;\n\nIt turns out that it is possible to make a bilinear map over elliptic curve points — that is, come up with a function e(P, Q) where the inputs P and Q are elliptic curve points, and where the output is what’s called an \\\\(F_{p^¹²}\\\\) (extension field) element\n\nAn elliptic curve pairing is a map G2 x G1 -> Gt, where:\n- G1 is an elliptic curve, where points satisfy an equation of the form y² = x³ + b, and where both coordinates are elements of F_p (ie. they are simple numbers, except arithmetic is all done modulo some prime number)\n- G2 is an elliptic curve, where points satisfy the same equation as G1, except where the coordinates are elements of \\\\(F_{p^¹²}\\\\). we define a new “magic number” w, which is defined by a 12th degree polynomial like w^12 - 18 * w^6 + 82 = 0\n- Gt is the type of object that the result of the elliptic curve goes into. In the curves that we look at, Gt is \\\\(F_{p^¹²}\\\\)\nThe main property that it must satisfy is bilinearity, which in presented above\n\nThere are two other important criteria:\n- **Efficient computability** (eg. we can make an easy pairing by simply taking the discrete logarithms of all points and multiplying them together, but this is as computationally hard as breaking elliptic curve cryptography in the first place, so it doesn’t count)\n- **Non-degeneracy** (sure, you could just define e(P, Q) = 1, but that’s not a particularly useful pairing)\n\n## math intuition behind paring\nThe math behind why pairing functions work is quite tricky and involves quite a bit of advanced algebra going even beyond what we’ve seen so far, but I’ll provide an outline. First of all, we need to define the concept of a divisor, basically an alternative way of representing functions on elliptic curve points. A **divisor** of a function basically counts the zeroes and the infinities of the function. To see what this means, let’s go through a few examples. Let us fix some point P = (P_x, P_y), and consider the following function:\n\\\\[f(x, y) = x - P_x\\\\]\n\nThe divisor is [P] + [-P] - 2 * [O] (the square brackets are used to represent the fact that we are referring to the presence of the point P in the set of zeroes and infinities of the function, not the point P itself; [P] + [Q] is not the same thing as [P + Q]). The reasoning is as follows:\n- The function is equal to zero at P, since x is P_x, so x - P_x = 0\n- The function is equal to zero at -P, since -P and P share the same x coordinate\n- The function goes to infinity as x goes to infinity, so we say the function is equal to infinity at O. There’s a technical reason why this infinity needs to be counted twice, so O gets added with a “multiplicity” of -2 (negative because it’s an infinity and not a zero, two because of this double counting).\nThe technical reason is roughly this: because the equation of the curve is x³ = y² + b, y goes to infinity “1.5 times faster” than x does in order for y² to keep up with x³; hence, if a linear function includes only x then it is represented as an infinity of multiplicity 2, but if it includes y then it is represented as an infinity of multiplicity 3.\n\nWhere a, b and c are carefully chosen so that the line passes through points P and Q. Because of how elliptic curve addition works (see the diagram at the top), this also means that it passes through -P-Q. And it goes up to infinity dependent on both x and y, so the divisor becomes [P]+ [Q] + [-P-Q] - 3 * [O]. (multiplicity of 3 because it involves y)\n![ec_pariling](/images/cryptography/elliptic_curve/paring_ec.png)\nWe know that every “rational function” (ie. a function defined only using a finite number of +, -, * and / operations on the coordinates of the point) uniquely corresponds to some divisor, up to multiplication by a constant (ie. if two functions F and G have the same divisor, then F = G * k for some constant k).\nFor any two functions F and G, the divisor of F * G is equal to the divisor of F plus the divisor of G (in math textbooks, you’ll see (F * G) = (F) + (G)), so for example if f(x, y) = P_x - x, then (f³) = 3 * [P] + 3 * [-P] - 6 * [O]; P and -P are “triple-counted” to account for the fact that f³ approaches 0 at those points “three times as quickly” in a certain mathematical sense.\n\nNote that there is a theorem that states that if you “remove the square brackets” from a divisor of a function, the points must add up to O ([P] + [Q] + [-P-Q] - 3 * [O] clearly fits, as P + Q - P - Q - 3 * O = O), and any divisor that has this property is the divisor of a function.\n\n## Tate pairings\nConsider the following functions, defined via their divisors:\n- (F_P) = n * [P] - n * [O], where n is the order of G1, ie. n * P = O for any P\n- (F_Q) = n * [Q] - n * [O]\n- (g) = [P + Q] - [P] - [Q] + [O]\nNow, let’s look at the product F_P * F_Q * g^n. The divisor is:\n\nn * [P] - n * [O] + n * [Q] - n * [O] + n * [P + Q] - n * [P] - n * [Q] + n * [O]\n\nWhich simplifies neatly to:\n\nn * [P + Q] - n * [O]\nNotice that this divisor is of exactly the same format as the divisor for F_P and F_Q above. Hence, F_P * F_Q * g^n = F_(P + Q).\n\nNow, we introduce a procedure called the “final exponentiation” step, where we take the result of our functions above (F_P, F_Q, etc.) and raise it to the power z = (p¹² - 1) / n, where p¹² - 1 is the order of the multiplicative group in F_p¹² (ie. for any x ϵ F_p¹², x^(p¹² - 1) = 1). Notice that if you apply this exponentiation to any result that has already been raised to the power of n, you get an exponentiation to the power of p¹² - 1, so the result turns into 1. Hence, after final exponentiation, g^n cancels out and we get F_P^z * F_Q^z = F_(P + Q)^z. There’s some bilinearity for you.\nNow, if you want to make a function that’s bilinear in both arguments, you need to go into spookier math, where instead of taking F_P of a value directly, you take F_P of a divisor, and that’s where the full “Tate pairing” comes from. To prove some more results you have to deal with notions like “linear equivalence” and “Weil reciprocity”, and the rabbit hole goes on from there. You can find more reading material on all of this [2] and he[3].\nEvery elliptic curve has a value called an embedding degree; essentially, the smallest k such that p^k - 1 is a multiple of n (where p is the prime used for the field and n is the curve order). In the fields above, k = 12, and in the fields used for traditional ECC (ie. where we don’t care about pairings), the embedding degree is often extremely large, to the point that pairings are computationally infeasible to compute; however, if we are not careful then we can generate fields where k = 4 or even 1.\n\n## references\n- [1] [vitalik's blog on paring](https://medium.com/@VitalikButerin/exploring-elliptic-curve-pairings-c73c1864e627)\n- [2] [bilinear pairings in cryptography by Dennis Meffert](https://www.math.ru.nl/~bosma/Students/MScThesis_DennisMeffert.pdf)\n- [3] [Elliptic Curves number theory and cryptography by Kenneth H. Rosen](https://people.cs.nctu.edu.tw/~rjchen/ECC2012S/Elliptic%20Curves%20Number%20Theory%20And%20Cryptography%202n.pdf)\n- [4] [Miller's Algorithm](https://crypto.stanford.edu/pbc/notes/ep/miller.html)","source":"_posts/cryptography/elliptic_curve/elliptic-curve-pairing.md","raw":"---\ntitle: elliptic curve paring\ndate: 2023-06-27 14:29:26\ntags: [cryptography,ec]\n---\n\n\n\n## introduction\n\nPairings allow you to check certain kinds of more complicated equations on elliptic curve points — for example, if \\\\(P = G * p\\\\), \\\\(Q = G * q\\\\) and \\\\(R = G * r\\\\), you can check whether or not \\\\(p * q = r\\\\), having just P, Q and R as inputs. This might seem like the fundamental security guarantees of elliptic curves are being broken, as information about p is leaking from just knowing P, but it turns out that the leakage is highly contained — specifically, the ***decisional Diffie Hellman problem*** is easy, but the computational Diffie Hellman problem (knowing P and Q in the above example, computing R = G * p * q) and the discrete logarithm problem (recovering p from P) remain computationally infeasible.\n\n whereas traditional elliptic curve math lets you check linear constraints on the numbers (eg. if P = G * p, Q = G * q and R = G * r, checking 5 * P + 7 * Q = 11 * R is really checking that 5 * p + 7 * q = 11 * r), pairings let you check quadratic constraints (eg. checking e(P, Q) * e(G, G * 5) = 1 is really checking that p * q + 5 = 0).\n\n Now, what is this funny e(P, Q) operator that we introduced above? This is the pairing. Mathematicians also sometimes call it a bilinear map; the word “bilinear” here basically means that it satisfies the constraints:\n\n\\\\[ e(P, Q + R) = e(P, Q) * e(P, R) \\\\]\n\\\\[ e(P + S, Q) = e(P, Q) * e(S, Q) \\\\]\nNote that + and * can be arbitrary operators; when you’re creating fancy new kinds of mathematical objects, abstract algebra doesn’t care how + and * are defined, as long as they are consistent in the usual ways.\n\nIf P, Q, R and S were simple numbers, then making a simple pairing is easy: we can do e(x, y) = 2^xy. Then, we can see:\n\\\\[e(3, 4+ 5) = 2^(3 * 9) = 2^{27}\\\\]\n\\\\[e(3, 4) * e(3, 5) = 2^(3 * 4) * 2^(3 * 5) = 2^{12} * 2^{15} = 2^{27} \\\\]\nHowever, such simple pairings are not suitable for cryptography because the objects that they work on are simple integers and are too easy to analyze;\n\nIt turns out that it is possible to make a bilinear map over elliptic curve points — that is, come up with a function e(P, Q) where the inputs P and Q are elliptic curve points, and where the output is what’s called an \\\\(F_{p^¹²}\\\\) (extension field) element\n\nAn elliptic curve pairing is a map G2 x G1 -> Gt, where:\n- G1 is an elliptic curve, where points satisfy an equation of the form y² = x³ + b, and where both coordinates are elements of F_p (ie. they are simple numbers, except arithmetic is all done modulo some prime number)\n- G2 is an elliptic curve, where points satisfy the same equation as G1, except where the coordinates are elements of \\\\(F_{p^¹²}\\\\). we define a new “magic number” w, which is defined by a 12th degree polynomial like w^12 - 18 * w^6 + 82 = 0\n- Gt is the type of object that the result of the elliptic curve goes into. In the curves that we look at, Gt is \\\\(F_{p^¹²}\\\\)\nThe main property that it must satisfy is bilinearity, which in presented above\n\nThere are two other important criteria:\n- **Efficient computability** (eg. we can make an easy pairing by simply taking the discrete logarithms of all points and multiplying them together, but this is as computationally hard as breaking elliptic curve cryptography in the first place, so it doesn’t count)\n- **Non-degeneracy** (sure, you could just define e(P, Q) = 1, but that’s not a particularly useful pairing)\n\n## math intuition behind paring\nThe math behind why pairing functions work is quite tricky and involves quite a bit of advanced algebra going even beyond what we’ve seen so far, but I’ll provide an outline. First of all, we need to define the concept of a divisor, basically an alternative way of representing functions on elliptic curve points. A **divisor** of a function basically counts the zeroes and the infinities of the function. To see what this means, let’s go through a few examples. Let us fix some point P = (P_x, P_y), and consider the following function:\n\\\\[f(x, y) = x - P_x\\\\]\n\nThe divisor is [P] + [-P] - 2 * [O] (the square brackets are used to represent the fact that we are referring to the presence of the point P in the set of zeroes and infinities of the function, not the point P itself; [P] + [Q] is not the same thing as [P + Q]). The reasoning is as follows:\n- The function is equal to zero at P, since x is P_x, so x - P_x = 0\n- The function is equal to zero at -P, since -P and P share the same x coordinate\n- The function goes to infinity as x goes to infinity, so we say the function is equal to infinity at O. There’s a technical reason why this infinity needs to be counted twice, so O gets added with a “multiplicity” of -2 (negative because it’s an infinity and not a zero, two because of this double counting).\nThe technical reason is roughly this: because the equation of the curve is x³ = y² + b, y goes to infinity “1.5 times faster” than x does in order for y² to keep up with x³; hence, if a linear function includes only x then it is represented as an infinity of multiplicity 2, but if it includes y then it is represented as an infinity of multiplicity 3.\n\nWhere a, b and c are carefully chosen so that the line passes through points P and Q. Because of how elliptic curve addition works (see the diagram at the top), this also means that it passes through -P-Q. And it goes up to infinity dependent on both x and y, so the divisor becomes [P]+ [Q] + [-P-Q] - 3 * [O]. (multiplicity of 3 because it involves y)\n![ec_pariling](/images/cryptography/elliptic_curve/paring_ec.png)\nWe know that every “rational function” (ie. a function defined only using a finite number of +, -, * and / operations on the coordinates of the point) uniquely corresponds to some divisor, up to multiplication by a constant (ie. if two functions F and G have the same divisor, then F = G * k for some constant k).\nFor any two functions F and G, the divisor of F * G is equal to the divisor of F plus the divisor of G (in math textbooks, you’ll see (F * G) = (F) + (G)), so for example if f(x, y) = P_x - x, then (f³) = 3 * [P] + 3 * [-P] - 6 * [O]; P and -P are “triple-counted” to account for the fact that f³ approaches 0 at those points “three times as quickly” in a certain mathematical sense.\n\nNote that there is a theorem that states that if you “remove the square brackets” from a divisor of a function, the points must add up to O ([P] + [Q] + [-P-Q] - 3 * [O] clearly fits, as P + Q - P - Q - 3 * O = O), and any divisor that has this property is the divisor of a function.\n\n## Tate pairings\nConsider the following functions, defined via their divisors:\n- (F_P) = n * [P] - n * [O], where n is the order of G1, ie. n * P = O for any P\n- (F_Q) = n * [Q] - n * [O]\n- (g) = [P + Q] - [P] - [Q] + [O]\nNow, let’s look at the product F_P * F_Q * g^n. The divisor is:\n\nn * [P] - n * [O] + n * [Q] - n * [O] + n * [P + Q] - n * [P] - n * [Q] + n * [O]\n\nWhich simplifies neatly to:\n\nn * [P + Q] - n * [O]\nNotice that this divisor is of exactly the same format as the divisor for F_P and F_Q above. Hence, F_P * F_Q * g^n = F_(P + Q).\n\nNow, we introduce a procedure called the “final exponentiation” step, where we take the result of our functions above (F_P, F_Q, etc.) and raise it to the power z = (p¹² - 1) / n, where p¹² - 1 is the order of the multiplicative group in F_p¹² (ie. for any x ϵ F_p¹², x^(p¹² - 1) = 1). Notice that if you apply this exponentiation to any result that has already been raised to the power of n, you get an exponentiation to the power of p¹² - 1, so the result turns into 1. Hence, after final exponentiation, g^n cancels out and we get F_P^z * F_Q^z = F_(P + Q)^z. There’s some bilinearity for you.\nNow, if you want to make a function that’s bilinear in both arguments, you need to go into spookier math, where instead of taking F_P of a value directly, you take F_P of a divisor, and that’s where the full “Tate pairing” comes from. To prove some more results you have to deal with notions like “linear equivalence” and “Weil reciprocity”, and the rabbit hole goes on from there. You can find more reading material on all of this [2] and he[3].\nEvery elliptic curve has a value called an embedding degree; essentially, the smallest k such that p^k - 1 is a multiple of n (where p is the prime used for the field and n is the curve order). In the fields above, k = 12, and in the fields used for traditional ECC (ie. where we don’t care about pairings), the embedding degree is often extremely large, to the point that pairings are computationally infeasible to compute; however, if we are not careful then we can generate fields where k = 4 or even 1.\n\n## references\n- [1] [vitalik's blog on paring](https://medium.com/@VitalikButerin/exploring-elliptic-curve-pairings-c73c1864e627)\n- [2] [bilinear pairings in cryptography by Dennis Meffert](https://www.math.ru.nl/~bosma/Students/MScThesis_DennisMeffert.pdf)\n- [3] [Elliptic Curves number theory and cryptography by Kenneth H. Rosen](https://people.cs.nctu.edu.tw/~rjchen/ECC2012S/Elliptic%20Curves%20Number%20Theory%20And%20Cryptography%202n.pdf)\n- [4] [Miller's Algorithm](https://crypto.stanford.edu/pbc/notes/ep/miller.html)","slug":"cryptography/elliptic_curve/elliptic-curve-pairing","published":1,"updated":"2023-12-01T02:50:40.735Z","_id":"clokyy8e1003gqwsj9p1scg9k","comments":1,"layout":"post","photos":[],"link":"","content":"\n\n\n

introduction

Pairings allow you to check certain kinds of more complicated equations on elliptic curve points — for example, if \\(P = G * p\\), \\(Q = G * q\\) and \\(R = G * r\\), you can check whether or not \\(p * q = r\\), having just P, Q and R as inputs. This might seem like the fundamental security guarantees of elliptic curves are being broken, as information about p is leaking from just knowing P, but it turns out that the leakage is highly contained — specifically, the decisional Diffie Hellman problem is easy, but the computational Diffie Hellman problem (knowing P and Q in the above example, computing R = G * p * q) and the discrete logarithm problem (recovering p from P) remain computationally infeasible.

\n

whereas traditional elliptic curve math lets you check linear constraints on the numbers (eg. if P = G * p, Q = G * q and R = G * r, checking 5 * P + 7 * Q = 11 * R is really checking that 5 * p + 7 * q = 11 * r), pairings let you check quadratic constraints (eg. checking e(P, Q) * e(G, G * 5) = 1 is really checking that p * q + 5 = 0).

\n

Now, what is this funny e(P, Q) operator that we introduced above? This is the pairing. Mathematicians also sometimes call it a bilinear map; the word “bilinear” here basically means that it satisfies the constraints:

\n

\\[ e(P, Q + R) = e(P, Q) * e(P, R) \\]
\\[ e(P + S, Q) = e(P, Q) * e(S, Q) \\]
Note that + and * can be arbitrary operators; when you’re creating fancy new kinds of mathematical objects, abstract algebra doesn’t care how + and * are defined, as long as they are consistent in the usual ways.

\n

If P, Q, R and S were simple numbers, then making a simple pairing is easy: we can do e(x, y) = 2^xy. Then, we can see:
\\[e(3, 4+ 5) = 2^(3 * 9) = 2^{27}\\]
\\[e(3, 4) * e(3, 5) = 2^(3 * 4) * 2^(3 * 5) = 2^{12} * 2^{15} = 2^{27} \\]
However, such simple pairings are not suitable for cryptography because the objects that they work on are simple integers and are too easy to analyze;

\n

It turns out that it is possible to make a bilinear map over elliptic curve points — that is, come up with a function e(P, Q) where the inputs P and Q are elliptic curve points, and where the output is what’s called an \\(F_{p^¹²}\\) (extension field) element

\n

An elliptic curve pairing is a map G2 x G1 -> Gt, where:

\n
    \n
  • G1 is an elliptic curve, where points satisfy an equation of the form y² = x³ + b, and where both coordinates are elements of F_p (ie. they are simple numbers, except arithmetic is all done modulo some prime number)
    • \n
  • G2 is an elliptic curve, where points satisfy the same equation as G1, except where the coordinates are elements of \\(F_{p^¹²}\\). we define a new “magic number” w, which is defined by a 12th degree polynomial like w^12 - 18 * w^6 + 82 = 0
    • \n
  • Gt is the type of object that the result of the elliptic curve goes into. In the curves that we look at, Gt is \\(F_{p^¹²}\\)
    The main property that it must satisfy is bilinearity, which in presented above
    • \n
\n

There are two other important criteria:

\n
    \n
  • Efficient computability (eg. we can make an easy pairing by simply taking the discrete logarithms of all points and multiplying them together, but this is as computationally hard as breaking elliptic curve cryptography in the first place, so it doesn’t count)
    • \n
  • Non-degeneracy (sure, you could just define e(P, Q) = 1, but that’s not a particularly useful pairing)
    • \n
\n

math intuition behind paring

The math behind why pairing functions work is quite tricky and involves quite a bit of advanced algebra going even beyond what we’ve seen so far, but I’ll provide an outline. First of all, we need to define the concept of a divisor, basically an alternative way of representing functions on elliptic curve points. A divisor of a function basically counts the zeroes and the infinities of the function. To see what this means, let’s go through a few examples. Let us fix some point P = (P_x, P_y), and consider the following function:
\\[f(x, y) = x - P_x\\]

\n

The divisor is [P] + [-P] - 2 * [O] (the square brackets are used to represent the fact that we are referring to the presence of the point P in the set of zeroes and infinities of the function, not the point P itself; [P] + [Q] is not the same thing as [P + Q]). The reasoning is as follows:

\n
    \n
  • The function is equal to zero at P, since x is P_x, so x - P_x = 0
    • \n
  • The function is equal to zero at -P, since -P and P share the same x coordinate
    • \n
  • The function goes to infinity as x goes to infinity, so we say the function is equal to infinity at O. There’s a technical reason why this infinity needs to be counted twice, so O gets added with a “multiplicity” of -2 (negative because it’s an infinity and not a zero, two because of this double counting).
    The technical reason is roughly this: because the equation of the curve is x³ = y² + b, y goes to infinity “1.5 times faster” than x does in order for y² to keep up with x³; hence, if a linear function includes only x then it is represented as an infinity of multiplicity 2, but if it includes y then it is represented as an infinity of multiplicity 3.
    • \n
\n

Where a, b and c are carefully chosen so that the line passes through points P and Q. Because of how elliptic curve addition works (see the diagram at the top), this also means that it passes through -P-Q. And it goes up to infinity dependent on both x and y, so the divisor becomes [P]+ [Q] + [-P-Q] - 3 * [O]. (multiplicity of 3 because it involves y)
\"ec_pariling\"
We know that every “rational function” (ie. a function defined only using a finite number of +, -, * and / operations on the coordinates of the point) uniquely corresponds to some divisor, up to multiplication by a constant (ie. if two functions F and G have the same divisor, then F = G * k for some constant k).
For any two functions F and G, the divisor of F * G is equal to the divisor of F plus the divisor of G (in math textbooks, you’ll see (F * G) = (F) + (G)), so for example if f(x, y) = P_x - x, then (f³) = 3 * [P] + 3 * [-P] - 6 * [O]; P and -P are “triple-counted” to account for the fact that f³ approaches 0 at those points “three times as quickly” in a certain mathematical sense.

\n

Note that there is a theorem that states that if you “remove the square brackets” from a divisor of a function, the points must add up to O ([P] + [Q] + [-P-Q] - 3 * [O] clearly fits, as P + Q - P - Q - 3 * O = O), and any divisor that has this property is the divisor of a function.

\n

Tate pairings

Consider the following functions, defined via their divisors:

\n
    \n
  • (F_P) = n * [P] - n * [O], where n is the order of G1, ie. n * P = O for any P
    • \n
  • (F_Q) = n * [Q] - n * [O]
    • \n
  • (g) = [P + Q] - [P] - [Q] + [O]
    Now, let’s look at the product F_P * F_Q * g^n. The divisor is:
    • \n
\n

n * [P] - n * [O] + n * [Q] - n * [O] + n * [P + Q] - n * [P] - n * [Q] + n * [O]

\n

Which simplifies neatly to:

\n

n * [P + Q] - n * [O]
Notice that this divisor is of exactly the same format as the divisor for F_P and F_Q above. Hence, F_P * F_Q * g^n = F_(P + Q).

\n

Now, we introduce a procedure called the “final exponentiation” step, where we take the result of our functions above (F_P, F_Q, etc.) and raise it to the power z = (p¹² - 1) / n, where p¹² - 1 is the order of the multiplicative group in F_p¹² (ie. for any x ϵ F_p¹², x^(p¹² - 1) = 1). Notice that if you apply this exponentiation to any result that has already been raised to the power of n, you get an exponentiation to the power of p¹² - 1, so the result turns into 1. Hence, after final exponentiation, g^n cancels out and we get F_P^z * F_Q^z = F_(P + Q)^z. There’s some bilinearity for you.
Now, if you want to make a function that’s bilinear in both arguments, you need to go into spookier math, where instead of taking F_P of a value directly, you take F_P of a divisor, and that’s where the full “Tate pairing” comes from. To prove some more results you have to deal with notions like “linear equivalence” and “Weil reciprocity”, and the rabbit hole goes on from there. You can find more reading material on all of this [2] and he[3].
Every elliptic curve has a value called an embedding degree; essentially, the smallest k such that p^k - 1 is a multiple of n (where p is the prime used for the field and n is the curve order). In the fields above, k = 12, and in the fields used for traditional ECC (ie. where we don’t care about pairings), the embedding degree is often extremely large, to the point that pairings are computationally infeasible to compute; however, if we are not careful then we can generate fields where k = 4 or even 1.

\n

references

\n","site":{"data":{}},"excerpt":"","more":"\n\n\n

introduction

Pairings allow you to check certain kinds of more complicated equations on elliptic curve points — for example, if \\(P = G * p\\), \\(Q = G * q\\) and \\(R = G * r\\), you can check whether or not \\(p * q = r\\), having just P, Q and R as inputs. This might seem like the fundamental security guarantees of elliptic curves are being broken, as information about p is leaking from just knowing P, but it turns out that the leakage is highly contained — specifically, the decisional Diffie Hellman problem is easy, but the computational Diffie Hellman problem (knowing P and Q in the above example, computing R = G * p * q) and the discrete logarithm problem (recovering p from P) remain computationally infeasible.

\n

whereas traditional elliptic curve math lets you check linear constraints on the numbers (eg. if P = G * p, Q = G * q and R = G * r, checking 5 * P + 7 * Q = 11 * R is really checking that 5 * p + 7 * q = 11 * r), pairings let you check quadratic constraints (eg. checking e(P, Q) * e(G, G * 5) = 1 is really checking that p * q + 5 = 0).

\n

Now, what is this funny e(P, Q) operator that we introduced above? This is the pairing. Mathematicians also sometimes call it a bilinear map; the word “bilinear” here basically means that it satisfies the constraints:

\n

\\[ e(P, Q + R) = e(P, Q) * e(P, R) \\]
\\[ e(P + S, Q) = e(P, Q) * e(S, Q) \\]
Note that + and * can be arbitrary operators; when you’re creating fancy new kinds of mathematical objects, abstract algebra doesn’t care how + and * are defined, as long as they are consistent in the usual ways.

\n

If P, Q, R and S were simple numbers, then making a simple pairing is easy: we can do e(x, y) = 2^xy. Then, we can see:
\\[e(3, 4+ 5) = 2^(3 * 9) = 2^{27}\\]
\\[e(3, 4) * e(3, 5) = 2^(3 * 4) * 2^(3 * 5) = 2^{12} * 2^{15} = 2^{27} \\]
However, such simple pairings are not suitable for cryptography because the objects that they work on are simple integers and are too easy to analyze;

\n

It turns out that it is possible to make a bilinear map over elliptic curve points — that is, come up with a function e(P, Q) where the inputs P and Q are elliptic curve points, and where the output is what’s called an \\(F_{p^¹²}\\) (extension field) element

\n

An elliptic curve pairing is a map G2 x G1 -> Gt, where:

\n
    \n
  • G1 is an elliptic curve, where points satisfy an equation of the form y² = x³ + b, and where both coordinates are elements of F_p (ie. they are simple numbers, except arithmetic is all done modulo some prime number)
    • \n
  • G2 is an elliptic curve, where points satisfy the same equation as G1, except where the coordinates are elements of \\(F_{p^¹²}\\). we define a new “magic number” w, which is defined by a 12th degree polynomial like w^12 - 18 * w^6 + 82 = 0
    • \n
  • Gt is the type of object that the result of the elliptic curve goes into. In the curves that we look at, Gt is \\(F_{p^¹²}\\)
    The main property that it must satisfy is bilinearity, which in presented above
    • \n
\n

There are two other important criteria:

\n
    \n
  • Efficient computability (eg. we can make an easy pairing by simply taking the discrete logarithms of all points and multiplying them together, but this is as computationally hard as breaking elliptic curve cryptography in the first place, so it doesn’t count)
    • \n
  • Non-degeneracy (sure, you could just define e(P, Q) = 1, but that’s not a particularly useful pairing)
    • \n
\n

math intuition behind paring

The math behind why pairing functions work is quite tricky and involves quite a bit of advanced algebra going even beyond what we’ve seen so far, but I’ll provide an outline. First of all, we need to define the concept of a divisor, basically an alternative way of representing functions on elliptic curve points. A divisor of a function basically counts the zeroes and the infinities of the function. To see what this means, let’s go through a few examples. Let us fix some point P = (P_x, P_y), and consider the following function:
\\[f(x, y) = x - P_x\\]

\n

The divisor is [P] + [-P] - 2 * [O] (the square brackets are used to represent the fact that we are referring to the presence of the point P in the set of zeroes and infinities of the function, not the point P itself; [P] + [Q] is not the same thing as [P + Q]). The reasoning is as follows:

\n
    \n
  • The function is equal to zero at P, since x is P_x, so x - P_x = 0
    • \n
  • The function is equal to zero at -P, since -P and P share the same x coordinate
    • \n
  • The function goes to infinity as x goes to infinity, so we say the function is equal to infinity at O. There’s a technical reason why this infinity needs to be counted twice, so O gets added with a “multiplicity” of -2 (negative because it’s an infinity and not a zero, two because of this double counting).
    The technical reason is roughly this: because the equation of the curve is x³ = y² + b, y goes to infinity “1.5 times faster” than x does in order for y² to keep up with x³; hence, if a linear function includes only x then it is represented as an infinity of multiplicity 2, but if it includes y then it is represented as an infinity of multiplicity 3.
    • \n
\n

Where a, b and c are carefully chosen so that the line passes through points P and Q. Because of how elliptic curve addition works (see the diagram at the top), this also means that it passes through -P-Q. And it goes up to infinity dependent on both x and y, so the divisor becomes [P]+ [Q] + [-P-Q] - 3 * [O]. (multiplicity of 3 because it involves y)
\"ec_pariling\"
We know that every “rational function” (ie. a function defined only using a finite number of +, -, * and / operations on the coordinates of the point) uniquely corresponds to some divisor, up to multiplication by a constant (ie. if two functions F and G have the same divisor, then F = G * k for some constant k).
For any two functions F and G, the divisor of F * G is equal to the divisor of F plus the divisor of G (in math textbooks, you’ll see (F * G) = (F) + (G)), so for example if f(x, y) = P_x - x, then (f³) = 3 * [P] + 3 * [-P] - 6 * [O]; P and -P are “triple-counted” to account for the fact that f³ approaches 0 at those points “three times as quickly” in a certain mathematical sense.

\n

Note that there is a theorem that states that if you “remove the square brackets” from a divisor of a function, the points must add up to O ([P] + [Q] + [-P-Q] - 3 * [O] clearly fits, as P + Q - P - Q - 3 * O = O), and any divisor that has this property is the divisor of a function.

\n

Tate pairings

Consider the following functions, defined via their divisors:

\n
    \n
  • (F_P) = n * [P] - n * [O], where n is the order of G1, ie. n * P = O for any P
    • \n
  • (F_Q) = n * [Q] - n * [O]
    • \n
  • (g) = [P + Q] - [P] - [Q] + [O]
    Now, let’s look at the product F_P * F_Q * g^n. The divisor is:
    • \n
\n

n * [P] - n * [O] + n * [Q] - n * [O] + n * [P + Q] - n * [P] - n * [Q] + n * [O]

\n

Which simplifies neatly to:

\n

n * [P + Q] - n * [O]
Notice that this divisor is of exactly the same format as the divisor for F_P and F_Q above. Hence, F_P * F_Q * g^n = F_(P + Q).

\n

Now, we introduce a procedure called the “final exponentiation” step, where we take the result of our functions above (F_P, F_Q, etc.) and raise it to the power z = (p¹² - 1) / n, where p¹² - 1 is the order of the multiplicative group in F_p¹² (ie. for any x ϵ F_p¹², x^(p¹² - 1) = 1). Notice that if you apply this exponentiation to any result that has already been raised to the power of n, you get an exponentiation to the power of p¹² - 1, so the result turns into 1. Hence, after final exponentiation, g^n cancels out and we get F_P^z * F_Q^z = F_(P + Q)^z. There’s some bilinearity for you.
Now, if you want to make a function that’s bilinear in both arguments, you need to go into spookier math, where instead of taking F_P of a value directly, you take F_P of a divisor, and that’s where the full “Tate pairing” comes from. To prove some more results you have to deal with notions like “linear equivalence” and “Weil reciprocity”, and the rabbit hole goes on from there. You can find more reading material on all of this [2] and he[3].
Every elliptic curve has a value called an embedding degree; essentially, the smallest k such that p^k - 1 is a multiple of n (where p is the prime used for the field and n is the curve order). In the fields above, k = 12, and in the fields used for traditional ECC (ie. where we don’t care about pairings), the embedding degree is often extremely large, to the point that pairings are computationally infeasible to compute; however, if we are not careful then we can generate fields where k = 4 or even 1.

\n

references

\n"},{"title":"zkp how and why it works","date":"2023-07-01T06:29:26.000Z","_content":"\n\n\n## The medium of proof: Polynomial\nIf a prover claims to know some polynomial (no matter how large its degree is) that the verifier also knows, they can follow a simple protocol to verify the statement\n• Verifier chooses a random value for x and evaluates his polynomial locally\n• Verifier gives x to the prover and asks to evaluate the polynomial in question\n• Prover evaluates his polynomial at x and gives the result to the verifier\n• Verifier checks if the local result is equal to the prover’s result, and if so then the statement is proven with a high confidence\n\n## Non-Interactive Zero-Knowledge of a Polynomial\n1. Proving Knowledge of a Polynomial\nA polynomial can be expressed in the form (where n is the degree of the polynomial):\n\\\\[c_n x^n + ...+ c_1 x^1 + c_0 x^0\\\\]\nIt one claims that he know a polynomial, it is actually the knowledge of the polynomial's coefficients.\n\n2. Factorization\nThe Fundamental Theorem of Algebra states that any polynomial can be factored into linear po- lynomials (i.e., a degree 1 polynomials representing a line), as long it is solvable. Consequently, we can represent any valid polynomial as a product of its factors:\n\\\\[(x-a_0)(x-a_1)...(x-a_n) = 0\\\\]\n\nif the prover wants to prove that indeed his polynomial has specific roots without disclosing the polynomial itself, he needs to prove that his polynomial \\\\(p(x)\\\\) is the multiplication of those cofactors \\\\(t(x) = (x − x_0)(x − x_1)\\\\), called target polynomial, where \\\\(x_0, x_1\\\\) are the specific roots. i.e.:\n\\\\[p(x) = t(x) \\cdot h(x)\\\\]\nA natural way to find \\\\(h(x)\\\\) is through the division \\\\(h(x) = \\frac{p(x)}{t(x)}\\\\)\n> for simplicity, onwards we will use polynomial’s letter variable to denote its evaluation, e.g., \\\\(p = p(r)\\\\)\n\nUsing our polynomial identity check protocol we can compare polynomials \\\\(p(x)\\\\) and \\\\(t(x)·h(x)\\\\):\n- Verifier samples a random value \\\\(r\\\\), calculates \\\\(t = t(r)\\\\) (i.e., evaluates) and gives \\\\(r\\\\) to the prover\n- Prover calculates \\\\(h(x) = \\frac{p(x)}{t(x)}\\\\) and evaluates \\\\(p(r)\\\\) and \\\\(h(r)\\\\); the resulting values \\\\(p,h\\\\) are\nprovided to the verifier\n- Verifier then checks that \\\\(p = t \\cdot h\\\\), if so those polynomials are equal, meaning that \\\\(p(x)\\\\) has \\\\(t(x)\\\\) as a cofactor.\n\n**Remark 3.1** Now we can check a polynomial for specific properties without learning the polyno- mial itself, so this already gives us some form of zero-knowledge and succinctness. Nonetheless, there are multiple issues with this construction:\n• Prover may not know the claimed polynomial \\\\(p(x)\\\\) at all. He can calculate evaluation \\\\(t = t(r)\\\\), select a random number \\\\(h\\\\) and set \\\\(p = t \\cdot h\\\\), which will be accepted by the verifier as valid, since equation holds.\n• Because prover knows the random point \\\\(x = r\\\\), he can construct any polynomial which has one shared point at \\\\(r\\\\) with \\\\(t(r) \\cdot h(r)\\\\).\n• In the original statement, prover claims to know a polynomial of a particular degree, in the current protocol there is no enforcement of degree. Hence prover can cheat by using a polynomial of higher degree which also satisfies the cofactors check.\n\n3. Obscure Evaluation\nTwo first issues of remark 3.1 are possible because values are presented at raw, prover knows r and t(r). It would be ideal if those values would be given as a black box, so one cannot temper with the protocol, but still able to compute operations on those obscure values. \n3.1. Homomorphic Encryption\nThere are multiple ways to achieve homomorphic properties of encryption, and we will briefly introduce a simple one. The general idea is to choose a base number \\\\(g\\\\) and do ecncryption of a value \\\\(x\\\\) by exponentiate of \\\\(g\\\\)\n\\\\[E(x) = g^{x} \\bmod p\\\\]\nFor example, let \\\\(E(x_1) = g^{x_1} \\bmod p\\\\), and \\\\(E(x_2) = g^{x_2} \\bmod p\\\\), then\n\\\\[E(x_2) \\cdot E(x_1) = g^{x_1 + x_2} = E(x_1 + x_2) \\\\]\n\n3.2. Encrypted Polynomial\nLet us see how we can evaluate a polynomial \\\\(p(x) = x^3 − 3x^2 + 2x\\\\). Because homomorphic encryption does not allows to exponentiate an encrypted value, we’ve must been given encrypted values of powers of x from 1 to 3: \\\\(E(x),E(x2),E(x3)\\\\) so that\n\\\\[ E(x^3)^1 \\cdot E(x^2)^{-3} \\cdot E(x)^2 = (g^{x^3})^{1} \\cdot (g^{x^2})^{-3} \\cdot (g^{x})^{2} = g^{1x^3} \\cdot g^{-3x^2} \\cdot g^{2x} = g^{x^3-3x^2+2x}\\\\]\nHence, we have an encrypted evaluation of our polynomial at some unknown to us \\\\(x\\\\) \n\nWe can now update the previous version of the protocol, for a polynomial fo degree \\\\(d\\\\):\n- Verifier\n - samples a random value \\\\(s\\\\), i.e., secret\n - calculates encryptions of \\\\(s\\\\) for all powers \\\\(i\\\\) in \\\\(0,1,...,d\\\\), i.e. : \\\\(E(s^{i}) = g^{s^{i}}\\\\)\n - evaluates unencrypted target polynomial with \\\\(s: t(s)\\\\)\n - encrypted powers of \\\\(s\\\\) are provided to the prover: \\\\(E(s^{0}),E(s^{1}),...,E(s^{d})\\\\)\n- Prover\n - calculates polynomial \\\\(h(x) = \\frac{p(x)}{t(x)}\\\\)\n - using encrypted powers \\\\(g^{s^{0}},g^{s^{1}},...,g^{s^{d}}\\\\) and coefficients \\\\(c_0, c_1,...,c_n \\\\) evaluates \\\\(E(p(s)) = g^{p(s)} = (g^{s^{d}})^{c_d} \\cdot \\cdot \\cdot (g^{s^{1}})^{c_1} \\cdot (g^{s^{0}})^{c_0}\\\\) and similarly \\\\(E(h(s)) = g^{h(s)}\\\\)\n - the resulting \\\\(g^p\\\\) and \\\\(g^h\\\\) are provided to the verifier\n- Verifier\n - The alst step for the verifier is to checks that \\\\(p = t(s) \\cdot h \\\\) in encrypted space: \\\\(g^p = (g^h)^{t(s)}\\\\) => \\\\(g^p = g^{t(s) \\cdot h}\\\\)\n\n> Note: because the prover does not know anything about s, it makes it hard to come up with non-legitimate but still matching evaluations.\n## referneces\n- [why and how zk-SNARK works by Maksym](https://arxiv.org/pdf/1906.07221.pdf)","source":"_posts/cryptography/zkp/zkp-under-the-hood.md","raw":"---\ntitle: zkp how and why it works\ndate: 2023-07-01 14:29:26\ntags: [cryptography,zkp]\n---\n\n\n\n## The medium of proof: Polynomial\nIf a prover claims to know some polynomial (no matter how large its degree is) that the verifier also knows, they can follow a simple protocol to verify the statement\n• Verifier chooses a random value for x and evaluates his polynomial locally\n• Verifier gives x to the prover and asks to evaluate the polynomial in question\n• Prover evaluates his polynomial at x and gives the result to the verifier\n• Verifier checks if the local result is equal to the prover’s result, and if so then the statement is proven with a high confidence\n\n## Non-Interactive Zero-Knowledge of a Polynomial\n1. Proving Knowledge of a Polynomial\nA polynomial can be expressed in the form (where n is the degree of the polynomial):\n\\\\[c_n x^n + ...+ c_1 x^1 + c_0 x^0\\\\]\nIt one claims that he know a polynomial, it is actually the knowledge of the polynomial's coefficients.\n\n2. Factorization\nThe Fundamental Theorem of Algebra states that any polynomial can be factored into linear po- lynomials (i.e., a degree 1 polynomials representing a line), as long it is solvable. Consequently, we can represent any valid polynomial as a product of its factors:\n\\\\[(x-a_0)(x-a_1)...(x-a_n) = 0\\\\]\n\nif the prover wants to prove that indeed his polynomial has specific roots without disclosing the polynomial itself, he needs to prove that his polynomial \\\\(p(x)\\\\) is the multiplication of those cofactors \\\\(t(x) = (x − x_0)(x − x_1)\\\\), called target polynomial, where \\\\(x_0, x_1\\\\) are the specific roots. i.e.:\n\\\\[p(x) = t(x) \\cdot h(x)\\\\]\nA natural way to find \\\\(h(x)\\\\) is through the division \\\\(h(x) = \\frac{p(x)}{t(x)}\\\\)\n> for simplicity, onwards we will use polynomial’s letter variable to denote its evaluation, e.g., \\\\(p = p(r)\\\\)\n\nUsing our polynomial identity check protocol we can compare polynomials \\\\(p(x)\\\\) and \\\\(t(x)·h(x)\\\\):\n- Verifier samples a random value \\\\(r\\\\), calculates \\\\(t = t(r)\\\\) (i.e., evaluates) and gives \\\\(r\\\\) to the prover\n- Prover calculates \\\\(h(x) = \\frac{p(x)}{t(x)}\\\\) and evaluates \\\\(p(r)\\\\) and \\\\(h(r)\\\\); the resulting values \\\\(p,h\\\\) are\nprovided to the verifier\n- Verifier then checks that \\\\(p = t \\cdot h\\\\), if so those polynomials are equal, meaning that \\\\(p(x)\\\\) has \\\\(t(x)\\\\) as a cofactor.\n\n**Remark 3.1** Now we can check a polynomial for specific properties without learning the polyno- mial itself, so this already gives us some form of zero-knowledge and succinctness. Nonetheless, there are multiple issues with this construction:\n• Prover may not know the claimed polynomial \\\\(p(x)\\\\) at all. He can calculate evaluation \\\\(t = t(r)\\\\), select a random number \\\\(h\\\\) and set \\\\(p = t \\cdot h\\\\), which will be accepted by the verifier as valid, since equation holds.\n• Because prover knows the random point \\\\(x = r\\\\), he can construct any polynomial which has one shared point at \\\\(r\\\\) with \\\\(t(r) \\cdot h(r)\\\\).\n• In the original statement, prover claims to know a polynomial of a particular degree, in the current protocol there is no enforcement of degree. Hence prover can cheat by using a polynomial of higher degree which also satisfies the cofactors check.\n\n3. Obscure Evaluation\nTwo first issues of remark 3.1 are possible because values are presented at raw, prover knows r and t(r). It would be ideal if those values would be given as a black box, so one cannot temper with the protocol, but still able to compute operations on those obscure values. \n3.1. Homomorphic Encryption\nThere are multiple ways to achieve homomorphic properties of encryption, and we will briefly introduce a simple one. The general idea is to choose a base number \\\\(g\\\\) and do ecncryption of a value \\\\(x\\\\) by exponentiate of \\\\(g\\\\)\n\\\\[E(x) = g^{x} \\bmod p\\\\]\nFor example, let \\\\(E(x_1) = g^{x_1} \\bmod p\\\\), and \\\\(E(x_2) = g^{x_2} \\bmod p\\\\), then\n\\\\[E(x_2) \\cdot E(x_1) = g^{x_1 + x_2} = E(x_1 + x_2) \\\\]\n\n3.2. Encrypted Polynomial\nLet us see how we can evaluate a polynomial \\\\(p(x) = x^3 − 3x^2 + 2x\\\\). Because homomorphic encryption does not allows to exponentiate an encrypted value, we’ve must been given encrypted values of powers of x from 1 to 3: \\\\(E(x),E(x2),E(x3)\\\\) so that\n\\\\[ E(x^3)^1 \\cdot E(x^2)^{-3} \\cdot E(x)^2 = (g^{x^3})^{1} \\cdot (g^{x^2})^{-3} \\cdot (g^{x})^{2} = g^{1x^3} \\cdot g^{-3x^2} \\cdot g^{2x} = g^{x^3-3x^2+2x}\\\\]\nHence, we have an encrypted evaluation of our polynomial at some unknown to us \\\\(x\\\\) \n\nWe can now update the previous version of the protocol, for a polynomial fo degree \\\\(d\\\\):\n- Verifier\n - samples a random value \\\\(s\\\\), i.e., secret\n - calculates encryptions of \\\\(s\\\\) for all powers \\\\(i\\\\) in \\\\(0,1,...,d\\\\), i.e. : \\\\(E(s^{i}) = g^{s^{i}}\\\\)\n - evaluates unencrypted target polynomial with \\\\(s: t(s)\\\\)\n - encrypted powers of \\\\(s\\\\) are provided to the prover: \\\\(E(s^{0}),E(s^{1}),...,E(s^{d})\\\\)\n- Prover\n - calculates polynomial \\\\(h(x) = \\frac{p(x)}{t(x)}\\\\)\n - using encrypted powers \\\\(g^{s^{0}},g^{s^{1}},...,g^{s^{d}}\\\\) and coefficients \\\\(c_0, c_1,...,c_n \\\\) evaluates \\\\(E(p(s)) = g^{p(s)} = (g^{s^{d}})^{c_d} \\cdot \\cdot \\cdot (g^{s^{1}})^{c_1} \\cdot (g^{s^{0}})^{c_0}\\\\) and similarly \\\\(E(h(s)) = g^{h(s)}\\\\)\n - the resulting \\\\(g^p\\\\) and \\\\(g^h\\\\) are provided to the verifier\n- Verifier\n - The alst step for the verifier is to checks that \\\\(p = t(s) \\cdot h \\\\) in encrypted space: \\\\(g^p = (g^h)^{t(s)}\\\\) => \\\\(g^p = g^{t(s) \\cdot h}\\\\)\n\n> Note: because the prover does not know anything about s, it makes it hard to come up with non-legitimate but still matching evaluations.\n## referneces\n- [why and how zk-SNARK works by Maksym](https://arxiv.org/pdf/1906.07221.pdf)","slug":"cryptography/zkp/zkp-under-the-hood","published":1,"updated":"2023-11-05T04:21:17.035Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8e2003iqwsj23zw9035","content":"\n\n\n

The medium of proof: Polynomial

If a prover claims to know some polynomial (no matter how large its degree is) that the verifier also knows, they can follow a simple protocol to verify the statement
• Verifier chooses a random value for x and evaluates his polynomial locally
• Verifier gives x to the prover and asks to evaluate the polynomial in question
• Prover evaluates his polynomial at x and gives the result to the verifier
• Verifier checks if the local result is equal to the prover’s result, and if so then the statement is proven with a high confidence

\n

Non-Interactive Zero-Knowledge of a Polynomial

    \n

  1. Proving Knowledge of a Polynomial
    A polynomial can be expressed in the form (where n is the degree of the polynomial):
    \\[c_n x^n + …+ c_1 x^1 + c_0 x^0\\]
    It one claims that he know a polynomial, it is actually the knowledge of the polynomial’s coefficients.

    \n
    2. \n

  2. Factorization
    The Fundamental Theorem of Algebra states that any polynomial can be factored into linear po- lynomials (i.e., a degree 1 polynomials representing a line), as long it is solvable. Consequently, we can represent any valid polynomial as a product of its factors:
    \\[(x-a_0)(x-a_1)…(x-a_n) = 0\\]

    \n
    3. \n
\n

if the prover wants to prove that indeed his polynomial has specific roots without disclosing the polynomial itself, he needs to prove that his polynomial \\(p(x)\\) is the multiplication of those cofactors \\(t(x) = (x − x_0)(x − x_1)\\), called target polynomial, where \\(x_0, x_1\\) are the specific roots. i.e.:
\\[p(x) = t(x) \\cdot h(x)\\]
A natural way to find \\(h(x)\\) is through the division \\(h(x) = \\frac{p(x)}{t(x)}\\)

\n
\n

for simplicity, onwards we will use polynomial’s letter variable to denote its evaluation, e.g., \\(p = p(r)\\)

\n
\n

Using our polynomial identity check protocol we can compare polynomials \\(p(x)\\) and \\(t(x)·h(x)\\):

\n
    \n
  • Verifier samples a random value \\(r\\), calculates \\(t = t(r)\\) (i.e., evaluates) and gives \\(r\\) to the prover
    • \n
  • Prover calculates \\(h(x) = \\frac{p(x)}{t(x)}\\) and evaluates \\(p(r)\\) and \\(h(r)\\); the resulting values \\(p,h\\) are
    provided to the verifier
    • \n
  • Verifier then checks that \\(p = t \\cdot h\\), if so those polynomials are equal, meaning that \\(p(x)\\) has \\(t(x)\\) as a cofactor.
    • \n
\n

Remark 3.1 Now we can check a polynomial for specific properties without learning the polyno- mial itself, so this already gives us some form of zero-knowledge and succinctness. Nonetheless, there are multiple issues with this construction:
• Prover may not know the claimed polynomial \\(p(x)\\) at all. He can calculate evaluation \\(t = t(r)\\), select a random number \\(h\\) and set \\(p = t \\cdot h\\), which will be accepted by the verifier as valid, since equation holds.
• Because prover knows the random point \\(x = r\\), he can construct any polynomial which has one shared point at \\(r\\) with \\(t(r) \\cdot h(r)\\).
• In the original statement, prover claims to know a polynomial of a particular degree, in the current protocol there is no enforcement of degree. Hence prover can cheat by using a polynomial of higher degree which also satisfies the cofactors check.

\n
    \n
  1. Obscure Evaluation
    Two first issues of remark 3.1 are possible because values are presented at raw, prover knows r and t(r). It would be ideal if those values would be given as a black box, so one cannot temper with the protocol, but still able to compute operations on those obscure values.
    3.1. Homomorphic Encryption
    There are multiple ways to achieve homomorphic properties of encryption, and we will briefly introduce a simple one. The general idea is to choose a base number \\(g\\) and do ecncryption of a value \\(x\\) by exponentiate of \\(g\\)
    \\[E(x) = g^{x} \\bmod p\\]
    For example, let \\(E(x_1) = g^{x_1} \\bmod p\\), and \\(E(x_2) = g^{x_2} \\bmod p\\), then
    \\[E(x_2) \\cdot E(x_1) = g^{x_1 + x_2} = E(x_1 + x_2) \\]
    2. \n
\n

3.2. Encrypted Polynomial
Let us see how we can evaluate a polynomial \\(p(x) = x^3 − 3x^2 + 2x\\). Because homomorphic encryption does not allows to exponentiate an encrypted value, we’ve must been given encrypted values of powers of x from 1 to 3: \\(E(x),E(x2),E(x3)\\) so that
\\[ E(x^3)^1 \\cdot E(x^2)^{-3} \\cdot E(x)^2 = (g^{x^3})^{1} \\cdot (g^{x^2})^{-3} \\cdot (g^{x})^{2} = g^{1x^3} \\cdot g^{-3x^2} \\cdot g^{2x} = g^{x^3-3x^2+2x}\\]
Hence, we have an encrypted evaluation of our polynomial at some unknown to us \\(x\\)

\n

We can now update the previous version of the protocol, for a polynomial fo degree \\(d\\):

\n
    \n
  • Verifier
      \n
    • samples a random value \\(s\\), i.e., secret
      • \n
    • calculates encryptions of \\(s\\) for all powers \\(i\\) in \\(0,1,…,d\\), i.e. : \\(E(s^{i}) = g^{s^{i}}\\)
      • \n
    • evaluates unencrypted target polynomial with \\(s: t(s)\\)
      • \n
    • encrypted powers of \\(s\\) are provided to the prover: \\(E(s^{0}),E(s^{1}),…,E(s^{d})\\)
      • \n
    \n
    • \n
  • Prover
      \n
    • calculates polynomial \\(h(x) = \\frac{p(x)}{t(x)}\\)
      • \n
    • using encrypted powers \\(g^{s^{0}},g^{s^{1}},…,g^{s^{d}}\\) and coefficients \\(c_0, c_1,…,c_n \\) evaluates \\(E(p(s)) = g^{p(s)} = (g^{s^{d}})^{c_d} \\cdot \\cdot \\cdot (g^{s^{1}})^{c_1} \\cdot (g^{s^{0}})^{c_0}\\) and similarly \\(E(h(s)) = g^{h(s)}\\)
      • \n
    • the resulting \\(g^p\\) and \\(g^h\\) are provided to the verifier
      • \n
    \n
    • \n
  • Verifier
      \n
    • The alst step for the verifier is to checks that \\(p = t(s) \\cdot h \\) in encrypted space: \\(g^p = (g^h)^{t(s)}\\) => \\(g^p = g^{t(s) \\cdot h}\\)
      • \n
    \n
    • \n
\n
\n

Note: because the prover does not know anything about s, it makes it hard to come up with non-legitimate but still matching evaluations.

\n
\n

referneces

\n","site":{"data":{}},"excerpt":"","more":"\n\n\n

The medium of proof: Polynomial

If a prover claims to know some polynomial (no matter how large its degree is) that the verifier also knows, they can follow a simple protocol to verify the statement
• Verifier chooses a random value for x and evaluates his polynomial locally
• Verifier gives x to the prover and asks to evaluate the polynomial in question
• Prover evaluates his polynomial at x and gives the result to the verifier
• Verifier checks if the local result is equal to the prover’s result, and if so then the statement is proven with a high confidence

\n

Non-Interactive Zero-Knowledge of a Polynomial

    \n

  1. Proving Knowledge of a Polynomial
    A polynomial can be expressed in the form (where n is the degree of the polynomial):
    \\[c_n x^n + …+ c_1 x^1 + c_0 x^0\\]
    It one claims that he know a polynomial, it is actually the knowledge of the polynomial’s coefficients.

    \n
    2. \n

  2. Factorization
    The Fundamental Theorem of Algebra states that any polynomial can be factored into linear po- lynomials (i.e., a degree 1 polynomials representing a line), as long it is solvable. Consequently, we can represent any valid polynomial as a product of its factors:
    \\[(x-a_0)(x-a_1)…(x-a_n) = 0\\]

    \n
    3. \n
\n

if the prover wants to prove that indeed his polynomial has specific roots without disclosing the polynomial itself, he needs to prove that his polynomial \\(p(x)\\) is the multiplication of those cofactors \\(t(x) = (x − x_0)(x − x_1)\\), called target polynomial, where \\(x_0, x_1\\) are the specific roots. i.e.:
\\[p(x) = t(x) \\cdot h(x)\\]
A natural way to find \\(h(x)\\) is through the division \\(h(x) = \\frac{p(x)}{t(x)}\\)

\n
\n

for simplicity, onwards we will use polynomial’s letter variable to denote its evaluation, e.g., \\(p = p(r)\\)

\n
\n

Using our polynomial identity check protocol we can compare polynomials \\(p(x)\\) and \\(t(x)·h(x)\\):

\n
    \n
  • Verifier samples a random value \\(r\\), calculates \\(t = t(r)\\) (i.e., evaluates) and gives \\(r\\) to the prover
    • \n
  • Prover calculates \\(h(x) = \\frac{p(x)}{t(x)}\\) and evaluates \\(p(r)\\) and \\(h(r)\\); the resulting values \\(p,h\\) are
    provided to the verifier
    • \n
  • Verifier then checks that \\(p = t \\cdot h\\), if so those polynomials are equal, meaning that \\(p(x)\\) has \\(t(x)\\) as a cofactor.
    • \n
\n

Remark 3.1 Now we can check a polynomial for specific properties without learning the polyno- mial itself, so this already gives us some form of zero-knowledge and succinctness. Nonetheless, there are multiple issues with this construction:
• Prover may not know the claimed polynomial \\(p(x)\\) at all. He can calculate evaluation \\(t = t(r)\\), select a random number \\(h\\) and set \\(p = t \\cdot h\\), which will be accepted by the verifier as valid, since equation holds.
• Because prover knows the random point \\(x = r\\), he can construct any polynomial which has one shared point at \\(r\\) with \\(t(r) \\cdot h(r)\\).
• In the original statement, prover claims to know a polynomial of a particular degree, in the current protocol there is no enforcement of degree. Hence prover can cheat by using a polynomial of higher degree which also satisfies the cofactors check.

\n
    \n
  1. Obscure Evaluation
    Two first issues of remark 3.1 are possible because values are presented at raw, prover knows r and t(r). It would be ideal if those values would be given as a black box, so one cannot temper with the protocol, but still able to compute operations on those obscure values.
    3.1. Homomorphic Encryption
    There are multiple ways to achieve homomorphic properties of encryption, and we will briefly introduce a simple one. The general idea is to choose a base number \\(g\\) and do ecncryption of a value \\(x\\) by exponentiate of \\(g\\)
    \\[E(x) = g^{x} \\bmod p\\]
    For example, let \\(E(x_1) = g^{x_1} \\bmod p\\), and \\(E(x_2) = g^{x_2} \\bmod p\\), then
    \\[E(x_2) \\cdot E(x_1) = g^{x_1 + x_2} = E(x_1 + x_2) \\]
    2. \n
\n

3.2. Encrypted Polynomial
Let us see how we can evaluate a polynomial \\(p(x) = x^3 − 3x^2 + 2x\\). Because homomorphic encryption does not allows to exponentiate an encrypted value, we’ve must been given encrypted values of powers of x from 1 to 3: \\(E(x),E(x2),E(x3)\\) so that
\\[ E(x^3)^1 \\cdot E(x^2)^{-3} \\cdot E(x)^2 = (g^{x^3})^{1} \\cdot (g^{x^2})^{-3} \\cdot (g^{x})^{2} = g^{1x^3} \\cdot g^{-3x^2} \\cdot g^{2x} = g^{x^3-3x^2+2x}\\]
Hence, we have an encrypted evaluation of our polynomial at some unknown to us \\(x\\)

\n

We can now update the previous version of the protocol, for a polynomial fo degree \\(d\\):

\n
    \n
  • Verifier
      \n
    • samples a random value \\(s\\), i.e., secret
      • \n
    • calculates encryptions of \\(s\\) for all powers \\(i\\) in \\(0,1,…,d\\), i.e. : \\(E(s^{i}) = g^{s^{i}}\\)
      • \n
    • evaluates unencrypted target polynomial with \\(s: t(s)\\)
      • \n
    • encrypted powers of \\(s\\) are provided to the prover: \\(E(s^{0}),E(s^{1}),…,E(s^{d})\\)
      • \n
    \n
    • \n
  • Prover
      \n
    • calculates polynomial \\(h(x) = \\frac{p(x)}{t(x)}\\)
      • \n
    • using encrypted powers \\(g^{s^{0}},g^{s^{1}},…,g^{s^{d}}\\) and coefficients \\(c_0, c_1,…,c_n \\) evaluates \\(E(p(s)) = g^{p(s)} = (g^{s^{d}})^{c_d} \\cdot \\cdot \\cdot (g^{s^{1}})^{c_1} \\cdot (g^{s^{0}})^{c_0}\\) and similarly \\(E(h(s)) = g^{h(s)}\\)
      • \n
    • the resulting \\(g^p\\) and \\(g^h\\) are provided to the verifier
      • \n
    \n
    • \n
  • Verifier
      \n
    • The alst step for the verifier is to checks that \\(p = t(s) \\cdot h \\) in encrypted space: \\(g^p = (g^h)^{t(s)}\\) => \\(g^p = g^{t(s) \\cdot h}\\)
      • \n
    \n
    • \n
\n
\n

Note: because the prover does not know anything about s, it makes it hard to come up with non-legitimate but still matching evaluations.

\n
\n

referneces

\n"},{"title":"rust frequently used crates","date":"2022-12-13T09:15:23.000Z","_content":"\n\ntokio-trace -> tracing\n\ncontexts, multi threads\ncausality -\nstructured diagnostics ( no grep)\n\ntracing is part of tokio, tokio not requied\n\nspans: a perido of time, entered and exited\nevents: singular moment in time\nsubscriber: collect trace","source":"_posts/rust/crates/rust-frequently-used-crates.md","raw":"---\ntitle: rust frequently used crates\ndate: 2022-12-13 17:15:23\ntags: [rust]\n---\n\n\ntokio-trace -> tracing\n\ncontexts, multi threads\ncausality -\nstructured diagnostics ( no grep)\n\ntracing is part of tokio, tokio not requied\n\nspans: a perido of time, entered and exited\nevents: singular moment in time\nsubscriber: collect trace","slug":"rust/crates/rust-frequently-used-crates","published":1,"updated":"2023-11-05T04:21:13.733Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8e2003kqwsjg4hf5noa","content":"

tokio-trace -> tracing

\n

contexts, multi threads
causality -
structured diagnostics ( no grep)

\n

tracing is part of tokio, tokio not requied

\n

spans: a perido of time, entered and exited
events: singular moment in time
subscriber: collect trace

\n","site":{"data":{}},"excerpt":"","more":"

tokio-trace -> tracing

\n

contexts, multi threads
causality -
structured diagnostics ( no grep)

\n

tracing is part of tokio, tokio not requied

\n

spans: a perido of time, entered and exited
events: singular moment in time
subscriber: collect trace

\n"},{"title":"rust crate serde","date":"2023-04-01T14:04:38.000Z","_content":"\n```rust\n#[serde(tag = \"filterType\")]\n#[serde(untagged)]\n#[serde(rename = \"PRICE_FILTER\")]\n#[serde(rename_all = \"camelCase\")]\n\n#[serde(with = \"string_or_float\")]\npub stop_price: f64,\n```","source":"_posts/rust/crates/rust-serde.md","raw":"---\ntitle: rust crate serde\ndate: 2023-04-01 22:04:38\ntags: [rust-crate]\n---\n\n```rust\n#[serde(tag = \"filterType\")]\n#[serde(untagged)]\n#[serde(rename = \"PRICE_FILTER\")]\n#[serde(rename_all = \"camelCase\")]\n\n#[serde(with = \"string_or_float\")]\npub stop_price: f64,\n```","slug":"rust/crates/rust-serde","published":1,"updated":"2023-11-05T04:21:13.734Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8e2003mqwsj9ztifql4","content":"
1
2
3
4
5
6
7
#[serde(tag = "filterType")]
#[serde(untagged)]
#[serde(rename = "PRICE_FILTER")]
#[serde(rename_all = "camelCase")]

#[serde(with = "string_or_float")]
pub stop_price: f64,
","site":{"data":{}},"excerpt":"","more":"
1
2
3
4
5
6
7
#[serde(tag = "filterType")]
#[serde(untagged)]
#[serde(rename = "PRICE_FILTER")]
#[serde(rename_all = "camelCase")]

#[serde(with = "string_or_float")]
pub stop_price: f64,
"},{"title":"rust std data structure (1D)","date":"2023-05-01T14:04:38.000Z","_content":"\n## array\nA **fixed-size** array, denoted [T; N], for the element type, T, and the non-negative compile-time constant size, N.\n```rust\ntodo!()\n```\n\n## slice\nA **dynamically-sized view** into a contiguous sequence, [T].\n- `len()`: Returns the number of elements in the slice\n- `is_empty()`\n- `first()` Returns the first element of the slice, or `None` if it is empty.\n- `first_mut()` Returns a mutable **pointer** to the first element of the slice, or `None` if it is empty\n- `split_first()` Returns the first and all the rest of the elements of the slice, or `None` if it is empty.\n- `split_first_mut()` \n- `split_last()`\n- `split_last_mut()`\n- `last()`\n- `last_mut()`\n- `get(index: I)` Returns a reference to an element or subslice depending on the type of index.\n```rust\nlet v = [10, 40, 30];\nassert_eq!(Some(&40), v.get(1));\nassert_eq!(Some(&[10, 40][..]), v.get(0..2));\n```\n- `get_mut(index: I)`\n- `get_unchecked(index: I)` Returns a reference to an element or subslice, without doing bounds checking\n- `get_unchecked_mut(index: I)`\n- `as_ptr(&self) -> *const T` Returns a raw pointer to the slice's buffer\n```rust\nlet x = &[1, 2, 4];\nlet x_ptr = x.as_ptr();\nunsafe {\n for i in 0..x.len() {\n assert_eq!(x.get_unchecked(i), &*x_ptr.add(i));\n }\n}\n```\n- `as_mut_ptr(&mut self) -> *mut T` \n```rust\nlet x = &mut [1, 2, 4];\nlet x_ptr = x.as_mut_ptr();\nunsafe {\n for i in 0..x.len() {\n *x_ptr.add(i) += 2;\n }\n}\nassert_eq!(x, &[3, 4, 6]);\n```\n- `as_ptr_range(&self) -> Range<*const T>` Returns the two raw pointers spanning the slice.\n```rust\npub const fn as_ptr_range(&self) -> Range<*const T> {\n let start = self.as_ptr();\n let end = unsafe { start.add(self.len()) };\n start..end\n}\n```\n- `as_mut_ptr_range(&mut self) -> Range<*mut T>`\n- `swap(&mut self, a: usize, b: usize)` Swaps two elements in the slice.\n- `reverse(&mut self)` Reverses the order of elements in the slice, in place.\n- `windows(&self, size: usize)` Returns an iterator over all contiguous windows of length `size`. The windows overlap. If the slice is shorter than `size`, the iterator returns no values.\n```rust\nlet slice = ['r', 'u', 's', 't'];\nlet mut iter = slice.windows(2);\nassert_eq!(iter.next().unwrap(), &['r', 'u']);\nassert_eq!(iter.next().unwrap(), &['u', 's']);\nassert_eq!(iter.next().unwrap(), &['s', 't']);\nassert!(iter.next().is_none());\n```\n- `chunks(&self, chunk_size: usize)` Returns an iterator over `chunk_size` elements of the slice at a time\n```rust\nlet slice = ['l', 'o', 'r', 'e', 'm'];\nlet mut iter = slice.chunks(2);\nassert_eq!(iter.next().unwrap(), &['l', 'o']);\nassert_eq!(iter.next().unwrap(), &['r', 'e']);\nassert_eq!(iter.next().unwrap(), &['m']);\nassert!(iter.next().is_none());\n```\n- `chunks_mut()`\n- `chunks_exact(&self, chunk_size: usize)`\n```rust\nlet slice = ['l', 'o', 'r', 'e', 'm'];\nlet mut iter = slice.chunks_exact(2);\nassert_eq!(iter.next().unwrap(), &['l', 'o']);\nassert_eq!(iter.next().unwrap(), &['r', 'e']);\nassert!(iter.next().is_none());\nassert_eq!(iter.remainder(), &['m']);\n```\n- `as_chunks_unchecked(&self)` Splits the slice into a slice of `N`-element arrays, assuming that there's no remainder\n- `as_chunks(&self)` Splits the slice into a slice of `N`-element arrays, starting at the beginning of the slice, and a remainder slice with length strictly less than `N`\n- `as_rchunks(&self)` r means reverse\n- `group_by(&self, pred: F)` Returns an iterator over the slice producing non-overlapping runs of elements using the predicate to separate them. The predicate is called on two elements following themselves, it means the predicate is called on `slice[0]` and `slice[1]` then on `slice[1]` and `slice[2]` and so on\n```rust\n#![feature(slice_group_by)]\nlet slice = &[1, 1, 2, 3, 2, 3, 2, 3, 4];\nlet mut iter = slice.group_by(|a, b| a <= b);\nassert_eq!(iter.next(), Some(&[1, 1, 2, 3][..]));\nassert_eq!(iter.next(), Some(&[2, 3][..]));\nassert_eq!(iter.next(), Some(&[2, 3, 4][..]));\nassert_eq!(iter.next(), None);\n```\n- `split_at(&self, mid: usize)` Divides one slice into two at an index.\n- `split(&self, pred: F)` Returns an iterator over subslices separated by elements that match `pred`. The matched element is not contained in the subslices.\n- `splitn(&self, n: usize, pred: F)` \n- `contains(&self, x: &T)` Returns `true` if the slice contains an element with the given value.\n- `starts_with(&self, needle: &[T])` eturns `true` if `needle` is a prefix of the slice\n```rust\nlet v = [10, 40, 30];\nassert!(v.starts_with(&[10]));\nassert!(v.starts_with(&[10, 40]));\nassert!(!v.starts_with(&[50]));\n```\n- `ends_with(&self, needle: &[T])` \n- `strip_prefix` Returns a subslice with the prefix removed.\n```rust\nlet v = &[10, 40, 30];\nassert_eq!(v.strip_prefix(&[10]), Some(&[40, 30][..]));\nassert_eq!(v.strip_prefix(&[50]), None);\nlet prefix : &str = \"he\";\nassert_eq!(b\"hello\".strip_prefix(prefix.as_bytes()),\n Some(b\"llo\".as_ref()));\n```\n- `strip_suffix`\n- `binary_search(&self, x: &T)` Binary searches this slice for a given element.\n- `sort_unstable(&mut self)` Sorts the slice, but might not preserve the order of equal elements.\n- `rotate_left(&mut self, mid: usize)` Rotates the slice in-place such that the first `mid` elements of the slice move to the end while the last `self.len() - mid` elements move to the front.\n- `fill(&mut self, value: T)` Fills `self` with elements by cloning `value`.\n- `clone_from_slice(&mut self, src: &[T])` Copies the elements from `src` into `self`.\n- `copy_from_slice(&mut self, src: &[T])` \n- `is_sorted(&self)` \n- `take<'a, R: OneSidedRange>(self: &mut &'a Self, range: R)` Removes the subslice corresponding to the given range\n- `get_many_mut` Returns mutable references to many indices at once.\n```rust\n#![feature(get_many_mut)]\nlet v = &mut [1, 2, 3];\nif let Ok([a, b]) = v.get_many_mut([0, 2]) {\n *a = 413;\n *b = 612;\n}\nassert_eq!(v, &[413, 2, 612]);\n```\n\n## alloc::vec::Vec\n- `fn truncate(&mut self, len: usize)` Shortens the vector, keeping the first `len` elements and dropping the rest\n\n## std::collections::VecDeque\nA double-ended queue (deque) implemented with a growable ring buffer.\nSince VecDeque is a ring buffer, its elements are not necessarily contiguous in memory. If you want to access the elements as a single slice, such as for efficient sorting, you can use make_contiguous. It rotates the VecDeque so that its elements do not wrap, and returns a mutable slice to the now-contiguous element sequence.\n\n- `swap(&mut self, i: usize, j: usize)`\n- `reserve_exact(&mut self, additional: usize)` Reserves the minimum capacity for at least `additional` more elements to be inserted in the given deque. Does nothing if the capacity is already sufficient.\n- `reserve(&mut self, additional: usize)`\n- `shrink_to_fit(&mut self)` Shrinks the capacity of the deque as much as possible.\n- `truncate(&mut self, len: usize)` Shortens the deque, keeping the first `len` elements and dropping the rest.\n```rust\nuse std::collections::VecDeque;\nlet mut buf = VecDeque::new();\nbuf.push_back(5);\nbuf.push_back(10);\nbuf.push_back(15);\nassert_eq!(buf, [5, 10, 15]);\nbuf.truncate(1);\nassert_eq!(buf, [5]);\n```\n- `iter(&self)`\n- `as_slices(&self)`\n- `slice_ranges(&self, range: R)` Given a range into the logical buffer of the deque, this function return two ranges into the physical buffer that correspond to the given range\n- `range(&self, range: R)` Creates an iterator that covers the specified range in the deque.\n```rust\nuse std::collections::VecDeque;\nlet deque: VecDeque<_> = [1, 2, 3].into();\nlet range = deque.range(2..).copied().collect::>();\nassert_eq!(range, [3]);\n// A full range covers all contents\nlet all = deque.range(..);\nassert_eq!(all.len(), 3);\n```\n- `drain(&mut self, range: R)` Removes the specified range from the deque in bulk, returning all removed elements as an iterator.\n- `clear(&mut self)`\n- `contains(&self, x: &T)` Returns `true` if the deque contains an element equal to the given value\n- `front(&self)` Provides a reference to the front element\n- `front_mut(&mut self)`\n- `back(&self)`\n- `back_mut(&mut self)`\n- `pop_front(&mut self)`\n- `pop_back(&mut self)`\n- `push_front(&mut self, value: T)`\n- `push_back(&mut self, value: T)`\n\n## [std::collections::LinkedList](https://doc.rust-lang.org/std/collections/struct.LinkedList.html)","source":"_posts/rust/rust_std/rust-std-data-structure-1.md","raw":"---\ntitle: rust std data structure (1D)\ndate: 2023-05-01 22:04:38\ntags: [rust-std]\n---\n\n## array\nA **fixed-size** array, denoted [T; N], for the element type, T, and the non-negative compile-time constant size, N.\n```rust\ntodo!()\n```\n\n## slice\nA **dynamically-sized view** into a contiguous sequence, [T].\n- `len()`: Returns the number of elements in the slice\n- `is_empty()`\n- `first()` Returns the first element of the slice, or `None` if it is empty.\n- `first_mut()` Returns a mutable **pointer** to the first element of the slice, or `None` if it is empty\n- `split_first()` Returns the first and all the rest of the elements of the slice, or `None` if it is empty.\n- `split_first_mut()` \n- `split_last()`\n- `split_last_mut()`\n- `last()`\n- `last_mut()`\n- `get(index: I)` Returns a reference to an element or subslice depending on the type of index.\n```rust\nlet v = [10, 40, 30];\nassert_eq!(Some(&40), v.get(1));\nassert_eq!(Some(&[10, 40][..]), v.get(0..2));\n```\n- `get_mut(index: I)`\n- `get_unchecked(index: I)` Returns a reference to an element or subslice, without doing bounds checking\n- `get_unchecked_mut(index: I)`\n- `as_ptr(&self) -> *const T` Returns a raw pointer to the slice's buffer\n```rust\nlet x = &[1, 2, 4];\nlet x_ptr = x.as_ptr();\nunsafe {\n for i in 0..x.len() {\n assert_eq!(x.get_unchecked(i), &*x_ptr.add(i));\n }\n}\n```\n- `as_mut_ptr(&mut self) -> *mut T` \n```rust\nlet x = &mut [1, 2, 4];\nlet x_ptr = x.as_mut_ptr();\nunsafe {\n for i in 0..x.len() {\n *x_ptr.add(i) += 2;\n }\n}\nassert_eq!(x, &[3, 4, 6]);\n```\n- `as_ptr_range(&self) -> Range<*const T>` Returns the two raw pointers spanning the slice.\n```rust\npub const fn as_ptr_range(&self) -> Range<*const T> {\n let start = self.as_ptr();\n let end = unsafe { start.add(self.len()) };\n start..end\n}\n```\n- `as_mut_ptr_range(&mut self) -> Range<*mut T>`\n- `swap(&mut self, a: usize, b: usize)` Swaps two elements in the slice.\n- `reverse(&mut self)` Reverses the order of elements in the slice, in place.\n- `windows(&self, size: usize)` Returns an iterator over all contiguous windows of length `size`. The windows overlap. If the slice is shorter than `size`, the iterator returns no values.\n```rust\nlet slice = ['r', 'u', 's', 't'];\nlet mut iter = slice.windows(2);\nassert_eq!(iter.next().unwrap(), &['r', 'u']);\nassert_eq!(iter.next().unwrap(), &['u', 's']);\nassert_eq!(iter.next().unwrap(), &['s', 't']);\nassert!(iter.next().is_none());\n```\n- `chunks(&self, chunk_size: usize)` Returns an iterator over `chunk_size` elements of the slice at a time\n```rust\nlet slice = ['l', 'o', 'r', 'e', 'm'];\nlet mut iter = slice.chunks(2);\nassert_eq!(iter.next().unwrap(), &['l', 'o']);\nassert_eq!(iter.next().unwrap(), &['r', 'e']);\nassert_eq!(iter.next().unwrap(), &['m']);\nassert!(iter.next().is_none());\n```\n- `chunks_mut()`\n- `chunks_exact(&self, chunk_size: usize)`\n```rust\nlet slice = ['l', 'o', 'r', 'e', 'm'];\nlet mut iter = slice.chunks_exact(2);\nassert_eq!(iter.next().unwrap(), &['l', 'o']);\nassert_eq!(iter.next().unwrap(), &['r', 'e']);\nassert!(iter.next().is_none());\nassert_eq!(iter.remainder(), &['m']);\n```\n- `as_chunks_unchecked(&self)` Splits the slice into a slice of `N`-element arrays, assuming that there's no remainder\n- `as_chunks(&self)` Splits the slice into a slice of `N`-element arrays, starting at the beginning of the slice, and a remainder slice with length strictly less than `N`\n- `as_rchunks(&self)` r means reverse\n- `group_by(&self, pred: F)` Returns an iterator over the slice producing non-overlapping runs of elements using the predicate to separate them. The predicate is called on two elements following themselves, it means the predicate is called on `slice[0]` and `slice[1]` then on `slice[1]` and `slice[2]` and so on\n```rust\n#![feature(slice_group_by)]\nlet slice = &[1, 1, 2, 3, 2, 3, 2, 3, 4];\nlet mut iter = slice.group_by(|a, b| a <= b);\nassert_eq!(iter.next(), Some(&[1, 1, 2, 3][..]));\nassert_eq!(iter.next(), Some(&[2, 3][..]));\nassert_eq!(iter.next(), Some(&[2, 3, 4][..]));\nassert_eq!(iter.next(), None);\n```\n- `split_at(&self, mid: usize)` Divides one slice into two at an index.\n- `split(&self, pred: F)` Returns an iterator over subslices separated by elements that match `pred`. The matched element is not contained in the subslices.\n- `splitn(&self, n: usize, pred: F)` \n- `contains(&self, x: &T)` Returns `true` if the slice contains an element with the given value.\n- `starts_with(&self, needle: &[T])` eturns `true` if `needle` is a prefix of the slice\n```rust\nlet v = [10, 40, 30];\nassert!(v.starts_with(&[10]));\nassert!(v.starts_with(&[10, 40]));\nassert!(!v.starts_with(&[50]));\n```\n- `ends_with(&self, needle: &[T])` \n- `strip_prefix` Returns a subslice with the prefix removed.\n```rust\nlet v = &[10, 40, 30];\nassert_eq!(v.strip_prefix(&[10]), Some(&[40, 30][..]));\nassert_eq!(v.strip_prefix(&[50]), None);\nlet prefix : &str = \"he\";\nassert_eq!(b\"hello\".strip_prefix(prefix.as_bytes()),\n Some(b\"llo\".as_ref()));\n```\n- `strip_suffix`\n- `binary_search(&self, x: &T)` Binary searches this slice for a given element.\n- `sort_unstable(&mut self)` Sorts the slice, but might not preserve the order of equal elements.\n- `rotate_left(&mut self, mid: usize)` Rotates the slice in-place such that the first `mid` elements of the slice move to the end while the last `self.len() - mid` elements move to the front.\n- `fill(&mut self, value: T)` Fills `self` with elements by cloning `value`.\n- `clone_from_slice(&mut self, src: &[T])` Copies the elements from `src` into `self`.\n- `copy_from_slice(&mut self, src: &[T])` \n- `is_sorted(&self)` \n- `take<'a, R: OneSidedRange>(self: &mut &'a Self, range: R)` Removes the subslice corresponding to the given range\n- `get_many_mut` Returns mutable references to many indices at once.\n```rust\n#![feature(get_many_mut)]\nlet v = &mut [1, 2, 3];\nif let Ok([a, b]) = v.get_many_mut([0, 2]) {\n *a = 413;\n *b = 612;\n}\nassert_eq!(v, &[413, 2, 612]);\n```\n\n## alloc::vec::Vec\n- `fn truncate(&mut self, len: usize)` Shortens the vector, keeping the first `len` elements and dropping the rest\n\n## std::collections::VecDeque\nA double-ended queue (deque) implemented with a growable ring buffer.\nSince VecDeque is a ring buffer, its elements are not necessarily contiguous in memory. If you want to access the elements as a single slice, such as for efficient sorting, you can use make_contiguous. It rotates the VecDeque so that its elements do not wrap, and returns a mutable slice to the now-contiguous element sequence.\n\n- `swap(&mut self, i: usize, j: usize)`\n- `reserve_exact(&mut self, additional: usize)` Reserves the minimum capacity for at least `additional` more elements to be inserted in the given deque. Does nothing if the capacity is already sufficient.\n- `reserve(&mut self, additional: usize)`\n- `shrink_to_fit(&mut self)` Shrinks the capacity of the deque as much as possible.\n- `truncate(&mut self, len: usize)` Shortens the deque, keeping the first `len` elements and dropping the rest.\n```rust\nuse std::collections::VecDeque;\nlet mut buf = VecDeque::new();\nbuf.push_back(5);\nbuf.push_back(10);\nbuf.push_back(15);\nassert_eq!(buf, [5, 10, 15]);\nbuf.truncate(1);\nassert_eq!(buf, [5]);\n```\n- `iter(&self)`\n- `as_slices(&self)`\n- `slice_ranges(&self, range: R)` Given a range into the logical buffer of the deque, this function return two ranges into the physical buffer that correspond to the given range\n- `range(&self, range: R)` Creates an iterator that covers the specified range in the deque.\n```rust\nuse std::collections::VecDeque;\nlet deque: VecDeque<_> = [1, 2, 3].into();\nlet range = deque.range(2..).copied().collect::>();\nassert_eq!(range, [3]);\n// A full range covers all contents\nlet all = deque.range(..);\nassert_eq!(all.len(), 3);\n```\n- `drain(&mut self, range: R)` Removes the specified range from the deque in bulk, returning all removed elements as an iterator.\n- `clear(&mut self)`\n- `contains(&self, x: &T)` Returns `true` if the deque contains an element equal to the given value\n- `front(&self)` Provides a reference to the front element\n- `front_mut(&mut self)`\n- `back(&self)`\n- `back_mut(&mut self)`\n- `pop_front(&mut self)`\n- `pop_back(&mut self)`\n- `push_front(&mut self, value: T)`\n- `push_back(&mut self, value: T)`\n\n## [std::collections::LinkedList](https://doc.rust-lang.org/std/collections/struct.LinkedList.html)","slug":"rust/rust_std/rust-std-data-structure-1","published":1,"updated":"2023-11-05T04:21:13.735Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8e3003oqwsj76cl15kp","content":"

array

A fixed-size array, denoted [T; N], for the element type, T, and the non-negative compile-time constant size, N.

\n
1
todo!()
\n\n

slice

A dynamically-sized view into a contiguous sequence, [T].

\n
    \n
  • len(): Returns the number of elements in the slice
    • \n
  • is_empty()
    • \n
  • first() Returns the first element of the slice, or None if it is empty.
    • \n
  • first_mut() Returns a mutable pointer to the first element of the slice, or None if it is empty
    • \n
  • split_first() Returns the first and all the rest of the elements of the slice, or None if it is empty.
    • \n
  • split_first_mut()
    • \n
  • split_last()
    • \n
  • split_last_mut()
    • \n
  • last()
    • \n
  • last_mut()
    • \n
  • get<I>(index: I) Returns a reference to an element or subslice depending on the type of index.
    1
    2
    3
    let v = [10, 40, 30];
    assert_eq!(Some(&40), v.get(1));
    assert_eq!(Some(&[10, 40][..]), v.get(0..2));
    • \n
  • get_mut<I>(index: I)
    • \n
  • get_unchecked<I>(index: I) Returns a reference to an element or subslice, without doing bounds checking
    • \n
  • get_unchecked_mut<I>(index: I)
    • \n
  • as_ptr(&self) -> *const T Returns a raw pointer to the slice’s buffer
    1
    2
    3
    4
    5
    6
    7
    let x = &[1, 2, 4];
    let x_ptr = x.as_ptr();
    unsafe {
        for i in 0..x.len() {
            assert_eq!(x.get_unchecked(i), &*x_ptr.add(i));
        }
    }
    • \n
  • as_mut_ptr(&mut self) -> *mut T 
    1
    2
    3
    4
    5
    6
    7
    8
    let x = &mut [1, 2, 4];
    let x_ptr = x.as_mut_ptr();
    unsafe {
        for i in 0..x.len() {
            *x_ptr.add(i) += 2;
        }
    }
    assert_eq!(x, &[3, 4, 6]);
    • \n
  • as_ptr_range(&self) -> Range<*const T> Returns the two raw pointers spanning the slice.
    1
    2
    3
    4
    5
    pub const fn as_ptr_range(&self) -> Range<*const T> {
        let start = self.as_ptr();
        let end = unsafe { start.add(self.len()) };
        start..end
    }
    • \n
  • as_mut_ptr_range(&mut self) -> Range<*mut T>
    • \n
  • swap(&mut self, a: usize, b: usize) Swaps two elements in the slice.
    • \n
  • reverse(&mut self) Reverses the order of elements in the slice, in place.
    • \n
  • windows(&self, size: usize) Returns an iterator over all contiguous windows of length size. The windows overlap. If the slice is shorter than size, the iterator returns no values.
    1
    2
    3
    4
    5
    6
    let slice = ['r', 'u', 's', 't'];
    let mut iter = slice.windows(2);
    assert_eq!(iter.next().unwrap(), &['r', 'u']);
    assert_eq!(iter.next().unwrap(), &['u', 's']);
    assert_eq!(iter.next().unwrap(), &['s', 't']);
    assert!(iter.next().is_none());
    • \n
  • chunks(&self, chunk_size: usize) Returns an iterator over chunk_size elements of the slice at a time
    1
    2
    3
    4
    5
    6
    let slice = ['l', 'o', 'r', 'e', 'm'];
    let mut iter = slice.chunks(2);
    assert_eq!(iter.next().unwrap(), &['l', 'o']);
    assert_eq!(iter.next().unwrap(), &['r', 'e']);
    assert_eq!(iter.next().unwrap(), &['m']);
    assert!(iter.next().is_none());
    • \n
  • chunks_mut()
    • \n
  • chunks_exact(&self, chunk_size: usize)
    1
    2
    3
    4
    5
    6
    let slice = ['l', 'o', 'r', 'e', 'm'];
    let mut iter = slice.chunks_exact(2);
    assert_eq!(iter.next().unwrap(), &['l', 'o']);
    assert_eq!(iter.next().unwrap(), &['r', 'e']);
    assert!(iter.next().is_none());
    assert_eq!(iter.remainder(), &['m']);
    • \n
  • as_chunks_unchecked<const N: usize>(&self) Splits the slice into a slice of N-element arrays, assuming that there’s no remainder
    • \n
  • as_chunks<const N: usize>(&self) Splits the slice into a slice of N-element arrays, starting at the beginning of the slice, and a remainder slice with length strictly less than N
    • \n
  • as_rchunks<const N: usize>(&self) r means reverse
    • \n
  • group_by<F>(&self, pred: F) Returns an iterator over the slice producing non-overlapping runs of elements using the predicate to separate them. The predicate is called on two elements following themselves, it means the predicate is called on slice[0] and slice[1] then on slice[1] and slice[2] and so on
    1
    2
    3
    4
    5
    6
    7
    #![feature(slice_group_by)]
    let slice = &[1, 1, 2, 3, 2, 3, 2, 3, 4];
    let mut iter = slice.group_by(|a, b| a <= b);
    assert_eq!(iter.next(), Some(&[1, 1, 2, 3][..]));
    assert_eq!(iter.next(), Some(&[2, 3][..]));
    assert_eq!(iter.next(), Some(&[2, 3, 4][..]));
    assert_eq!(iter.next(), None);
    • \n
  • split_at(&self, mid: usize) Divides one slice into two at an index.
    • \n
  • split<F>(&self, pred: F) Returns an iterator over subslices separated by elements that match pred. The matched element is not contained in the subslices.
    • \n
  • splitn<F>(&self, n: usize, pred: F)
    • \n
  • contains(&self, x: &T) Returns true if the slice contains an element with the given value.
    • \n
  • starts_with(&self, needle: &[T]) eturns true if needle is a prefix of the slice
    1
    2
    3
    4
    let v = [10, 40, 30];
    assert!(v.starts_with(&[10]));
    assert!(v.starts_with(&[10, 40]));
    assert!(!v.starts_with(&[50]));
    • \n
  • ends_with(&self, needle: &[T])
    • \n
  • strip_prefix Returns a subslice with the prefix removed.
    1
    2
    3
    4
    5
    6
    let v = &[10, 40, 30];
    assert_eq!(v.strip_prefix(&[10]), Some(&[40, 30][..]));
    assert_eq!(v.strip_prefix(&[50]), None);
    let prefix : &str = "he";
    assert_eq!(b"hello".strip_prefix(prefix.as_bytes()),
               Some(b"llo".as_ref()));
    • \n
  • strip_suffix
    • \n
  • binary_search(&self, x: &T) Binary searches this slice for a given element.
    • \n
  • sort_unstable(&mut self) Sorts the slice, but might not preserve the order of equal elements.
    • \n
  • rotate_left(&mut self, mid: usize) Rotates the slice in-place such that the first mid elements of the slice move to the end while the last self.len() - mid elements move to the front.
    • \n
  • fill(&mut self, value: T) Fills self with elements by cloning value.
    • \n
  • clone_from_slice(&mut self, src: &[T]) Copies the elements from src into self.
    • \n
  • copy_from_slice(&mut self, src: &[T])
    • \n
  • is_sorted(&self)
    • \n
  • take<'a, R: OneSidedRange<usize>>(self: &mut &'a Self, range: R) Removes the subslice corresponding to the given range
    • \n
  • get_many_mut<const N: usize> Returns mutable references to many indices at once.
    1
    2
    3
    4
    5
    6
    7
    #![feature(get_many_mut)]
    let v = &mut [1, 2, 3];
    if let Ok([a, b]) = v.get_many_mut([0, 2]) {
        *a = 413;
        *b = 612;
    }
    assert_eq!(v, &[413, 2, 612]);
    • \n
\n

alloc::vec::Vec

    \n
  • fn truncate(&mut self, len: usize) Shortens the vector, keeping the first len elements and dropping the rest
    • \n
\n

std::collections::VecDeque

A double-ended queue (deque) implemented with a growable ring buffer.
Since VecDeque is a ring buffer, its elements are not necessarily contiguous in memory. If you want to access the elements as a single slice, such as for efficient sorting, you can use make_contiguous. It rotates the VecDeque so that its elements do not wrap, and returns a mutable slice to the now-contiguous element sequence.

\n
    \n
  • swap(&mut self, i: usize, j: usize)
    • \n
  • reserve_exact(&mut self, additional: usize) Reserves the minimum capacity for at least additional more elements to be inserted in the given deque. Does nothing if the capacity is already sufficient.
    • \n
  • reserve(&mut self, additional: usize)
    • \n
  • shrink_to_fit(&mut self) Shrinks the capacity of the deque as much as possible.
    • \n
  • truncate(&mut self, len: usize) Shortens the deque, keeping the first len elements and dropping the rest.
    1
    2
    3
    4
    5
    6
    7
    8
    use std::collections::VecDeque;
    let mut buf = VecDeque::new();
    buf.push_back(5);
    buf.push_back(10);
    buf.push_back(15);
    assert_eq!(buf, [5, 10, 15]);
    buf.truncate(1);
    assert_eq!(buf, [5]);
    • \n
  • iter(&self)
    • \n
  • as_slices(&self)
    • \n
  • slice_ranges<R>(&self, range: R) Given a range into the logical buffer of the deque, this function return two ranges into the physical buffer that correspond to the given range
    • \n
  • range<R>(&self, range: R) Creates an iterator that covers the specified range in the deque.
    1
    2
    3
    4
    5
    6
    7
    use std::collections::VecDeque;
    let deque: VecDeque<_> = [1, 2, 3].into();
    let range = deque.range(2..).copied().collect::<VecDeque<_>>();
    assert_eq!(range, [3]);
    // A full range covers all contents
    let all = deque.range(..);
    assert_eq!(all.len(), 3);
    • \n
  • drain<R>(&mut self, range: R) Removes the specified range from the deque in bulk, returning all removed elements as an iterator.
    • \n
  • clear(&mut self)
    • \n
  • contains(&self, x: &T) Returns true if the deque contains an element equal to the given value
    • \n
  • front(&self) Provides a reference to the front element
    • \n
  • front_mut(&mut self)
    • \n
  • back(&self)
    • \n
  • back_mut(&mut self)
    • \n
  • pop_front(&mut self)
    • \n
  • pop_back(&mut self)
    • \n
  • push_front(&mut self, value: T)
    • \n
  • push_back(&mut self, value: T)
    • \n
\n

std::collections::LinkedList

","site":{"data":{}},"excerpt":"","more":"

array

A fixed-size array, denoted [T; N], for the element type, T, and the non-negative compile-time constant size, N.

\n
1
todo!()
\n\n

slice

A dynamically-sized view into a contiguous sequence, [T].

\n
    \n
  • len(): Returns the number of elements in the slice
    • \n
  • is_empty()
    • \n
  • first() Returns the first element of the slice, or None if it is empty.
    • \n
  • first_mut() Returns a mutable pointer to the first element of the slice, or None if it is empty
    • \n
  • split_first() Returns the first and all the rest of the elements of the slice, or None if it is empty.
    • \n
  • split_first_mut()
    • \n
  • split_last()
    • \n
  • split_last_mut()
    • \n
  • last()
    • \n
  • last_mut()
    • \n
  • get<I>(index: I) Returns a reference to an element or subslice depending on the type of index.
    1
    2
    3
    let v = [10, 40, 30];
    assert_eq!(Some(&40), v.get(1));
    assert_eq!(Some(&[10, 40][..]), v.get(0..2));
    • \n
  • get_mut<I>(index: I)
    • \n
  • get_unchecked<I>(index: I) Returns a reference to an element or subslice, without doing bounds checking
    • \n
  • get_unchecked_mut<I>(index: I)
    • \n
  • as_ptr(&self) -> *const T Returns a raw pointer to the slice’s buffer
    1
    2
    3
    4
    5
    6
    7
    let x = &[1, 2, 4];
    let x_ptr = x.as_ptr();
    unsafe {
        for i in 0..x.len() {
            assert_eq!(x.get_unchecked(i), &*x_ptr.add(i));
        }
    }
    • \n
  • as_mut_ptr(&mut self) -> *mut T 
    1
    2
    3
    4
    5
    6
    7
    8
    let x = &mut [1, 2, 4];
    let x_ptr = x.as_mut_ptr();
    unsafe {
        for i in 0..x.len() {
            *x_ptr.add(i) += 2;
        }
    }
    assert_eq!(x, &[3, 4, 6]);
    • \n
  • as_ptr_range(&self) -> Range<*const T> Returns the two raw pointers spanning the slice.
    1
    2
    3
    4
    5
    pub const fn as_ptr_range(&self) -> Range<*const T> {
        let start = self.as_ptr();
        let end = unsafe { start.add(self.len()) };
        start..end
    }
    • \n
  • as_mut_ptr_range(&mut self) -> Range<*mut T>
    • \n
  • swap(&mut self, a: usize, b: usize) Swaps two elements in the slice.
    • \n
  • reverse(&mut self) Reverses the order of elements in the slice, in place.
    • \n
  • windows(&self, size: usize) Returns an iterator over all contiguous windows of length size. The windows overlap. If the slice is shorter than size, the iterator returns no values.
    1
    2
    3
    4
    5
    6
    let slice = ['r', 'u', 's', 't'];
    let mut iter = slice.windows(2);
    assert_eq!(iter.next().unwrap(), &['r', 'u']);
    assert_eq!(iter.next().unwrap(), &['u', 's']);
    assert_eq!(iter.next().unwrap(), &['s', 't']);
    assert!(iter.next().is_none());
    • \n
  • chunks(&self, chunk_size: usize) Returns an iterator over chunk_size elements of the slice at a time
    1
    2
    3
    4
    5
    6
    let slice = ['l', 'o', 'r', 'e', 'm'];
    let mut iter = slice.chunks(2);
    assert_eq!(iter.next().unwrap(), &['l', 'o']);
    assert_eq!(iter.next().unwrap(), &['r', 'e']);
    assert_eq!(iter.next().unwrap(), &['m']);
    assert!(iter.next().is_none());
    • \n
  • chunks_mut()
    • \n
  • chunks_exact(&self, chunk_size: usize)
    1
    2
    3
    4
    5
    6
    let slice = ['l', 'o', 'r', 'e', 'm'];
    let mut iter = slice.chunks_exact(2);
    assert_eq!(iter.next().unwrap(), &['l', 'o']);
    assert_eq!(iter.next().unwrap(), &['r', 'e']);
    assert!(iter.next().is_none());
    assert_eq!(iter.remainder(), &['m']);
    • \n
  • as_chunks_unchecked<const N: usize>(&self) Splits the slice into a slice of N-element arrays, assuming that there’s no remainder
    • \n
  • as_chunks<const N: usize>(&self) Splits the slice into a slice of N-element arrays, starting at the beginning of the slice, and a remainder slice with length strictly less than N
    • \n
  • as_rchunks<const N: usize>(&self) r means reverse
    • \n
  • group_by<F>(&self, pred: F) Returns an iterator over the slice producing non-overlapping runs of elements using the predicate to separate them. The predicate is called on two elements following themselves, it means the predicate is called on slice[0] and slice[1] then on slice[1] and slice[2] and so on
    1
    2
    3
    4
    5
    6
    7
    #![feature(slice_group_by)]
    let slice = &[1, 1, 2, 3, 2, 3, 2, 3, 4];
    let mut iter = slice.group_by(|a, b| a <= b);
    assert_eq!(iter.next(), Some(&[1, 1, 2, 3][..]));
    assert_eq!(iter.next(), Some(&[2, 3][..]));
    assert_eq!(iter.next(), Some(&[2, 3, 4][..]));
    assert_eq!(iter.next(), None);
    • \n
  • split_at(&self, mid: usize) Divides one slice into two at an index.
    • \n
  • split<F>(&self, pred: F) Returns an iterator over subslices separated by elements that match pred. The matched element is not contained in the subslices.
    • \n
  • splitn<F>(&self, n: usize, pred: F)
    • \n
  • contains(&self, x: &T) Returns true if the slice contains an element with the given value.
    • \n
  • starts_with(&self, needle: &[T]) eturns true if needle is a prefix of the slice
    1
    2
    3
    4
    let v = [10, 40, 30];
    assert!(v.starts_with(&[10]));
    assert!(v.starts_with(&[10, 40]));
    assert!(!v.starts_with(&[50]));
    • \n
  • ends_with(&self, needle: &[T])
    • \n
  • strip_prefix Returns a subslice with the prefix removed.
    1
    2
    3
    4
    5
    6
    let v = &[10, 40, 30];
    assert_eq!(v.strip_prefix(&[10]), Some(&[40, 30][..]));
    assert_eq!(v.strip_prefix(&[50]), None);
    let prefix : &str = "he";
    assert_eq!(b"hello".strip_prefix(prefix.as_bytes()),
               Some(b"llo".as_ref()));
    • \n
  • strip_suffix
    • \n
  • binary_search(&self, x: &T) Binary searches this slice for a given element.
    • \n
  • sort_unstable(&mut self) Sorts the slice, but might not preserve the order of equal elements.
    • \n
  • rotate_left(&mut self, mid: usize) Rotates the slice in-place such that the first mid elements of the slice move to the end while the last self.len() - mid elements move to the front.
    • \n
  • fill(&mut self, value: T) Fills self with elements by cloning value.
    • \n
  • clone_from_slice(&mut self, src: &[T]) Copies the elements from src into self.
    • \n
  • copy_from_slice(&mut self, src: &[T])
    • \n
  • is_sorted(&self)
    • \n
  • take<'a, R: OneSidedRange<usize>>(self: &mut &'a Self, range: R) Removes the subslice corresponding to the given range
    • \n
  • get_many_mut<const N: usize> Returns mutable references to many indices at once.
    1
    2
    3
    4
    5
    6
    7
    #![feature(get_many_mut)]
    let v = &mut [1, 2, 3];
    if let Ok([a, b]) = v.get_many_mut([0, 2]) {
        *a = 413;
        *b = 612;
    }
    assert_eq!(v, &[413, 2, 612]);
    • \n
\n

alloc::vec::Vec

    \n
  • fn truncate(&mut self, len: usize) Shortens the vector, keeping the first len elements and dropping the rest
    • \n
\n

std::collections::VecDeque

A double-ended queue (deque) implemented with a growable ring buffer.
Since VecDeque is a ring buffer, its elements are not necessarily contiguous in memory. If you want to access the elements as a single slice, such as for efficient sorting, you can use make_contiguous. It rotates the VecDeque so that its elements do not wrap, and returns a mutable slice to the now-contiguous element sequence.

\n
    \n
  • swap(&mut self, i: usize, j: usize)
    • \n
  • reserve_exact(&mut self, additional: usize) Reserves the minimum capacity for at least additional more elements to be inserted in the given deque. Does nothing if the capacity is already sufficient.
    • \n
  • reserve(&mut self, additional: usize)
    • \n
  • shrink_to_fit(&mut self) Shrinks the capacity of the deque as much as possible.
    • \n
  • truncate(&mut self, len: usize) Shortens the deque, keeping the first len elements and dropping the rest.
    1
    2
    3
    4
    5
    6
    7
    8
    use std::collections::VecDeque;
    let mut buf = VecDeque::new();
    buf.push_back(5);
    buf.push_back(10);
    buf.push_back(15);
    assert_eq!(buf, [5, 10, 15]);
    buf.truncate(1);
    assert_eq!(buf, [5]);
    • \n
  • iter(&self)
    • \n
  • as_slices(&self)
    • \n
  • slice_ranges<R>(&self, range: R) Given a range into the logical buffer of the deque, this function return two ranges into the physical buffer that correspond to the given range
    • \n
  • range<R>(&self, range: R) Creates an iterator that covers the specified range in the deque.
    1
    2
    3
    4
    5
    6
    7
    use std::collections::VecDeque;
    let deque: VecDeque<_> = [1, 2, 3].into();
    let range = deque.range(2..).copied().collect::<VecDeque<_>>();
    assert_eq!(range, [3]);
    // A full range covers all contents
    let all = deque.range(..);
    assert_eq!(all.len(), 3);
    • \n
  • drain<R>(&mut self, range: R) Removes the specified range from the deque in bulk, returning all removed elements as an iterator.
    • \n
  • clear(&mut self)
    • \n
  • contains(&self, x: &T) Returns true if the deque contains an element equal to the given value
    • \n
  • front(&self) Provides a reference to the front element
    • \n
  • front_mut(&mut self)
    • \n
  • back(&self)
    • \n
  • back_mut(&mut self)
    • \n
  • pop_front(&mut self)
    • \n
  • pop_back(&mut self)
    • \n
  • push_front(&mut self, value: T)
    • \n
  • push_back(&mut self, value: T)
    • \n
\n

std::collections::LinkedList

"},{"title":"rust std smart pointer & interior mutability","date":"2023-06-03T14:04:38.000Z","_content":"# smart pointer\n## [Rc](https://doc.rust-lang.org/std/rc/struct.Rc.html)\nA single-threaded reference-counting pointer. The inherent methods of Rc are all associated functions, which means that you have to call them as e.g., Rc::get_mut(&mut value) instead of value.get_mut(). This avoids conflicts with methods of the inner type T.\n\n\n\n# internal mutibility\n## [Cell](https://doc.rust-lang.org/stable/std/cell/struct.Cell.html)\n`Cell` enables mutation inside an immutable value. In other words, it enables `interior mutability`. It never gives out mutable pointer to the inner value; A Cell can be shared by multiple references.\n### methods\n- `fn get(&self) -> T`\n- `fn set(&self, val: T)`\n- `fn swap(&self, other: &Cell)`\n- `fn replace(&self, val: T) -> T`\nReplaces the contained value with val, and returns the old contained value\n- `fn into_inner(self) -> T`\n- `const fn as_ptr(&self) -> *mut T`\n- `fn get_mut(&mut self) -> &mut T`\n- `fn from_mut(t: &mut T) -> &Cell`\n\n### traits\n```rust\nimpl !Sync for Cell // cannot be used in other threads\n```\n\n## [OnceCell](https://doc.rust-lang.org/stable/std/cell/struct.OnceCell.html)\nA cell which can be written to only once.\n### special methods\n- `fn get_or_init(&self, f: F) -> &T`\n\n## [LazyCell](https://doc.rust-lang.org/stable/std/cell/struct.LazyCell.html)\nA value which is initialized on the first access\n\n## [UnsafeCell](https://doc.rust-lang.org/stable/std/cell/struct.UnsafeCell.html#)\n`UnsafeCell` opts-out of the immutability guarantee for `&T`: a shared reference `&UnsafeCell` may point to data that is being mutated. This is called `interior mutability`.\nAll other types that allow internal mutability, such as `Cell` and `RefCell`, internally use `UnsafeCell` to wrap their data.\nNote that only the immutability guarantee for shared references is affected by `UnsafeCell`. The uniqueness guarantee for mutable references is unaffected (only one mutable reference at one time, or multiple immutable reference). \n\n### methods\n- `pub const fn get(&self) -> *mut T`\nGets a mutable pointer to the wrapped value.\n- `pub fn get_mut(&mut self) -> &mut T`\nReturns a mutable reference to the underlying data\n- `pub const fn raw_get(this: *const UnsafeCell) -> *mut T`\nGets a mutable pointer to the wrapped value. The difference from get is that this function accepts a raw pointer, which is useful to avoid the creation of temporary references. e.g. Gradual initialization of an UnsafeCell requires raw_get, as calling get would require creating a reference to uninitialized data:\n```rust\nuse std::cell::UnsafeCell;\nuse std::mem::MaybeUninit;\n\nlet m = MaybeUninit::>::uninit();\nunsafe { UnsafeCell::raw_get(m.as_ptr()).write(5); }\nlet uc = unsafe { m.assume_init() };\n\nassert_eq!(uc.into_inner(), 5);\n```\n- `fn into_inner(self) -> T`\nUnwraps the value, consuming the cell.\n\n## [SyncUnsafeCell](https://doc.rust-lang.org/stable/std/cell/struct.SyncUnsafeCell.html)\nThis is just an `UnsafeCell`, except it implements `Sync` if T implements Sync.\n\n## [std::cell::RefCell](https://doc.rust-lang.org/stable/std/cell/struct.RefCell.html)\nA mutable memory location with **dynamically** checked borrow rules\n- `fn borrow(&self) -> Ref<'_, T>`\n- `fn borrow_mut(&self) -> RefMut<'_, T>`\n- `fn as_ptr(&self) -> *mut T`\n\n# borrow\n## [std::borrow::Cow](https://doc.rust-lang.org/std/borrow/enum.Cow.html)\n","source":"_posts/rust/rust_std/rust-smart-pointer-and-internal-mutibility.md","raw":"---\ntitle: rust std smart pointer & interior mutability\ndate: 2023-06-03 22:04:38\ntags: [rust-std]\n---\n# smart pointer\n## [Rc](https://doc.rust-lang.org/std/rc/struct.Rc.html)\nA single-threaded reference-counting pointer. The inherent methods of Rc are all associated functions, which means that you have to call them as e.g., Rc::get_mut(&mut value) instead of value.get_mut(). This avoids conflicts with methods of the inner type T.\n\n\n\n# internal mutibility\n## [Cell](https://doc.rust-lang.org/stable/std/cell/struct.Cell.html)\n`Cell` enables mutation inside an immutable value. In other words, it enables `interior mutability`. It never gives out mutable pointer to the inner value; A Cell can be shared by multiple references.\n### methods\n- `fn get(&self) -> T`\n- `fn set(&self, val: T)`\n- `fn swap(&self, other: &Cell)`\n- `fn replace(&self, val: T) -> T`\nReplaces the contained value with val, and returns the old contained value\n- `fn into_inner(self) -> T`\n- `const fn as_ptr(&self) -> *mut T`\n- `fn get_mut(&mut self) -> &mut T`\n- `fn from_mut(t: &mut T) -> &Cell`\n\n### traits\n```rust\nimpl !Sync for Cell // cannot be used in other threads\n```\n\n## [OnceCell](https://doc.rust-lang.org/stable/std/cell/struct.OnceCell.html)\nA cell which can be written to only once.\n### special methods\n- `fn get_or_init(&self, f: F) -> &T`\n\n## [LazyCell](https://doc.rust-lang.org/stable/std/cell/struct.LazyCell.html)\nA value which is initialized on the first access\n\n## [UnsafeCell](https://doc.rust-lang.org/stable/std/cell/struct.UnsafeCell.html#)\n`UnsafeCell` opts-out of the immutability guarantee for `&T`: a shared reference `&UnsafeCell` may point to data that is being mutated. This is called `interior mutability`.\nAll other types that allow internal mutability, such as `Cell` and `RefCell`, internally use `UnsafeCell` to wrap their data.\nNote that only the immutability guarantee for shared references is affected by `UnsafeCell`. The uniqueness guarantee for mutable references is unaffected (only one mutable reference at one time, or multiple immutable reference). \n\n### methods\n- `pub const fn get(&self) -> *mut T`\nGets a mutable pointer to the wrapped value.\n- `pub fn get_mut(&mut self) -> &mut T`\nReturns a mutable reference to the underlying data\n- `pub const fn raw_get(this: *const UnsafeCell) -> *mut T`\nGets a mutable pointer to the wrapped value. The difference from get is that this function accepts a raw pointer, which is useful to avoid the creation of temporary references. e.g. Gradual initialization of an UnsafeCell requires raw_get, as calling get would require creating a reference to uninitialized data:\n```rust\nuse std::cell::UnsafeCell;\nuse std::mem::MaybeUninit;\n\nlet m = MaybeUninit::>::uninit();\nunsafe { UnsafeCell::raw_get(m.as_ptr()).write(5); }\nlet uc = unsafe { m.assume_init() };\n\nassert_eq!(uc.into_inner(), 5);\n```\n- `fn into_inner(self) -> T`\nUnwraps the value, consuming the cell.\n\n## [SyncUnsafeCell](https://doc.rust-lang.org/stable/std/cell/struct.SyncUnsafeCell.html)\nThis is just an `UnsafeCell`, except it implements `Sync` if T implements Sync.\n\n## [std::cell::RefCell](https://doc.rust-lang.org/stable/std/cell/struct.RefCell.html)\nA mutable memory location with **dynamically** checked borrow rules\n- `fn borrow(&self) -> Ref<'_, T>`\n- `fn borrow_mut(&self) -> RefMut<'_, T>`\n- `fn as_ptr(&self) -> *mut T`\n\n# borrow\n## [std::borrow::Cow](https://doc.rust-lang.org/std/borrow/enum.Cow.html)\n","slug":"rust/rust_std/rust-smart-pointer-and-internal-mutibility","published":1,"updated":"2023-11-05T04:21:13.735Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8e3003rqwsje2592e0r","content":"

smart pointer

Rc

A single-threaded reference-counting pointer. The inherent methods of Rc are all associated functions, which means that you have to call them as e.g., Rc::get_mut(&mut value) instead of value.get_mut(). This avoids conflicts with methods of the inner type T.

\n

internal mutibility

Cell

Cell<T> enables mutation inside an immutable value. In other words, it enables interior mutability. It never gives out mutable pointer to the inner value; A Cell can be shared by multiple references.

\n

methods

    \n
  • fn get(&self) -> T
    • \n
  • fn set(&self, val: T)
    • \n
  • fn swap(&self, other: &Cell<T>)
    • \n
  • fn replace(&self, val: T) -> T
    Replaces the contained value with val, and returns the old contained value
    • \n
  • fn into_inner(self) -> T
    • \n
  • const fn as_ptr(&self) -> *mut T
    • \n
  • fn get_mut(&mut self) -> &mut T
    • \n
  • fn from_mut(t: &mut T) -> &Cell<T>
    • \n
\n

traits

1
impl<T> !Sync for Cell<T>  // cannot be used in other threads
\n\n

OnceCell

A cell which can be written to only once.

\n

special methods

    \n
  • fn get_or_init<F>(&self, f: F) -> &T
    • \n
\n

LazyCell

A value which is initialized on the first access

\n

UnsafeCell

UnsafeCell<T> opts-out of the immutability guarantee for &T: a shared reference &UnsafeCell<T> may point to data that is being mutated. This is called interior mutability.
All other types that allow internal mutability, such as Cell<T> and RefCell<T>, internally use UnsafeCell to wrap their data.
Note that only the immutability guarantee for shared references is affected by UnsafeCell. The uniqueness guarantee for mutable references is unaffected (only one mutable reference at one time, or multiple immutable reference).

\n

methods

    \n
  • pub const fn get(&self) -> *mut T
    Gets a mutable pointer to the wrapped value.
    • \n
  • pub fn get_mut(&mut self) -> &mut T
    Returns a mutable reference to the underlying data
    • \n
  • pub const fn raw_get(this: *const UnsafeCell<T>) -> *mut T
    Gets a mutable pointer to the wrapped value. The difference from get is that this function accepts a raw pointer, which is useful to avoid the creation of temporary references. e.g. Gradual initialization of an UnsafeCell requires raw_get, as calling get would require creating a reference to uninitialized data:
    1
    2
    3
    4
    5
    6
    7
    8
    use std::cell::UnsafeCell;
    use std::mem::MaybeUninit;

    let m = MaybeUninit::<UnsafeCell<i32>>::uninit();
    unsafe { UnsafeCell::raw_get(m.as_ptr()).write(5); }
    let uc = unsafe { m.assume_init() };

    assert_eq!(uc.into_inner(), 5);
    • \n
  • fn into_inner(self) -> T
    Unwraps the value, consuming the cell.
    • \n
\n

SyncUnsafeCell

This is just an UnsafeCell, except it implements Sync if T implements Sync.

\n

std::cell::RefCell

A mutable memory location with dynamically checked borrow rules

\n
    \n
  • fn borrow(&self) -> Ref<'_, T>
    • \n
  • fn borrow_mut(&self) -> RefMut<'_, T>
    • \n
  • fn as_ptr(&self) -> *mut T
    • \n
\n

borrow

std::borrow::Cow

","site":{"data":{}},"excerpt":"","more":"

smart pointer

Rc

A single-threaded reference-counting pointer. The inherent methods of Rc are all associated functions, which means that you have to call them as e.g., Rc::get_mut(&mut value) instead of value.get_mut(). This avoids conflicts with methods of the inner type T.

\n

internal mutibility

Cell

Cell<T> enables mutation inside an immutable value. In other words, it enables interior mutability. It never gives out mutable pointer to the inner value; A Cell can be shared by multiple references.

\n

methods

    \n
  • fn get(&self) -> T
    • \n
  • fn set(&self, val: T)
    • \n
  • fn swap(&self, other: &Cell<T>)
    • \n
  • fn replace(&self, val: T) -> T
    Replaces the contained value with val, and returns the old contained value
    • \n
  • fn into_inner(self) -> T
    • \n
  • const fn as_ptr(&self) -> *mut T
    • \n
  • fn get_mut(&mut self) -> &mut T
    • \n
  • fn from_mut(t: &mut T) -> &Cell<T>
    • \n
\n

traits

1
impl<T> !Sync for Cell<T>  // cannot be used in other threads
\n\n

OnceCell

A cell which can be written to only once.

\n

special methods

    \n
  • fn get_or_init<F>(&self, f: F) -> &T
    • \n
\n

LazyCell

A value which is initialized on the first access

\n

UnsafeCell

UnsafeCell<T> opts-out of the immutability guarantee for &T: a shared reference &UnsafeCell<T> may point to data that is being mutated. This is called interior mutability.
All other types that allow internal mutability, such as Cell<T> and RefCell<T>, internally use UnsafeCell to wrap their data.
Note that only the immutability guarantee for shared references is affected by UnsafeCell. The uniqueness guarantee for mutable references is unaffected (only one mutable reference at one time, or multiple immutable reference).

\n

methods

    \n
  • pub const fn get(&self) -> *mut T
    Gets a mutable pointer to the wrapped value.
    • \n
  • pub fn get_mut(&mut self) -> &mut T
    Returns a mutable reference to the underlying data
    • \n
  • pub const fn raw_get(this: *const UnsafeCell<T>) -> *mut T
    Gets a mutable pointer to the wrapped value. The difference from get is that this function accepts a raw pointer, which is useful to avoid the creation of temporary references. e.g. Gradual initialization of an UnsafeCell requires raw_get, as calling get would require creating a reference to uninitialized data:
    1
    2
    3
    4
    5
    6
    7
    8
    use std::cell::UnsafeCell;
    use std::mem::MaybeUninit;

    let m = MaybeUninit::<UnsafeCell<i32>>::uninit();
    unsafe { UnsafeCell::raw_get(m.as_ptr()).write(5); }
    let uc = unsafe { m.assume_init() };

    assert_eq!(uc.into_inner(), 5);
    • \n
  • fn into_inner(self) -> T
    Unwraps the value, consuming the cell.
    • \n
\n

SyncUnsafeCell

This is just an UnsafeCell, except it implements Sync if T implements Sync.

\n

std::cell::RefCell

A mutable memory location with dynamically checked borrow rules

\n
    \n
  • fn borrow(&self) -> Ref<'_, T>
    • \n
  • fn borrow_mut(&self) -> RefMut<'_, T>
    • \n
  • fn as_ptr(&self) -> *mut T
    • \n
\n

borrow

std::borrow::Cow

"},{"title":"rust std data structure (2D)","date":"2023-05-02T14:04:38.000Z","_content":"\n## collections\n### BTreeMap\n- `clear(&mut self)` Clears the map, removing all elements.\n- `get(&self, key: &Q)` Returns a reference to the value corresponding to the key.\n- `get_key_value(&self, k: &Q)`\n- `first_key_value(&self)` eturns the first key-value pair in the map.\n- `first_entry(&mut self)` Returns the first entry in the map for in-place manipulation\n- `pop_first(&mut self)` \n- `last_key_value(&self)`\n- `last_entry(&mut self)`\n- `pop_last(&mut self)`\n- `contains_key(&self, key: &Q)`\n- `get_mut(&mut self, key: &Q)` Returns a mutable reference to the value corresponding to the key\n- `insert(&mut self, key: K, value: V)`\n- `try_insert(&mut self, key: K, value: V)` If the map already had this key present, nothing is updated, and an error containing the occupied entry and the value is returned.\n- `remove(&mut self, key: &Q)`\n- `remove_entry(&mut self, key: &Q)`\n- `retain(&mut self, mut f: F)` Retains only the elements specified by the predicate.\n```rust\nuse std::collections::BTreeMap;\nlet mut map: BTreeMap = (0..8).map(|x| (x, x*10)).collect();\n// Keep only the elements with even-numbered keys.\nmap.retain(|&k, _| k % 2 == 0);\nassert!(map.into_iter().eq(vec![(0, 0), (2, 20), (4, 40), (6, 60)]));\n```\n- `append(&mut self, other: &mut Self)` Moves all elements from `other` into `self`, leaving `other` empty.\n- `range(&self, range: R) -> Range<'_, K, V>` Constructs a double-ended iterator over a sub-range of elements in the map.\n```rust\nuse std::collections::BTreeMap;\nuse std::ops::Bound::Included;\nlet mut map = BTreeMap::new();\nmap.insert(3, \"a\");\nmap.insert(5, \"b\");\nmap.insert(8, \"c\");\nfor (&key, &value) in map.range((Included(&4), Included(&8))) {\n println!(\"{key}: {value}\");\n}\nassert_eq!(Some((&5, &\"b\")), map.range(4..).next());\n```\n- `range_mut(&mut self, range: R) -> RangeMut<'_, K, V>` \n- `entry(&mut self, key: K)` Gets the given key's corresponding entry in the map for in-place manipulation.\n```rust\nuse std::collections::BTreeMap;\nlet mut count: BTreeMap<&str, usize> = BTreeMap::new();\n// count the number of occurrences of letters in the vec\nfor x in [\"a\", \"b\", \"a\", \"c\", \"a\", \"b\"] {\n count.entry(x).and_modify(|curr| *curr += 1).or_insert(1);\n}\nassert_eq!(count[\"a\"], 3);\nassert_eq!(count[\"b\"], 2);\nassert_eq!(count[\"c\"], 1);\n```\n- `split_off(&mut self, key: &Q)` Splits the collection into two at the given key. Returns everything after the given key,\n- `drain_filter(&mut self, pred: F)` Creates an iterator that visits all elements (key-value pairs) in ascending key order and uses a closure to determine if an element should be removed. If the closure returns `true`, the element is removed from the map and yielded. If the closure returns `false`, or panics, the element remains in the map and will not be yielded\n- `into_keys(self)` Creates a consuming iterator visiting all the keys, in sorted order. The map cannot be used after calling this\n- `into_values(self)`\n","source":"_posts/rust/rust_std/rust-std-data-structure-2.md","raw":"---\ntitle: rust std data structure (2D)\ndate: 2023-05-02 22:04:38\ntags: [rust-std]\n---\n\n## collections\n### BTreeMap\n- `clear(&mut self)` Clears the map, removing all elements.\n- `get(&self, key: &Q)` Returns a reference to the value corresponding to the key.\n- `get_key_value(&self, k: &Q)`\n- `first_key_value(&self)` eturns the first key-value pair in the map.\n- `first_entry(&mut self)` Returns the first entry in the map for in-place manipulation\n- `pop_first(&mut self)` \n- `last_key_value(&self)`\n- `last_entry(&mut self)`\n- `pop_last(&mut self)`\n- `contains_key(&self, key: &Q)`\n- `get_mut(&mut self, key: &Q)` Returns a mutable reference to the value corresponding to the key\n- `insert(&mut self, key: K, value: V)`\n- `try_insert(&mut self, key: K, value: V)` If the map already had this key present, nothing is updated, and an error containing the occupied entry and the value is returned.\n- `remove(&mut self, key: &Q)`\n- `remove_entry(&mut self, key: &Q)`\n- `retain(&mut self, mut f: F)` Retains only the elements specified by the predicate.\n```rust\nuse std::collections::BTreeMap;\nlet mut map: BTreeMap = (0..8).map(|x| (x, x*10)).collect();\n// Keep only the elements with even-numbered keys.\nmap.retain(|&k, _| k % 2 == 0);\nassert!(map.into_iter().eq(vec![(0, 0), (2, 20), (4, 40), (6, 60)]));\n```\n- `append(&mut self, other: &mut Self)` Moves all elements from `other` into `self`, leaving `other` empty.\n- `range(&self, range: R) -> Range<'_, K, V>` Constructs a double-ended iterator over a sub-range of elements in the map.\n```rust\nuse std::collections::BTreeMap;\nuse std::ops::Bound::Included;\nlet mut map = BTreeMap::new();\nmap.insert(3, \"a\");\nmap.insert(5, \"b\");\nmap.insert(8, \"c\");\nfor (&key, &value) in map.range((Included(&4), Included(&8))) {\n println!(\"{key}: {value}\");\n}\nassert_eq!(Some((&5, &\"b\")), map.range(4..).next());\n```\n- `range_mut(&mut self, range: R) -> RangeMut<'_, K, V>` \n- `entry(&mut self, key: K)` Gets the given key's corresponding entry in the map for in-place manipulation.\n```rust\nuse std::collections::BTreeMap;\nlet mut count: BTreeMap<&str, usize> = BTreeMap::new();\n// count the number of occurrences of letters in the vec\nfor x in [\"a\", \"b\", \"a\", \"c\", \"a\", \"b\"] {\n count.entry(x).and_modify(|curr| *curr += 1).or_insert(1);\n}\nassert_eq!(count[\"a\"], 3);\nassert_eq!(count[\"b\"], 2);\nassert_eq!(count[\"c\"], 1);\n```\n- `split_off(&mut self, key: &Q)` Splits the collection into two at the given key. Returns everything after the given key,\n- `drain_filter(&mut self, pred: F)` Creates an iterator that visits all elements (key-value pairs) in ascending key order and uses a closure to determine if an element should be removed. If the closure returns `true`, the element is removed from the map and yielded. If the closure returns `false`, or panics, the element remains in the map and will not be yielded\n- `into_keys(self)` Creates a consuming iterator visiting all the keys, in sorted order. The map cannot be used after calling this\n- `into_values(self)`\n","slug":"rust/rust_std/rust-std-data-structure-2","published":1,"updated":"2023-11-05T04:21:13.735Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8e3003tqwsj3w7e71hn","content":"

collections

BTreeMap

    \n
  • clear(&mut self) Clears the map, removing all elements.
    • \n
  • get(&self, key: &Q) Returns a reference to the value corresponding to the key.
    • \n
  • get_key_value(&self, k: &Q)
    • \n
  • first_key_value(&self) eturns the first key-value pair in the map.
    • \n
  • first_entry(&mut self) Returns the first entry in the map for in-place manipulation
    • \n
  • pop_first(&mut self)
    • \n
  • last_key_value(&self)
    • \n
  • last_entry(&mut self)
    • \n
  • pop_last(&mut self)
    • \n
  • contains_key(&self, key: &Q)
    • \n
  • get_mut(&mut self, key: &Q) Returns a mutable reference to the value corresponding to the key
    • \n
  • insert(&mut self, key: K, value: V)
    • \n
  • try_insert(&mut self, key: K, value: V) If the map already had this key present, nothing is updated, and an error containing the occupied entry and the value is returned.
    • \n
  • remove(&mut self, key: &Q)
    • \n
  • remove_entry(&mut self, key: &Q)
    • \n
  • retain<F>(&mut self, mut f: F) Retains only the elements specified by the predicate.
    1
    2
    3
    4
    5
    use std::collections::BTreeMap;
    let mut map: BTreeMap<i32, i32> = (0..8).map(|x| (x, x*10)).collect();
    // Keep only the elements with even-numbered keys.
    map.retain(|&k, _| k % 2 == 0);
    assert!(map.into_iter().eq(vec![(0, 0), (2, 20), (4, 40), (6, 60)]));
    • \n
  • append(&mut self, other: &mut Self) Moves all elements from other into self, leaving other empty.
    • \n
  • range<T: ?Sized, R>(&self, range: R) -> Range<'_, K, V> Constructs a double-ended iterator over a sub-range of elements in the map.
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    use std::collections::BTreeMap;
    use std::ops::Bound::Included;
    let mut map = BTreeMap::new();
    map.insert(3, "a");
    map.insert(5, "b");
    map.insert(8, "c");
    for (&key, &value) in map.range((Included(&4), Included(&8))) {
        println!("{key}: {value}");
    }
    assert_eq!(Some((&5, &"b")), map.range(4..).next());
    • \n
  • range_mut<T: ?Sized, R>(&mut self, range: R) -> RangeMut<'_, K, V>
    • \n
  • entry(&mut self, key: K) Gets the given key’s corresponding entry in the map for in-place manipulation.
    1
    2
    3
    4
    5
    6
    7
    8
    9
    use std::collections::BTreeMap;
    let mut count: BTreeMap<&str, usize> = BTreeMap::new();
    // count the number of occurrences of letters in the vec
    for x in ["a", "b", "a", "c", "a", "b"] {
        count.entry(x).and_modify(|curr| *curr += 1).or_insert(1);
    }
    assert_eq!(count["a"], 3);
    assert_eq!(count["b"], 2);
    assert_eq!(count["c"], 1);
    • \n
  • split_off<Q: ?Sized + Ord>(&mut self, key: &Q) Splits the collection into two at the given key. Returns everything after the given key,
    • \n
  • drain_filter<F>(&mut self, pred: F) Creates an iterator that visits all elements (key-value pairs) in ascending key order and uses a closure to determine if an element should be removed. If the closure returns true, the element is removed from the map and yielded. If the closure returns false, or panics, the element remains in the map and will not be yielded
    • \n
  • into_keys(self) Creates a consuming iterator visiting all the keys, in sorted order. The map cannot be used after calling this
    • \n
  • into_values(self)
    • \n
\n","site":{"data":{}},"excerpt":"","more":"

collections

BTreeMap

    \n
  • clear(&mut self) Clears the map, removing all elements.
    • \n
  • get(&self, key: &Q) Returns a reference to the value corresponding to the key.
    • \n
  • get_key_value(&self, k: &Q)
    • \n
  • first_key_value(&self) eturns the first key-value pair in the map.
    • \n
  • first_entry(&mut self) Returns the first entry in the map for in-place manipulation
    • \n
  • pop_first(&mut self)
    • \n
  • last_key_value(&self)
    • \n
  • last_entry(&mut self)
    • \n
  • pop_last(&mut self)
    • \n
  • contains_key(&self, key: &Q)
    • \n
  • get_mut(&mut self, key: &Q) Returns a mutable reference to the value corresponding to the key
    • \n
  • insert(&mut self, key: K, value: V)
    • \n
  • try_insert(&mut self, key: K, value: V) If the map already had this key present, nothing is updated, and an error containing the occupied entry and the value is returned.
    • \n
  • remove(&mut self, key: &Q)
    • \n
  • remove_entry(&mut self, key: &Q)
    • \n
  • retain<F>(&mut self, mut f: F) Retains only the elements specified by the predicate.
    1
    2
    3
    4
    5
    use std::collections::BTreeMap;
    let mut map: BTreeMap<i32, i32> = (0..8).map(|x| (x, x*10)).collect();
    // Keep only the elements with even-numbered keys.
    map.retain(|&k, _| k % 2 == 0);
    assert!(map.into_iter().eq(vec![(0, 0), (2, 20), (4, 40), (6, 60)]));
    • \n
  • append(&mut self, other: &mut Self) Moves all elements from other into self, leaving other empty.
    • \n
  • range<T: ?Sized, R>(&self, range: R) -> Range<'_, K, V> Constructs a double-ended iterator over a sub-range of elements in the map.
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    use std::collections::BTreeMap;
    use std::ops::Bound::Included;
    let mut map = BTreeMap::new();
    map.insert(3, "a");
    map.insert(5, "b");
    map.insert(8, "c");
    for (&key, &value) in map.range((Included(&4), Included(&8))) {
        println!("{key}: {value}");
    }
    assert_eq!(Some((&5, &"b")), map.range(4..).next());
    • \n
  • range_mut<T: ?Sized, R>(&mut self, range: R) -> RangeMut<'_, K, V>
    • \n
  • entry(&mut self, key: K) Gets the given key’s corresponding entry in the map for in-place manipulation.
    1
    2
    3
    4
    5
    6
    7
    8
    9
    use std::collections::BTreeMap;
    let mut count: BTreeMap<&str, usize> = BTreeMap::new();
    // count the number of occurrences of letters in the vec
    for x in ["a", "b", "a", "c", "a", "b"] {
        count.entry(x).and_modify(|curr| *curr += 1).or_insert(1);
    }
    assert_eq!(count["a"], 3);
    assert_eq!(count["b"], 2);
    assert_eq!(count["c"], 1);
    • \n
  • split_off<Q: ?Sized + Ord>(&mut self, key: &Q) Splits the collection into two at the given key. Returns everything after the given key,
    • \n
  • drain_filter<F>(&mut self, pred: F) Creates an iterator that visits all elements (key-value pairs) in ascending key order and uses a closure to determine if an element should be removed. If the closure returns true, the element is removed from the map and yielded. If the closure returns false, or panics, the element remains in the map and will not be yielded
    • \n
  • into_keys(self) Creates a consuming iterator visiting all the keys, in sorted order. The map cannot be used after calling this
    • \n
  • into_values(self)
    • \n
\n"},{"title":"rust std sync","date":"2023-06-08T14:04:38.000Z","_content":"\n## [lock free & wait free](https://en.wikipedia.org/wiki/Non-blocking_algorithm)\n\"Lock-free\" and \"wait-free\" are two different approaches to designing concurrent algorithms and data structures. Both aim to provide efficient and non-blocking synchronization in concurrent environments.\n- **lock-free** A lock-free algorithm or data structure guarantees progress for at least one thread, regardless of the behavior or state of other threads. In a lock-free design, threads can independently perform their operations without being blocked by other threads. If one thread gets delayed or suspended, other threads can continue to make progress. Lock-free algorithms typically use low-level synchronization primitives such as atomic operations to ensure progress and prevent data races.\n- **wait-free** A wait-free algorithm or data structure guarantees progress for every thread, regardless of the behavior or state of other threads. In a wait-free design, every thread executing an operation completes its operation within a finite number of steps, without being delayed by other threads. Wait-free algorithms are more stringent in their requirements compared to lock-free algorithms and often require more complex synchronization mechanisms.\n\nIt's important to note that both lock-free and wait-free designs aim to avoid traditional locks or blocking synchronization mechanisms (such as mutexes or condition variables) that can lead to contention and thread blocking. Instead, they rely on techniques like atomic operations, compare-and-swap (CAS), or memory fences to ensure progress and prevent data races in concurrent execution.\n\n## [atomic](https://doc.rust-lang.org/stable/std/sync/atomic/index.html)\nRust atomics currently follow the same rules as [C++20 atomics](https://en.cppreference.com/w/cpp/atomic), specifically `atomic_ref`. Basically, creating a shared reference to one of the Rust atomic types corresponds to creating an `atomic_ref` in C++; the atomic_ref is destroyed when the lifetime of the shared reference ends. \nEach method takes an `Ordering` which represents the strength of the memory barrier for that operation. These orderings are the same as the [C++20 atomic orderings](https://en.cppreference.com/w/cpp/atomic/memory_order). For more information see the [nomicon](https://doc.rust-lang.org/stable/nomicon/atomics.html)\nAtomic variables are safe to share between threads (they implement Sync) but they do not themselves provide the mechanism for sharing and follow the threading model of Rust. The most common way to share an atomic variable is to put it into an Arc (an atomically-reference-counted shared pointer).\n\n### Compiler Reordering\nCompilers may change the actual order of events, or make events never occur! If we write something like\n```rust\nx = 1;\ny = 3;\nx = 2;\n```\nThe compiler may conclude that it would be best if your program did:\n```rust\nx = 2;\ny = 3;\n```\nThis has inverted the order of events and completely eliminated one event. But if our program is multi-threaded, we may have been relying on x to actually be assigned to 1 before y was assigned. \n\n### Hardware Reordering\nhere is indeed a global shared memory space somewhere in your hardware, but from the perspective of each CPU core it is so very far away and so very slow. Each CPU would rather work with its local cache of the data and only go through all the anguish of talking to shared memory only when it doesn't actually have that memory in cache. The end result is that the hardware doesn't guarantee that events that occur in some order on one thread, occur in the same order on another thread. To guarantee this, we must issue special instructions to the CPU telling it to be a bit less smart.\nFor instance, say we convince the compiler to emit this logic:\n```\ninitial state: x = 0, y = 1\n\nTHREAD 1 THREAD2\ny = 3; if x == 1 {\nx = 1; y *= 2;\n }\n```\nIdeally this program has 2 possible final states:\n- y = 3: (thread 2 did the check before thread 1 completed)\n- y = 6: (thread 2 did the check after thread 1 completed)\nHowever there's a third potential state that the hardware enables:\n- y = 2: (thread 2 saw x = 1, but not y = 3, and then overwrote y = 3)\nIt's worth noting that different kinds of CPU provide different guarantees. It is common to separate hardware into two categories: strongly-ordered and weakly-ordered. Most notably x86/64 provides strong ordering guarantees, while ARM provides weak ordering guarantees. \n\n### Data Accesses\nAtomic accesses are how we tell the hardware and compiler that our program is multi-threaded. Each atomic access can be marked with an ordering that specifies what kind of relationship it establishes with other accesses. For the compiler, this largely revolves around re-ordering of instructions. For the hardware, this largely revolves around how writes are propagated to other threads. The set of orderings Rust exposes are:\n- Sequentially Consistent (SeqCst)\n- Release\n- Acquire\n- Relaxed\n\n### Sequentially Consistent\nSequentially Consistent is the most powerful of all, implying the restrictions of all other orderings. Intuitively, a sequentially consistent operation cannot be reordered: all accesses on one thread that happen before and after a SeqCst access stay before and after it.\n\n### Acquire-Release\nAcquire and Release are largely intended to be paired. they're perfectly suited for acquiring and releasing locks. \nIntuitively, an acquire access ensures that every access after it stays after it. However operations that occur before an acquire are free to be reordered to occur after it. Similarly, a release access ensures that every access before it stays before it. However operations that occur after a release are free to be reordered to occur before it.\n```rust\nuse std::sync::Arc;\nuse std::sync::atomic::{AtomicBool, Ordering};\nuse std::thread;\n\nfn main() {\n let lock = Arc::new(AtomicBool::new(false)); // value answers \"am I locked?\"\n\n // ... distribute lock to threads somehow ...\n\n // Try to acquire the lock by setting it to true\n while lock.compare_and_swap(false, true, Ordering::Acquire) { }\n // broke out of the loop, so we successfully acquired the lock!\n\n // ... scary data accesses ...\n\n // ok we're done, release the lock\n lock.store(false, Ordering::Release);\n}\n```\n### Relaxed\nRelaxed accesses are the absolute weakest. They can be freely re-ordered and provide no happens-before relationship. Still, relaxed operations are still atomic. That is, they don't count as data accesses and any read-modify-write operations done to them occur atomically. For instance, incrementing a counter can be safely done by multiple threads using a relaxed `fetch_add` if you're not using the counter to synchronize any other accesses.\n\n## an example (spinlock)\n```rust\nuse std::sync::Arc;\nuse std::sync::atomic::{AtomicUsize, Ordering};\nuse std::{hint, thread};\n\nfn main() {\n let spinlock = Arc::new(AtomicUsize::new(1));\n\n let spinlock_clone = Arc::clone(&spinlock);\n let thread = thread::spawn(move|| {\n spinlock_clone.store(0, Ordering::SeqCst);\n });\n\n // Wait for the other thread to release the lock\n while spinlock.load(Ordering::SeqCst) != 0 {\n hint::spin_loop();\n }\n\n if let Err(panic) = thread.join() {\n println!(\"Thread had an error: {panic:?}\");\n }\n}\n```\n\n## usual structs\n1. [AtomicBool](https://doc.rust-lang.org/stable/std/sync/atomic/struct.AtomicBool.html)\n### methods\n- `fn get_mut(&mut self) -> &mut bool`\n- `fn into_inner(self) -> bool`\n- `fn load(&self, order: Ordering) -> bool`\n- `fn store(&self, val: bool, order: Ordering)`\n- `fn compare_exchange(&self, current: bool,new: bool,success: Ordering,failure: Ordering) -> Result`\nStores a value into the bool if the current value is the same as the current value.\ncompare_exchange takes two Ordering arguments to describe the memory ordering of this operation. success describes the required ordering for the read-modify-write operation that takes place if the comparison with current succeeds. failure describes the required ordering for the load operation that takes place when the comparison fails. \n- `fn fetch_and(&self, val: bool, order: Ordering) -> bool`\nLogical “and” with a boolean value.\nPerforms a logical “and” operation on the current value and the argument val, and sets the new value to the result.\n- `const fn as_ptr(&self) -> *mut bool`\nReturns a mutable pointer to the underlying bool.\nDoing non-atomic reads and writes on the resulting integer can be a data race. This method is mostly useful for FFI, where the function signature may use *mut bool instead of &AtomicBool.\n\n2. [AtomicUsize](https://doc.rust-lang.org/stable/std/sync/atomic/struct.AtomicUsize.html)\n2. [AtomicPtr](https://doc.rust-lang.org/stable/std/sync/atomic/struct.AtomicPtr.html)\nA raw pointer type which can be safely shared between threads.\nThis type has the same in-memory representation as a *mut T\n\n\n## Higher-level synchronization objects\nMost of the low-level synchronization primitives are quite error-prone and inconvenient to use, which is why the standard library also exposes some higher-level synchronization objects.\n- **Arc**: Atomically Reference-Counted pointer, which can be used in multithreaded environments to prolong the lifetime of some data until all the threads have finished using it.\n- **Barrier**: Ensures multiple threads will wait for each other to reach a point in the program, before continuing execution all together.\n- **Condvar**: Condition Variable, providing the ability to block a thread while waiting for an event to occur.\n- **mpsc**: Multi-producer, single-consumer queues, used for message-based communication. Can provide a lightweight inter-thread synchronisation mechanism, at the cost of some extra memory.\n- **Mutex**: Mutual Exclusion mechanism, which ensures that at most one thread at a time is able to access some data.\n- **Once**: Used for a thread-safe, one-time global initialization routine\n- **OnceLock**: Used for thread-safe, one-time initialization of a global variable.\n- **RwLock**: Provides a mutual exclusion mechanism which allows multiple readers at the same time, while allowing only one writer at a time. In some cases, this can be more efficient than a mutex.\n\n## mpsc\nThis module provides message-based communication over channels, concretely defined among three types:\n- Sender\n- SyncSender\n- Receiver","source":"_posts/rust/rust_std/rust-std-sync.md","raw":"---\ntitle: rust std sync\ndate: 2023-06-08 22:04:38\ntags: [rust-std]\n---\n\n## [lock free & wait free](https://en.wikipedia.org/wiki/Non-blocking_algorithm)\n\"Lock-free\" and \"wait-free\" are two different approaches to designing concurrent algorithms and data structures. Both aim to provide efficient and non-blocking synchronization in concurrent environments.\n- **lock-free** A lock-free algorithm or data structure guarantees progress for at least one thread, regardless of the behavior or state of other threads. In a lock-free design, threads can independently perform their operations without being blocked by other threads. If one thread gets delayed or suspended, other threads can continue to make progress. Lock-free algorithms typically use low-level synchronization primitives such as atomic operations to ensure progress and prevent data races.\n- **wait-free** A wait-free algorithm or data structure guarantees progress for every thread, regardless of the behavior or state of other threads. In a wait-free design, every thread executing an operation completes its operation within a finite number of steps, without being delayed by other threads. Wait-free algorithms are more stringent in their requirements compared to lock-free algorithms and often require more complex synchronization mechanisms.\n\nIt's important to note that both lock-free and wait-free designs aim to avoid traditional locks or blocking synchronization mechanisms (such as mutexes or condition variables) that can lead to contention and thread blocking. Instead, they rely on techniques like atomic operations, compare-and-swap (CAS), or memory fences to ensure progress and prevent data races in concurrent execution.\n\n## [atomic](https://doc.rust-lang.org/stable/std/sync/atomic/index.html)\nRust atomics currently follow the same rules as [C++20 atomics](https://en.cppreference.com/w/cpp/atomic), specifically `atomic_ref`. Basically, creating a shared reference to one of the Rust atomic types corresponds to creating an `atomic_ref` in C++; the atomic_ref is destroyed when the lifetime of the shared reference ends. \nEach method takes an `Ordering` which represents the strength of the memory barrier for that operation. These orderings are the same as the [C++20 atomic orderings](https://en.cppreference.com/w/cpp/atomic/memory_order). For more information see the [nomicon](https://doc.rust-lang.org/stable/nomicon/atomics.html)\nAtomic variables are safe to share between threads (they implement Sync) but they do not themselves provide the mechanism for sharing and follow the threading model of Rust. The most common way to share an atomic variable is to put it into an Arc (an atomically-reference-counted shared pointer).\n\n### Compiler Reordering\nCompilers may change the actual order of events, or make events never occur! If we write something like\n```rust\nx = 1;\ny = 3;\nx = 2;\n```\nThe compiler may conclude that it would be best if your program did:\n```rust\nx = 2;\ny = 3;\n```\nThis has inverted the order of events and completely eliminated one event. But if our program is multi-threaded, we may have been relying on x to actually be assigned to 1 before y was assigned. \n\n### Hardware Reordering\nhere is indeed a global shared memory space somewhere in your hardware, but from the perspective of each CPU core it is so very far away and so very slow. Each CPU would rather work with its local cache of the data and only go through all the anguish of talking to shared memory only when it doesn't actually have that memory in cache. The end result is that the hardware doesn't guarantee that events that occur in some order on one thread, occur in the same order on another thread. To guarantee this, we must issue special instructions to the CPU telling it to be a bit less smart.\nFor instance, say we convince the compiler to emit this logic:\n```\ninitial state: x = 0, y = 1\n\nTHREAD 1 THREAD2\ny = 3; if x == 1 {\nx = 1; y *= 2;\n }\n```\nIdeally this program has 2 possible final states:\n- y = 3: (thread 2 did the check before thread 1 completed)\n- y = 6: (thread 2 did the check after thread 1 completed)\nHowever there's a third potential state that the hardware enables:\n- y = 2: (thread 2 saw x = 1, but not y = 3, and then overwrote y = 3)\nIt's worth noting that different kinds of CPU provide different guarantees. It is common to separate hardware into two categories: strongly-ordered and weakly-ordered. Most notably x86/64 provides strong ordering guarantees, while ARM provides weak ordering guarantees. \n\n### Data Accesses\nAtomic accesses are how we tell the hardware and compiler that our program is multi-threaded. Each atomic access can be marked with an ordering that specifies what kind of relationship it establishes with other accesses. For the compiler, this largely revolves around re-ordering of instructions. For the hardware, this largely revolves around how writes are propagated to other threads. The set of orderings Rust exposes are:\n- Sequentially Consistent (SeqCst)\n- Release\n- Acquire\n- Relaxed\n\n### Sequentially Consistent\nSequentially Consistent is the most powerful of all, implying the restrictions of all other orderings. Intuitively, a sequentially consistent operation cannot be reordered: all accesses on one thread that happen before and after a SeqCst access stay before and after it.\n\n### Acquire-Release\nAcquire and Release are largely intended to be paired. they're perfectly suited for acquiring and releasing locks. \nIntuitively, an acquire access ensures that every access after it stays after it. However operations that occur before an acquire are free to be reordered to occur after it. Similarly, a release access ensures that every access before it stays before it. However operations that occur after a release are free to be reordered to occur before it.\n```rust\nuse std::sync::Arc;\nuse std::sync::atomic::{AtomicBool, Ordering};\nuse std::thread;\n\nfn main() {\n let lock = Arc::new(AtomicBool::new(false)); // value answers \"am I locked?\"\n\n // ... distribute lock to threads somehow ...\n\n // Try to acquire the lock by setting it to true\n while lock.compare_and_swap(false, true, Ordering::Acquire) { }\n // broke out of the loop, so we successfully acquired the lock!\n\n // ... scary data accesses ...\n\n // ok we're done, release the lock\n lock.store(false, Ordering::Release);\n}\n```\n### Relaxed\nRelaxed accesses are the absolute weakest. They can be freely re-ordered and provide no happens-before relationship. Still, relaxed operations are still atomic. That is, they don't count as data accesses and any read-modify-write operations done to them occur atomically. For instance, incrementing a counter can be safely done by multiple threads using a relaxed `fetch_add` if you're not using the counter to synchronize any other accesses.\n\n## an example (spinlock)\n```rust\nuse std::sync::Arc;\nuse std::sync::atomic::{AtomicUsize, Ordering};\nuse std::{hint, thread};\n\nfn main() {\n let spinlock = Arc::new(AtomicUsize::new(1));\n\n let spinlock_clone = Arc::clone(&spinlock);\n let thread = thread::spawn(move|| {\n spinlock_clone.store(0, Ordering::SeqCst);\n });\n\n // Wait for the other thread to release the lock\n while spinlock.load(Ordering::SeqCst) != 0 {\n hint::spin_loop();\n }\n\n if let Err(panic) = thread.join() {\n println!(\"Thread had an error: {panic:?}\");\n }\n}\n```\n\n## usual structs\n1. [AtomicBool](https://doc.rust-lang.org/stable/std/sync/atomic/struct.AtomicBool.html)\n### methods\n- `fn get_mut(&mut self) -> &mut bool`\n- `fn into_inner(self) -> bool`\n- `fn load(&self, order: Ordering) -> bool`\n- `fn store(&self, val: bool, order: Ordering)`\n- `fn compare_exchange(&self, current: bool,new: bool,success: Ordering,failure: Ordering) -> Result`\nStores a value into the bool if the current value is the same as the current value.\ncompare_exchange takes two Ordering arguments to describe the memory ordering of this operation. success describes the required ordering for the read-modify-write operation that takes place if the comparison with current succeeds. failure describes the required ordering for the load operation that takes place when the comparison fails. \n- `fn fetch_and(&self, val: bool, order: Ordering) -> bool`\nLogical “and” with a boolean value.\nPerforms a logical “and” operation on the current value and the argument val, and sets the new value to the result.\n- `const fn as_ptr(&self) -> *mut bool`\nReturns a mutable pointer to the underlying bool.\nDoing non-atomic reads and writes on the resulting integer can be a data race. This method is mostly useful for FFI, where the function signature may use *mut bool instead of &AtomicBool.\n\n2. [AtomicUsize](https://doc.rust-lang.org/stable/std/sync/atomic/struct.AtomicUsize.html)\n2. [AtomicPtr](https://doc.rust-lang.org/stable/std/sync/atomic/struct.AtomicPtr.html)\nA raw pointer type which can be safely shared between threads.\nThis type has the same in-memory representation as a *mut T\n\n\n## Higher-level synchronization objects\nMost of the low-level synchronization primitives are quite error-prone and inconvenient to use, which is why the standard library also exposes some higher-level synchronization objects.\n- **Arc**: Atomically Reference-Counted pointer, which can be used in multithreaded environments to prolong the lifetime of some data until all the threads have finished using it.\n- **Barrier**: Ensures multiple threads will wait for each other to reach a point in the program, before continuing execution all together.\n- **Condvar**: Condition Variable, providing the ability to block a thread while waiting for an event to occur.\n- **mpsc**: Multi-producer, single-consumer queues, used for message-based communication. Can provide a lightweight inter-thread synchronisation mechanism, at the cost of some extra memory.\n- **Mutex**: Mutual Exclusion mechanism, which ensures that at most one thread at a time is able to access some data.\n- **Once**: Used for a thread-safe, one-time global initialization routine\n- **OnceLock**: Used for thread-safe, one-time initialization of a global variable.\n- **RwLock**: Provides a mutual exclusion mechanism which allows multiple readers at the same time, while allowing only one writer at a time. In some cases, this can be more efficient than a mutex.\n\n## mpsc\nThis module provides message-based communication over channels, concretely defined among three types:\n- Sender\n- SyncSender\n- Receiver","slug":"rust/rust_std/rust-std-sync","published":1,"updated":"2023-11-05T04:21:13.735Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clokyy8e4003vqwsj77aj2fxl","content":"

lock free & wait free

“Lock-free” and “wait-free” are two different approaches to designing concurrent algorithms and data structures. Both aim to provide efficient and non-blocking synchronization in concurrent environments.

\n
    \n
  • lock-free A lock-free algorithm or data structure guarantees progress for at least one thread, regardless of the behavior or state of other threads. In a lock-free design, threads can independently perform their operations without being blocked by other threads. If one thread gets delayed or suspended, other threads can continue to make progress. Lock-free algorithms typically use low-level synchronization primitives such as atomic operations to ensure progress and prevent data races.
    • \n
  • wait-free A wait-free algorithm or data structure guarantees progress for every thread, regardless of the behavior or state of other threads. In a wait-free design, every thread executing an operation completes its operation within a finite number of steps, without being delayed by other threads. Wait-free algorithms are more stringent in their requirements compared to lock-free algorithms and often require more complex synchronization mechanisms.
    • \n
\n

It’s important to note that both lock-free and wait-free designs aim to avoid traditional locks or blocking synchronization mechanisms (such as mutexes or condition variables) that can lead to contention and thread blocking. Instead, they rely on techniques like atomic operations, compare-and-swap (CAS), or memory fences to ensure progress and prevent data races in concurrent execution.

\n

atomic

Rust atomics currently follow the same rules as C++20 atomics, specifically atomic_ref. Basically, creating a shared reference to one of the Rust atomic types corresponds to creating an atomic_ref in C++; the atomic_ref is destroyed when the lifetime of the shared reference ends.
Each method takes an Ordering which represents the strength of the memory barrier for that operation. These orderings are the same as the C++20 atomic orderings. For more information see the nomicon
Atomic variables are safe to share between threads (they implement Sync) but they do not themselves provide the mechanism for sharing and follow the threading model of Rust. The most common way to share an atomic variable is to put it into an Arc (an atomically-reference-counted shared pointer).

\n

Compiler Reordering

Compilers may change the actual order of events, or make events never occur! If we write something like

\n
1
2
3
x = 1;
y = 3;
x = 2;
\n

The compiler may conclude that it would be best if your program did:

\n
1
2
x = 2;
y = 3;
\n

This has inverted the order of events and completely eliminated one event. But if our program is multi-threaded, we may have been relying on x to actually be assigned to 1 before y was assigned.

\n

Hardware Reordering

here is indeed a global shared memory space somewhere in your hardware, but from the perspective of each CPU core it is so very far away and so very slow. Each CPU would rather work with its local cache of the data and only go through all the anguish of talking to shared memory only when it doesn’t actually have that memory in cache. The end result is that the hardware doesn’t guarantee that events that occur in some order on one thread, occur in the same order on another thread. To guarantee this, we must issue special instructions to the CPU telling it to be a bit less smart.
For instance, say we convince the compiler to emit this logic:

\n
1
2
3
4
5
6
initial state: x = 0, y = 1

THREAD 1        THREAD2
y = 3;          if x == 1 {
x = 1;              y *= 2;
                }
\n

Ideally this program has 2 possible final states:

\n
    \n
  • y = 3: (thread 2 did the check before thread 1 completed)
    • \n
  • y = 6: (thread 2 did the check after thread 1 completed)
    However there’s a third potential state that the hardware enables:
    • \n
  • y = 2: (thread 2 saw x = 1, but not y = 3, and then overwrote y = 3)
    It’s worth noting that different kinds of CPU provide different guarantees. It is common to separate hardware into two categories: strongly-ordered and weakly-ordered. Most notably x86/64 provides strong ordering guarantees, while ARM provides weak ordering guarantees.
    • \n
\n

Data Accesses

Atomic accesses are how we tell the hardware and compiler that our program is multi-threaded. Each atomic access can be marked with an ordering that specifies what kind of relationship it establishes with other accesses. For the compiler, this largely revolves around re-ordering of instructions. For the hardware, this largely revolves around how writes are propagated to other threads. The set of orderings Rust exposes are:

\n
    \n
  • Sequentially Consistent (SeqCst)
    • \n
  • Release
    • \n
  • Acquire
    • \n
  • Relaxed
    • \n
\n

Sequentially Consistent

Sequentially Consistent is the most powerful of all, implying the restrictions of all other orderings. Intuitively, a sequentially consistent operation cannot be reordered: all accesses on one thread that happen before and after a SeqCst access stay before and after it.

\n

Acquire-Release

Acquire and Release are largely intended to be paired. they’re perfectly suited for acquiring and releasing locks.
Intuitively, an acquire access ensures that every access after it stays after it. However operations that occur before an acquire are free to be reordered to occur after it. Similarly, a release access ensures that every access before it stays before it. However operations that occur after a release are free to be reordered to occur before it.

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::thread;

fn main() {
    let lock = Arc::new(AtomicBool::new(false)); // value answers "am I locked?"

    // ... distribute lock to threads somehow ...

    // Try to acquire the lock by setting it to true
    while lock.compare_and_swap(false, true, Ordering::Acquire) { }
    // broke out of the loop, so we successfully acquired the lock!

    // ... scary data accesses ...

    // ok we're done, release the lock
    lock.store(false, Ordering::Release);
}
\n

Relaxed

Relaxed accesses are the absolute weakest. They can be freely re-ordered and provide no happens-before relationship. Still, relaxed operations are still atomic. That is, they don’t count as data accesses and any read-modify-write operations done to them occur atomically. For instance, incrementing a counter can be safely done by multiple threads using a relaxed fetch_add if you’re not using the counter to synchronize any other accesses.

\n

an example (spinlock)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::{hint, thread};

fn main() {
    let spinlock = Arc::new(AtomicUsize::new(1));

    let spinlock_clone = Arc::clone(&spinlock);
    let thread = thread::spawn(move|| {
        spinlock_clone.store(0, Ordering::SeqCst);
    });

    // Wait for the other thread to release the lock
    while spinlock.load(Ordering::SeqCst) != 0 {
        hint::spin_loop();
    }

    if let Err(panic) = thread.join() {
        println!("Thread had an error: {panic:?}");
    }
}
\n\n

usual structs

    \n
  1. AtomicBool
    2. \n
\n

methods

    \n
  • fn get_mut(&mut self) -> &mut bool
    • \n
  • fn into_inner(self) -> bool
    • \n
  • fn load(&self, order: Ordering) -> bool
    • \n
  • fn store(&self, val: bool, order: Ordering)
    • \n
  • fn compare_exchange(&self, current: bool,new: bool,success: Ordering,failure: Ordering) -> Result<bool, bool>
    Stores a value into the bool if the current value is the same as the current value.
    compare_exchange takes two Ordering arguments to describe the memory ordering of this operation. success describes the required ordering for the read-modify-write operation that takes place if the comparison with current succeeds. failure describes the required ordering for the load operation that takes place when the comparison fails.
    • \n
  • fn fetch_and(&self, val: bool, order: Ordering) -> bool
    Logical “and” with a boolean value.
    Performs a logical “and” operation on the current value and the argument val, and sets the new value to the result.
    • \n
  • const fn as_ptr(&self) -> *mut bool
    Returns a mutable pointer to the underlying bool.
    Doing non-atomic reads and writes on the resulting integer can be a data race. This method is mostly useful for FFI, where the function signature may use *mut bool instead of &AtomicBool.
    • \n
\n
    \n
  1. AtomicUsize
    2. \n
  2. AtomicPtr
    A raw pointer type which can be safely shared between threads.
    This type has the same in-memory representation as a *mut T
    3. \n
\n

Higher-level synchronization objects

Most of the low-level synchronization primitives are quite error-prone and inconvenient to use, which is why the standard library also exposes some higher-level synchronization objects.

\n
    \n
  • Arc: Atomically Reference-Counted pointer, which can be used in multithreaded environments to prolong the lifetime of some data until all the threads have finished using it.
    • \n
  • Barrier: Ensures multiple threads will wait for each other to reach a point in the program, before continuing execution all together.
    • \n
  • Condvar: Condition Variable, providing the ability to block a thread while waiting for an event to occur.
    • \n
  • mpsc: Multi-producer, single-consumer queues, used for message-based communication. Can provide a lightweight inter-thread synchronisation mechanism, at the cost of some extra memory.
    • \n
  • Mutex: Mutual Exclusion mechanism, which ensures that at most one thread at a time is able to access some data.
    • \n
  • Once: Used for a thread-safe, one-time global initialization routine
    • \n
  • OnceLock: Used for thread-safe, one-time initialization of a global variable.
    • \n
  • RwLock: Provides a mutual exclusion mechanism which allows multiple readers at the same time, while allowing only one writer at a time. In some cases, this can be more efficient than a mutex.
    • \n
\n

mpsc

This module provides message-based communication over channels, concretely defined among three types:

\n
    \n
  • Sender
    • \n
  • SyncSender
    • \n
  • Receiver
    • \n
\n","site":{"data":{}},"excerpt":"","more":"

lock free & wait free

“Lock-free” and “wait-free” are two different approaches to designing concurrent algorithms and data structures. Both aim to provide efficient and non-blocking synchronization in concurrent environments.

\n
    \n
  • lock-free A lock-free algorithm or data structure guarantees progress for at least one thread, regardless of the behavior or state of other threads. In a lock-free design, threads can independently perform their operations without being blocked by other threads. If one thread gets delayed or suspended, other threads can continue to make progress. Lock-free algorithms typically use low-level synchronization primitives such as atomic operations to ensure progress and prevent data races.
    • \n
  • wait-free A wait-free algorithm or data structure guarantees progress for every thread, regardless of the behavior or state of other threads. In a wait-free design, every thread executing an operation completes its operation within a finite number of steps, without being delayed by other threads. Wait-free algorithms are more stringent in their requirements compared to lock-free algorithms and often require more complex synchronization mechanisms.
    • \n
\n

It’s important to note that both lock-free and wait-free designs aim to avoid traditional locks or blocking synchronization mechanisms (such as mutexes or condition variables) that can lead to contention and thread blocking. Instead, they rely on techniques like atomic operations, compare-and-swap (CAS), or memory fences to ensure progress and prevent data races in concurrent execution.

\n

atomic

Rust atomics currently follow the same rules as C++20 atomics, specifically atomic_ref. Basically, creating a shared reference to one of the Rust atomic types corresponds to creating an atomic_ref in C++; the atomic_ref is destroyed when the lifetime of the shared reference ends.
Each method takes an Ordering which represents the strength of the memory barrier for that operation. These orderings are the same as the C++20 atomic orderings. For more information see the nomicon
Atomic variables are safe to share between threads (they implement Sync) but they do not themselves provide the mechanism for sharing and follow the threading model of Rust. The most common way to share an atomic variable is to put it into an Arc (an atomically-reference-counted shared pointer).

\n

Compiler Reordering

Compilers may change the actual order of events, or make events never occur! If we write something like

\n
1
2
3
x = 1;
y = 3;
x = 2;
\n

The compiler may conclude that it would be best if your program did:

\n
1
2
x = 2;
y = 3;
\n

This has inverted the order of events and completely eliminated one event. But if our program is multi-threaded, we may have been relying on x to actually be assigned to 1 before y was assigned.

\n

Hardware Reordering

here is indeed a global shared memory space somewhere in your hardware, but from the perspective of each CPU core it is so very far away and so very slow. Each CPU would rather work with its local cache of the data and only go through all the anguish of talking to shared memory only when it doesn’t actually have that memory in cache. The end result is that the hardware doesn’t guarantee that events that occur in some order on one thread, occur in the same order on another thread. To guarantee this, we must issue special instructions to the CPU telling it to be a bit less smart.
For instance, say we convince the compiler to emit this logic:

\n
1
2
3
4
5
6
initial state: x = 0, y = 1

THREAD 1        THREAD2
y = 3;          if x == 1 {
x = 1;              y *= 2;
                }
\n

Ideally this program has 2 possible final states:

\n
    \n
  • y = 3: (thread 2 did the check before thread 1 completed)
    • \n
  • y = 6: (thread 2 did the check after thread 1 completed)
    However there’s a third potential state that the hardware enables:
    • \n
  • y = 2: (thread 2 saw x = 1, but not y = 3, and then overwrote y = 3)
    It’s worth noting that different kinds of CPU provide different guarantees. It is common to separate hardware into two categories: strongly-ordered and weakly-ordered. Most notably x86/64 provides strong ordering guarantees, while ARM provides weak ordering guarantees.
    • \n
\n

Data Accesses

Atomic accesses are how we tell the hardware and compiler that our program is multi-threaded. Each atomic access can be marked with an ordering that specifies what kind of relationship it establishes with other accesses. For the compiler, this largely revolves around re-ordering of instructions. For the hardware, this largely revolves around how writes are propagated to other threads. The set of orderings Rust exposes are:

\n
    \n
  • Sequentially Consistent (SeqCst)
    • \n
  • Release
    • \n
  • Acquire
    • \n
  • Relaxed
    • \n
\n

Sequentially Consistent

Sequentially Consistent is the most powerful of all, implying the restrictions of all other orderings. Intuitively, a sequentially consistent operation cannot be reordered: all accesses on one thread that happen before and after a SeqCst access stay before and after it.

\n

Acquire-Release

Acquire and Release are largely intended to be paired. they’re perfectly suited for acquiring and releasing locks.
Intuitively, an acquire access ensures that every access after it stays after it. However operations that occur before an acquire are free to be reordered to occur after it. Similarly, a release access ensures that every access before it stays before it. However operations that occur after a release are free to be reordered to occur before it.

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::thread;

fn main() {
    let lock = Arc::new(AtomicBool::new(false)); // value answers "am I locked?"

    // ... distribute lock to threads somehow ...

    // Try to acquire the lock by setting it to true
    while lock.compare_and_swap(false, true, Ordering::Acquire) { }
    // broke out of the loop, so we successfully acquired the lock!

    // ... scary data accesses ...

    // ok we're done, release the lock
    lock.store(false, Ordering::Release);
}
\n

Relaxed

Relaxed accesses are the absolute weakest. They can be freely re-ordered and provide no happens-before relationship. Still, relaxed operations are still atomic. That is, they don’t count as data accesses and any read-modify-write operations done to them occur atomically. For instance, incrementing a counter can be safely done by multiple threads using a relaxed fetch_add if you’re not using the counter to synchronize any other accesses.

\n

an example (spinlock)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::{hint, thread};

fn main() {
    let spinlock = Arc::new(AtomicUsize::new(1));

    let spinlock_clone = Arc::clone(&spinlock);
    let thread = thread::spawn(move|| {
        spinlock_clone.store(0, Ordering::SeqCst);
    });

    // Wait for the other thread to release the lock
    while spinlock.load(Ordering::SeqCst) != 0 {
        hint::spin_loop();
    }

    if let Err(panic) = thread.join() {
        println!("Thread had an error: {panic:?}");
    }
}
\n\n

usual structs

    \n
  1. AtomicBool
    2. \n
\n

methods

    \n
  • fn get_mut(&mut self) -> &mut bool
    • \n
  • fn into_inner(self) -> bool
    • \n
  • fn load(&self, order: Ordering) -> bool
    • \n
  • fn store(&self, val: bool, order: Ordering)
    • \n
  • fn compare_exchange(&self, current: bool,new: bool,success: Ordering,failure: Ordering) -> Result<bool, bool>
    Stores a value into the bool if the current value is the same as the current value.
    compare_exchange takes two Ordering arguments to describe the memory ordering of this operation. success describes the required ordering for the read-modify-write operation that takes place if the comparison with current succeeds. failure describes the required ordering for the load operation that takes place when the comparison fails.
    • \n
  • fn fetch_and(&self, val: bool, order: Ordering) -> bool
    Logical “and” with a boolean value.
    Performs a logical “and” operation on the current value and the argument val, and sets the new value to the result.
    • \n
  • const fn as_ptr(&self) -> *mut bool
    Returns a mutable pointer to the underlying bool.
    Doing non-atomic reads and writes on the resulting integer can be a data race. This method is mostly useful for FFI, where the function signature may use *mut bool instead of &AtomicBool.
    • \n
\n
    \n
  1. AtomicUsize
    2. \n
  2. AtomicPtr
    A raw pointer type which can be safely shared between threads.
    This type has the same in-memory representation as a *mut T
    3. \n
\n

Higher-level synchronization objects

Most of the low-level synchronization primitives are quite error-prone and inconvenient to use, which is why the standard library also exposes some higher-level synchronization objects.

\n
    \n
  • Arc: Atomically Reference-Counted pointer, which can be used in multithreaded environments to prolong the lifetime of some data until all the threads have finished using it.
    • \n
  • Barrier: Ensures multiple threads will wait for each other to reach a point in the program, before continuing execution all together.
    • \n
  • Condvar: Condition Variable, providing the ability to block a thread while waiting for an event to occur.
    • \n
  • mpsc: Multi-producer, single-consumer queues, used for message-based communication. Can provide a lightweight inter-thread synchronisation mechanism, at the cost of some extra memory.
    • \n
  • Mutex: Mutual Exclusion mechanism, which ensures that at most one thread at a time is able to access some data.
    • \n
  • Once: Used for a thread-safe, one-time global initialization routine
    • \n
  • OnceLock: Used for thread-safe, one-time initialization of a global variable.
    • \n
  • RwLock: Provides a mutual exclusion mechanism which allows multiple readers at the same time, while allowing only one writer at a time. In some cases, this can be more efficient than a mutex.
    • \n
\n

mpsc

This module provides message-based communication over channels, concretely defined among three types:

\n
    \n
  • Sender
    • \n
  • SyncSender
    • \n
  • Receiver
    • \n
\n"},{"title":"inline ptx assembly in cuda","date":"2023-11-15T06:47:28.000Z","_content":"# introduction\nPTX, a low-level parallel thread execution virtual machine and instruction set architecture (ISA). PTX exposes the GPU as a data-parallel computing device.\nThe PTX Instruction Set Architecture (ISA) is lower-level compared to CUDA C++, but the programming model is entirely the same, with only some terminology differences. For example: CTA (Cooperative Thread Array): Equivalent to the Block in the CUDA thread model.\n\n# assembler statements\n## parameters\n```cpp\nasm(\"template-string\" : \"constraint\"(output) : \"constraint\"(input));\n```\nan example\n```cpp\nasm(\"add.s32 %0, %1, %2;\" : \"=r\"(i) : \"r\"(j), \"r\"(k));\n```\nEach `%n` in the template string is an index into the following list of operands, in text order. So `%0` refers to the first operand, `%1` to the second operand, and so on. Since the output operands are always listed ahead of the input operands, they are assigned the smallest indices. \nNote that the numbered references in the string can be in arbitrary order. The following is equivalent to the above example:\n```cpp\nasm(\"add.s32 %0, %2, %1;\" : \"=r\"(i) : \"r\"(k), \"r\"(j));\n```\nYou can also repeat a reference, e.g.:\n```cpp\nasm(\"add.s32 %0, %1, %1;\" : \"=r\"(i) : \"r\"(k));\n```\nIf there is no input operand, you can drop the final colon, e.g.:\n```cpp\nasm(\"mov.s32 %0, 2;\" : \"=r\"(i));\n```\n\nIf there is no output operand, the colon separators are adjacent, e.g.:\n\n\n```cpp\nasm(\"mov.s32 r1, %0;\" :: \"r\"(i));\n```\nthe “r” constraint refers to a 32bit integer register. The “=” modifier in “=r” specifies that the register is written to. There is also available a “+” modifier that specifies the register is both read and written\n\nMultiple instructions can be combined into a single asm() statement; \nam example\n```cpp\n__device__ int cube (int x)\n{\n int y;\n asm(\".reg .u32 t1;\\n\\t\" // temp reg t1\n \" mul.lo.u32 t1, %1, %1;\\n\\t\" // t1 = x * x\n \" mul.lo.u32 %0, t1, %1;\" // y = t1 * x\n : \"=r\"(y) : \"r\" (x));\n return y;\n}\n```\n\n## constraints\nThere is a separate constraint letter for each PTX register type:\n```cpp\n\"h\" = .u16 reg\n\"r\" = .u32 reg\n\"l\" = .u64 reg\n\"f\" = .f32 reg\n\"d\" = .f64 reg\n```\nThe constraint \"n\" may be used for immediate integer operands with a known value. Example:\n```cpp\nasm(\"add.u32 %0, %0, %1;\" : \"=r\"(x) : \"n\"(42));\n```\n\n\n# State Space + Type + Identifier\n## state spaces\nA state space is a storage area with particular characteristics. All variables reside in some state space. \n| Name | Description |\n| ----------- | ----------- |\n| .reg | Registers, fast. |\n| .sreg | Special registers. Read-only; pre-defined; platform-specific. |\n|.const|Shared, read-only memory.|\n|.global|Global memory, shared by all threads.|\n|.local|Local memory, private to each thread.|\n|.param|Kernel parameters, defined per-grid; or Function or local parameters, defined per-thread.|\n|.shared|Addressable memory, defined per CTA, accessible to all threads in the cluster throughout the lifetime of the CTA that defines it.|\n\n\n## types\n### fundamental types\nIn PTX, the fundamental types reflect the native data types supported by the target architectures.\n| Basic Type | Fundamental Type Specifiers |\n| ----------- | ----------- |\n| Signed integer | .s8, .s16, .s32, .s64 |\n|Unsigned integer|.u8, .u16, .u32, .u64|\n|Floating-point|.f16, .f16x2, .f32, .f64|\n|Bits (untyped)|.b8, .b16, .b32, .b64, .b128|\n|Predicate|.pred|\n\n## variables\nIn PTX, a variable declaration describes both the variable’s type and its state space. In addition to fundamental types, PTX supports types for simple aggregate objects such as vectors and arrays.\n**examples**\n```cpp\n.global .v4 .f32 V; // a length-4 vector of floats\n.shared .v2 .u16 uv; // a length-2 vector of unsigned ints\n.global .v4 .b8 v; // a length-4 vector of bytes\n```\n\n# references\n- [nvidia docs inlline ptx assembly](https://docs.nvidia.com/cuda/inline-ptx-assembly/index.html)\n- [IBM Open XL C/C++ Inline Assembly](https://www.ibm.com/docs/en/openxl-c-and-cpp-aix/17.1.0?topic=features-inline-assembly-statements)\n- [parallel thread execution ISA guide](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#)","source":"_posts/cuda/inline-ptx-assembly-in-cuda.md","raw":"---\ntitle: inline ptx assembly in cuda\ndate: 2023-11-15 14:47:28\ntags: [cuda]\n---\n# introduction\nPTX, a low-level parallel thread execution virtual machine and instruction set architecture (ISA). PTX exposes the GPU as a data-parallel computing device.\nThe PTX Instruction Set Architecture (ISA) is lower-level compared to CUDA C++, but the programming model is entirely the same, with only some terminology differences. For example: CTA (Cooperative Thread Array): Equivalent to the Block in the CUDA thread model.\n\n# assembler statements\n## parameters\n```cpp\nasm(\"template-string\" : \"constraint\"(output) : \"constraint\"(input));\n```\nan example\n```cpp\nasm(\"add.s32 %0, %1, %2;\" : \"=r\"(i) : \"r\"(j), \"r\"(k));\n```\nEach `%n` in the template string is an index into the following list of operands, in text order. So `%0` refers to the first operand, `%1` to the second operand, and so on. Since the output operands are always listed ahead of the input operands, they are assigned the smallest indices. \nNote that the numbered references in the string can be in arbitrary order. The following is equivalent to the above example:\n```cpp\nasm(\"add.s32 %0, %2, %1;\" : \"=r\"(i) : \"r\"(k), \"r\"(j));\n```\nYou can also repeat a reference, e.g.:\n```cpp\nasm(\"add.s32 %0, %1, %1;\" : \"=r\"(i) : \"r\"(k));\n```\nIf there is no input operand, you can drop the final colon, e.g.:\n```cpp\nasm(\"mov.s32 %0, 2;\" : \"=r\"(i));\n```\n\nIf there is no output operand, the colon separators are adjacent, e.g.:\n\n\n```cpp\nasm(\"mov.s32 r1, %0;\" :: \"r\"(i));\n```\nthe “r” constraint refers to a 32bit integer register. The “=” modifier in “=r” specifies that the register is written to. There is also available a “+” modifier that specifies the register is both read and written\n\nMultiple instructions can be combined into a single asm() statement; \nam example\n```cpp\n__device__ int cube (int x)\n{\n int y;\n asm(\".reg .u32 t1;\\n\\t\" // temp reg t1\n \" mul.lo.u32 t1, %1, %1;\\n\\t\" // t1 = x * x\n \" mul.lo.u32 %0, t1, %1;\" // y = t1 * x\n : \"=r\"(y) : \"r\" (x));\n return y;\n}\n```\n\n## constraints\nThere is a separate constraint letter for each PTX register type:\n```cpp\n\"h\" = .u16 reg\n\"r\" = .u32 reg\n\"l\" = .u64 reg\n\"f\" = .f32 reg\n\"d\" = .f64 reg\n```\nThe constraint \"n\" may be used for immediate integer operands with a known value. Example:\n```cpp\nasm(\"add.u32 %0, %0, %1;\" : \"=r\"(x) : \"n\"(42));\n```\n\n\n# State Space + Type + Identifier\n## state spaces\nA state space is a storage area with particular characteristics. All variables reside in some state space. \n| Name | Description |\n| ----------- | ----------- |\n| .reg | Registers, fast. |\n| .sreg | Special registers. Read-only; pre-defined; platform-specific. |\n|.const|Shared, read-only memory.|\n|.global|Global memory, shared by all threads.|\n|.local|Local memory, private to each thread.|\n|.param|Kernel parameters, defined per-grid; or Function or local parameters, defined per-thread.|\n|.shared|Addressable memory, defined per CTA, accessible to all threads in the cluster throughout the lifetime of the CTA that defines it.|\n\n\n## types\n### fundamental types\nIn PTX, the fundamental types reflect the native data types supported by the target architectures.\n| Basic Type | Fundamental Type Specifiers |\n| ----------- | ----------- |\n| Signed integer | .s8, .s16, .s32, .s64 |\n|Unsigned integer|.u8, .u16, .u32, .u64|\n|Floating-point|.f16, .f16x2, .f32, .f64|\n|Bits (untyped)|.b8, .b16, .b32, .b64, .b128|\n|Predicate|.pred|\n\n## variables\nIn PTX, a variable declaration describes both the variable’s type and its state space. In addition to fundamental types, PTX supports types for simple aggregate objects such as vectors and arrays.\n**examples**\n```cpp\n.global .v4 .f32 V; // a length-4 vector of floats\n.shared .v2 .u16 uv; // a length-2 vector of unsigned ints\n.global .v4 .b8 v; // a length-4 vector of bytes\n```\n\n# references\n- [nvidia docs inlline ptx assembly](https://docs.nvidia.com/cuda/inline-ptx-assembly/index.html)\n- [IBM Open XL C/C++ Inline Assembly](https://www.ibm.com/docs/en/openxl-c-and-cpp-aix/17.1.0?topic=features-inline-assembly-statements)\n- [parallel thread execution ISA guide](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#)","slug":"cuda/inline-ptx-assembly-in-cuda","published":1,"updated":"2023-11-22T02:15:56.342Z","_id":"clp3h29tl0000p97ug63eajx3","comments":1,"layout":"post","photos":[],"link":"","content":"

introduction

PTX, a low-level parallel thread execution virtual machine and instruction set architecture (ISA). PTX exposes the GPU as a data-parallel computing device.
The PTX Instruction Set Architecture (ISA) is lower-level compared to CUDA C++, but the programming model is entirely the same, with only some terminology differences. For example: CTA (Cooperative Thread Array): Equivalent to the Block in the CUDA thread model.

\n

assembler statements

parameters

1
asm("template-string" : "constraint"(output) : "constraint"(input));
\n

an example

\n
1
asm("add.s32 %0, %1, %2;" : "=r"(i) : "r"(j), "r"(k));
\n

Each %n in the template string is an index into the following list of operands, in text order. So %0 refers to the first operand, %1 to the second operand, and so on. Since the output operands are always listed ahead of the input operands, they are assigned the smallest indices.
Note that the numbered references in the string can be in arbitrary order. The following is equivalent to the above example:

\n
1
asm("add.s32 %0, %2, %1;" : "=r"(i) : "r"(k), "r"(j));
\n

You can also repeat a reference, e.g.:

\n
1
asm("add.s32 %0, %1, %1;" : "=r"(i) : "r"(k));
\n

If there is no input operand, you can drop the final colon, e.g.:

\n
1
asm("mov.s32 %0, 2;" : "=r"(i));
\n\n

If there is no output operand, the colon separators are adjacent, e.g.:

\n
1
asm("mov.s32 r1, %0;" :: "r"(i));
\n

the “r” constraint refers to a 32bit integer register. The “=” modifier in “=r” specifies that the register is written to. There is also available a “+” modifier that specifies the register is both read and written

\n

Multiple instructions can be combined into a single asm() statement;
am example

\n
1
2
3
4
5
6
7
8
9
__device__ int cube (int x)
{
  int y;
  asm(".reg .u32 t1;\\n\\t"              // temp reg t1
      " mul.lo.u32 t1, %1, %1;\\n\\t"    // t1 = x * x
      " mul.lo.u32 %0, t1, %1;"        // y = t1 * x
      : "=r"(y) : "r" (x));
  return y;
}
\n\n

constraints

There is a separate constraint letter for each PTX register type:

\n
1
2
3
4
5
"h" = .u16 reg
"r" = .u32 reg
"l" = .u64 reg
"f" = .f32 reg
"d" = .f64 reg
\n

The constraint “n” may be used for immediate integer operands with a known value. Example:

\n
1
asm("add.u32 %0, %0, %1;" : "=r"(x) : "n"(42));
\n\n\n

State Space + Type + Identifier

state spaces

A state space is a storage area with particular characteristics. All variables reside in some state space.

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
NameDescription
.regRegisters, fast.
.sregSpecial registers. Read-only; pre-defined; platform-specific.
.constShared, read-only memory.
.globalGlobal memory, shared by all threads.
.localLocal memory, private to each thread.
.paramKernel parameters, defined per-grid; or Function or local parameters, defined per-thread.
.sharedAddressable memory, defined per CTA, accessible to all threads in the cluster throughout the lifetime of the CTA that defines it.
\n

types

fundamental types

In PTX, the fundamental types reflect the native data types supported by the target architectures.

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
Basic TypeFundamental Type Specifiers
Signed integer.s8, .s16, .s32, .s64
Unsigned integer.u8, .u16, .u32, .u64
Floating-point.f16, .f16x2, .f32, .f64
Bits (untyped).b8, .b16, .b32, .b64, .b128
Predicate.pred
\n

variables

In PTX, a variable declaration describes both the variable’s type and its state space. In addition to fundamental types, PTX supports types for simple aggregate objects such as vectors and arrays.
examples

\n
1
2
3
.global .v4 .f32 V;   // a length-4 vector of floats
.shared .v2 .u16 uv;  // a length-2 vector of unsigned ints
.global .v4 .b8  v;   // a length-4 vector of bytes
\n\n

references

\n","site":{"data":{}},"excerpt":"","more":"

introduction

PTX, a low-level parallel thread execution virtual machine and instruction set architecture (ISA). PTX exposes the GPU as a data-parallel computing device.
The PTX Instruction Set Architecture (ISA) is lower-level compared to CUDA C++, but the programming model is entirely the same, with only some terminology differences. For example: CTA (Cooperative Thread Array): Equivalent to the Block in the CUDA thread model.

\n

assembler statements

parameters

1
asm("template-string" : "constraint"(output) : "constraint"(input));
\n

an example

\n
1
asm("add.s32 %0, %1, %2;" : "=r"(i) : "r"(j), "r"(k));
\n

Each %n in the template string is an index into the following list of operands, in text order. So %0 refers to the first operand, %1 to the second operand, and so on. Since the output operands are always listed ahead of the input operands, they are assigned the smallest indices.
Note that the numbered references in the string can be in arbitrary order. The following is equivalent to the above example:

\n
1
asm("add.s32 %0, %2, %1;" : "=r"(i) : "r"(k), "r"(j));
\n

You can also repeat a reference, e.g.:

\n
1
asm("add.s32 %0, %1, %1;" : "=r"(i) : "r"(k));
\n

If there is no input operand, you can drop the final colon, e.g.:

\n
1
asm("mov.s32 %0, 2;" : "=r"(i));
\n\n

If there is no output operand, the colon separators are adjacent, e.g.:

\n
1
asm("mov.s32 r1, %0;" :: "r"(i));
\n

the “r” constraint refers to a 32bit integer register. The “=” modifier in “=r” specifies that the register is written to. There is also available a “+” modifier that specifies the register is both read and written

\n

Multiple instructions can be combined into a single asm() statement;
am example

\n
1
2
3
4
5
6
7
8
9
__device__ int cube (int x)
{
  int y;
  asm(".reg .u32 t1;\\n\\t"              // temp reg t1
      " mul.lo.u32 t1, %1, %1;\\n\\t"    // t1 = x * x
      " mul.lo.u32 %0, t1, %1;"        // y = t1 * x
      : "=r"(y) : "r" (x));
  return y;
}
\n\n

constraints

There is a separate constraint letter for each PTX register type:

\n
1
2
3
4
5
"h" = .u16 reg
"r" = .u32 reg
"l" = .u64 reg
"f" = .f32 reg
"d" = .f64 reg
\n

The constraint “n” may be used for immediate integer operands with a known value. Example:

\n
1
asm("add.u32 %0, %0, %1;" : "=r"(x) : "n"(42));
\n\n\n

State Space + Type + Identifier

state spaces

A state space is a storage area with particular characteristics. All variables reside in some state space.

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
NameDescription
.regRegisters, fast.
.sregSpecial registers. Read-only; pre-defined; platform-specific.
.constShared, read-only memory.
.globalGlobal memory, shared by all threads.
.localLocal memory, private to each thread.
.paramKernel parameters, defined per-grid; or Function or local parameters, defined per-thread.
.sharedAddressable memory, defined per CTA, accessible to all threads in the cluster throughout the lifetime of the CTA that defines it.
\n

types

fundamental types

In PTX, the fundamental types reflect the native data types supported by the target architectures.

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
Basic TypeFundamental Type Specifiers
Signed integer.s8, .s16, .s32, .s64
Unsigned integer.u8, .u16, .u32, .u64
Floating-point.f16, .f16x2, .f32, .f64
Bits (untyped).b8, .b16, .b32, .b64, .b128
Predicate.pred
\n

variables

In PTX, a variable declaration describes both the variable’s type and its state space. In addition to fundamental types, PTX supports types for simple aggregate objects such as vectors and arrays.
examples

\n
1
2
3
.global .v4 .f32 V;   // a length-4 vector of floats
.shared .v2 .u16 uv;  // a length-2 vector of unsigned ints
.global .v4 .b8  v;   // a length-4 vector of bytes
\n\n

references

\n"},{"title":"cpp generics","date":"2023-11-21T03:35:20.000Z","_content":"\n# generics\n## variadic template parameter\n```cpp\ntemplate\nvoid printValues(const Types&... values) {\n // Fold expression (C++17 and later) to print all values\n (std::cout << ... << values) << std::endl;\n}\n```\n\n# macros\n## examples\n```cpp\n#if defined(_WIN32)\n#else\n#endif\n```\n- `__cplusplus` macro is a predefined macro in C++ that is used to indicate the version of the C++ standard being used by the compiler. The value of `__cplusplus` is an integer that represents the version.\n- __FILE__\n`__FILE__` is a predefined macro in C and C++ that expands to the name of the current source file as a string literal.\n- __LINE__\n`__LINE__` is a predefined macro in C and C++ that expands to the current line number in the source code.","source":"_posts/cpp/generics_and_macros.md","raw":"---\ntitle: cpp generics\ndate: 2023-11-21 11:35:20\ntags: [cpp]\n---\n\n# generics\n## variadic template parameter\n```cpp\ntemplate\nvoid printValues(const Types&... values) {\n // Fold expression (C++17 and later) to print all values\n (std::cout << ... << values) << std::endl;\n}\n```\n\n# macros\n## examples\n```cpp\n#if defined(_WIN32)\n#else\n#endif\n```\n- `__cplusplus` macro is a predefined macro in C++ that is used to indicate the version of the C++ standard being used by the compiler. The value of `__cplusplus` is an integer that represents the version.\n- __FILE__\n`__FILE__` is a predefined macro in C and C++ that expands to the name of the current source file as a string literal.\n- __LINE__\n`__LINE__` is a predefined macro in C and C++ that expands to the current line number in the source code.","slug":"cpp/generics_and_macros","published":1,"updated":"2023-11-23T02:30:16.168Z","_id":"clp8cm0vx0000l57u7eqtbi6s","comments":1,"layout":"post","photos":[],"link":"","content":"

generics

variadic template parameter

1
2
3
4
5
template<typename... Types>
void printValues(const Types&... values) {
    // Fold expression (C++17 and later) to print all values
    (std::cout << ... << values) << std::endl;
}
\n\n

macros

examples

1
2
3
#if defined(_WIN32)
#else
#endif
\n
    \n
  • __cplusplus macro is a predefined macro in C++ that is used to indicate the version of the C++ standard being used by the compiler. The value of __cplusplus is an integer that represents the version.
    • \n
  • FILE
    __FILE__ is a predefined macro in C and C++ that expands to the name of the current source file as a string literal.
    • \n
  • LINE
    __LINE__ is a predefined macro in C and C++ that expands to the current line number in the source code.
    • \n
\n","site":{"data":{}},"excerpt":"","more":"

generics

variadic template parameter

1
2
3
4
5
template<typename... Types>
void printValues(const Types&... values) {
    // Fold expression (C++17 and later) to print all values
    (std::cout << ... << values) << std::endl;
}
\n\n

macros

examples

1
2
3
#if defined(_WIN32)
#else
#endif
\n
    \n
  • __cplusplus macro is a predefined macro in C++ that is used to indicate the version of the C++ standard being used by the compiler. The value of __cplusplus is an integer that represents the version.
    • \n
  • FILE
    __FILE__ is a predefined macro in C and C++ that expands to the name of the current source file as a string literal.
    • \n
  • LINE
    __LINE__ is a predefined macro in C and C++ that expands to the current line number in the source code.
    • \n
\n"},{"title":"cpp key workds","date":"2023-11-21T03:35:20.000Z","_content":"\n- `decltype`\n`decltype` is a keyword that is used to obtain the type of an expression or an entity\n```cpp\n#include \n\nint main() {\n int x = 42;\n double y = 3.14;\n\n // Using decltype to declare a variable with the same type as another variable\n decltype(x) result1 = x; // result1 has the type int\n decltype(y) result2 = y; // result2 has the type double\n\n // Using decltype with an expression\n int a = 10;\n int b = 20;\n decltype(a + b) sum = a + b; // sum has the type int\n\n return 0;\n}\n```\n\n- `friend`\nIn C++, the friend keyword is used to grant non-member functions or other classes access to the private and protected members of a class. When a function or class is declared as a friend of another class, it is allowed to access private and protected members of that class as if it were a member of that class.","source":"_posts/cpp/key_words.md","raw":"---\ntitle: cpp key workds\ndate: 2023-11-21 11:35:20\ntags: [cpp]\n---\n\n- `decltype`\n`decltype` is a keyword that is used to obtain the type of an expression or an entity\n```cpp\n#include \n\nint main() {\n int x = 42;\n double y = 3.14;\n\n // Using decltype to declare a variable with the same type as another variable\n decltype(x) result1 = x; // result1 has the type int\n decltype(y) result2 = y; // result2 has the type double\n\n // Using decltype with an expression\n int a = 10;\n int b = 20;\n decltype(a + b) sum = a + b; // sum has the type int\n\n return 0;\n}\n```\n\n- `friend`\nIn C++, the friend keyword is used to grant non-member functions or other classes access to the private and protected members of a class. When a function or class is declared as a friend of another class, it is allowed to access private and protected members of that class as if it were a member of that class.","slug":"cpp/key_words","published":1,"updated":"2023-11-23T07:05:38.353Z","_id":"clp8cm0w50003l57ubl7e5d6y","comments":1,"layout":"post","photos":[],"link":"","content":"
    \n

  • decltype
    decltype is a keyword that is used to obtain the type of an expression or an entity

    \n
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    #include <iostream>

    int main() {
        int x = 42;
        double y = 3.14;

        // Using decltype to declare a variable with the same type as another variable
        decltype(x) result1 = x;  // result1 has the type int
        decltype(y) result2 = y;  // result2 has the type double

        // Using decltype with an expression
        int a = 10;
        int b = 20;
        decltype(a + b) sum = a + b;  // sum has the type int

        return 0;
    }
    \n
    • \n

  • friend
    In C++, the friend keyword is used to grant non-member functions or other classes access to the private and protected members of a class. When a function or class is declared as a friend of another class, it is allowed to access private and protected members of that class as if it were a member of that class.

    \n
    • \n
\n","site":{"data":{}},"excerpt":"","more":"
    \n

  • decltype
    decltype is a keyword that is used to obtain the type of an expression or an entity

    \n
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    #include <iostream>

    int main() {
        int x = 42;
        double y = 3.14;

        // Using decltype to declare a variable with the same type as another variable
        decltype(x) result1 = x;  // result1 has the type int
        decltype(y) result2 = y;  // result2 has the type double

        // Using decltype with an expression
        int a = 10;
        int b = 20;
        decltype(a + b) sum = a + b;  // sum has the type int

        return 0;
    }
    \n
    • \n

  • friend
    In C++, the friend keyword is used to grant non-member functions or other classes access to the private and protected members of a class. When a function or class is declared as a friend of another class, it is allowed to access private and protected members of that class as if it were a member of that class.

    \n
    • \n
\n"},{"title":"cpp std","date":"2023-11-21T03:35:20.000Z","_content":"\n\n# snprintf\n`snprintf` is a function that formats a string and writes the resulting characters to a character array.\n```cpp\nint snprintf(char *str, size_t size, const char *format, ...);\n```\n- `str`: A pointer to the destination buffer where the resulting string is stored.\n- `size`: The size of the destination buffer (including the null-terminating character).\n- `format`: A format string that specifies the desired format of the output.\n- `...`: Additional arguments corresponding to the placeholders in the format string.\n```cpp\nint main() {\n size_t n = std::snprintf(nullptr, 0, \"%s\", \"hello, world\");\n std::cout << n << std::endl; // print 12\n return 0;\n}\n```\n\n# string\n## `std::string(n, '\\0')` \ncreates a string with a length of `n` and fills it with null characters ('\\0').\n```cpp\nsize_t n = 10;\nstd::string myString(n, '\\0');\n```\n## `strerror_r` \nis a function that is commonly used for thread-safe retrieval of error messages. The purpose of this function is to convert an error number (an integer typically set by a system call or library function to indicate an error) into a human-readable error message\n```cpp\n#include \n\nint strerror_r(int errnum, char *buf, size_t buflen);\n```\n- `errnum`: The error number for which you want to retrieve the error message.\n- `buf`: A pointer to the buffer where the error message will be stored.\n- `buflen`: The size of the buffer pointed to by buf.\n\n## others\n- `strncpy`: Copy no more than N characters of SRC to DEST.\n- `std::strlen`: Return the length of String.\n- `std::strstr` is a standard C library function that is used to find the first occurrence of a substring within a string. The function returns a pointer to the first occurrence of the substring in the given string, or NULL if the substring is not found.","source":"_posts/cpp/std.md","raw":"---\ntitle: cpp std\ndate: 2023-11-21 11:35:20\ntags: [cpp]\n---\n\n\n# snprintf\n`snprintf` is a function that formats a string and writes the resulting characters to a character array.\n```cpp\nint snprintf(char *str, size_t size, const char *format, ...);\n```\n- `str`: A pointer to the destination buffer where the resulting string is stored.\n- `size`: The size of the destination buffer (including the null-terminating character).\n- `format`: A format string that specifies the desired format of the output.\n- `...`: Additional arguments corresponding to the placeholders in the format string.\n```cpp\nint main() {\n size_t n = std::snprintf(nullptr, 0, \"%s\", \"hello, world\");\n std::cout << n << std::endl; // print 12\n return 0;\n}\n```\n\n# string\n## `std::string(n, '\\0')` \ncreates a string with a length of `n` and fills it with null characters ('\\0').\n```cpp\nsize_t n = 10;\nstd::string myString(n, '\\0');\n```\n## `strerror_r` \nis a function that is commonly used for thread-safe retrieval of error messages. The purpose of this function is to convert an error number (an integer typically set by a system call or library function to indicate an error) into a human-readable error message\n```cpp\n#include \n\nint strerror_r(int errnum, char *buf, size_t buflen);\n```\n- `errnum`: The error number for which you want to retrieve the error message.\n- `buf`: A pointer to the buffer where the error message will be stored.\n- `buflen`: The size of the buffer pointed to by buf.\n\n## others\n- `strncpy`: Copy no more than N characters of SRC to DEST.\n- `std::strlen`: Return the length of String.\n- `std::strstr` is a standard C library function that is used to find the first occurrence of a substring within a string. The function returns a pointer to the first occurrence of the substring in the given string, or NULL if the substring is not found.","slug":"cpp/std","published":1,"updated":"2023-11-21T13:05:50.461Z","_id":"clp8cm0w60004l57uckd04dmz","comments":1,"layout":"post","photos":[],"link":"","content":"

snprintf

snprintf is a function that formats a string and writes the resulting characters to a character array.

\n
1
int snprintf(char *str, size_t size, const char *format, ...);
\n
    \n
  • str: A pointer to the destination buffer where the resulting string is stored.
    • \n
  • size: The size of the destination buffer (including the null-terminating character).
    • \n
  • format: A format string that specifies the desired format of the output.
    • \n
  • ...: Additional arguments corresponding to the placeholders in the format string.
    1
    2
    3
    4
    5
    int main() {
        size_t n = std::snprintf(nullptr, 0, "%s", "hello, world");
        std::cout << n << std::endl;  // print 12
        return 0;
    }
    • \n
\n

string

std::string(n, '\\0')

creates a string with a length of n and fills it with null characters (‘\\0’).

\n
1
2
size_t n = 10;
std::string myString(n, '\\0');
\n

strerror_r

is a function that is commonly used for thread-safe retrieval of error messages. The purpose of this function is to convert an error number (an integer typically set by a system call or library function to indicate an error) into a human-readable error message

\n
1
2
3
#include <cstring>

int strerror_r(int errnum, char *buf, size_t buflen);
\n
    \n
  • errnum: The error number for which you want to retrieve the error message.
    • \n
  • buf: A pointer to the buffer where the error message will be stored.
    • \n
  • buflen: The size of the buffer pointed to by buf.
    • \n
\n

others

    \n
  • strncpy: Copy no more than N characters of SRC to DEST.
    • \n
  • std::strlen: Return the length of String.
    • \n
  • std::strstr is a standard C library function that is used to find the first occurrence of a substring within a string. The function returns a pointer to the first occurrence of the substring in the given string, or NULL if the substring is not found.
    • \n
\n","site":{"data":{}},"excerpt":"","more":"

snprintf

snprintf is a function that formats a string and writes the resulting characters to a character array.

\n
1
int snprintf(char *str, size_t size, const char *format, ...);
\n
    \n
  • str: A pointer to the destination buffer where the resulting string is stored.
    • \n
  • size: The size of the destination buffer (including the null-terminating character).
    • \n
  • format: A format string that specifies the desired format of the output.
    • \n
  • ...: Additional arguments corresponding to the placeholders in the format string.
    1
    2
    3
    4
    5
    int main() {
        size_t n = std::snprintf(nullptr, 0, "%s", "hello, world");
        std::cout << n << std::endl;  // print 12
        return 0;
    }
    • \n
\n

string

std::string(n, '\\0')

creates a string with a length of n and fills it with null characters (‘\\0’).

\n
1
2
size_t n = 10;
std::string myString(n, '\\0');
\n

strerror_r

is a function that is commonly used for thread-safe retrieval of error messages. The purpose of this function is to convert an error number (an integer typically set by a system call or library function to indicate an error) into a human-readable error message

\n
1
2
3
#include <cstring>

int strerror_r(int errnum, char *buf, size_t buflen);
\n
    \n
  • errnum: The error number for which you want to retrieve the error message.
    • \n
  • buf: A pointer to the buffer where the error message will be stored.
    • \n
  • buflen: The size of the buffer pointed to by buf.
    • \n
\n

others

    \n
  • strncpy: Copy no more than N characters of SRC to DEST.
    • \n
  • std::strlen: Return the length of String.
    • \n
  • std::strstr is a standard C library function that is used to find the first occurrence of a substring within a string. The function returns a pointer to the first occurrence of the substring in the given string, or NULL if the substring is not found.
    • \n
\n"},{"title":"cpp types","date":"2023-11-15T03:35:20.000Z","_content":"\n## Conversion Operator\n```cpp\nclass MyIntegerWrapper {\nprivate:\n int value;\n\npublic:\n // Constructor\n MyIntegerWrapper(int val) : value(val) {}\n\n // Conversion operator to int\n operator int() const {\n return value;\n }\n};\n\nint main() {\n MyIntegerWrapper myObj(42);\n\n // Implicit conversion using the conversion operator\n int intValue = myObj;\n std::cout << \"Implicit Conversion: \" << intValue << std::endl;\n\n // Explicit conversion using static_cast\n double doubleValue = static_cast(myObj);\n std::cout << \"Explicit Conversion: \" << doubleValue << std::endl;\n\n return 0;\n}\n```","source":"_posts/cpp/types.md","raw":"---\ntitle: cpp types\ndate: 2023-11-15 11:35:20\ntags: [cpp]\n---\n\n## Conversion Operator\n```cpp\nclass MyIntegerWrapper {\nprivate:\n int value;\n\npublic:\n // Constructor\n MyIntegerWrapper(int val) : value(val) {}\n\n // Conversion operator to int\n operator int() const {\n return value;\n }\n};\n\nint main() {\n MyIntegerWrapper myObj(42);\n\n // Implicit conversion using the conversion operator\n int intValue = myObj;\n std::cout << \"Implicit Conversion: \" << intValue << std::endl;\n\n // Explicit conversion using static_cast\n double doubleValue = static_cast(myObj);\n std::cout << \"Explicit Conversion: \" << doubleValue << std::endl;\n\n return 0;\n}\n```","slug":"cpp/types","published":1,"updated":"2023-11-21T12:26:56.654Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clp8cm0w60006l57u0fvme5s2","content":"

Conversion Operator

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
class MyIntegerWrapper {
private:
    int value;

public:
    // Constructor
    MyIntegerWrapper(int val) : value(val) {}

    // Conversion operator to int
    operator int() const {
        return value;
    }
};

int main() {
    MyIntegerWrapper myObj(42);

    // Implicit conversion using the conversion operator
    int intValue = myObj;
    std::cout << "Implicit Conversion: " << intValue << std::endl;

    // Explicit conversion using static_cast
    double doubleValue = static_cast<double>(myObj);
    std::cout << "Explicit Conversion: " << doubleValue << std::endl;

    return 0;
}
","site":{"data":{}},"excerpt":"","more":"

Conversion Operator

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
class MyIntegerWrapper {
private:
    int value;

public:
    // Constructor
    MyIntegerWrapper(int val) : value(val) {}

    // Conversion operator to int
    operator int() const {
        return value;
    }
};

int main() {
    MyIntegerWrapper myObj(42);

    // Implicit conversion using the conversion operator
    int intValue = myObj;
    std::cout << "Implicit Conversion: " << intValue << std::endl;

    // Explicit conversion using static_cast
    double doubleValue = static_cast<double>(myObj);
    std::cout << "Explicit Conversion: " << doubleValue << std::endl;

    return 0;
}
"},{"title":"key concepts in cuda","date":"2023-11-22T06:47:28.000Z","_content":"\n## key concepts\n- **Warps**: In CUDA, a warp is a fundamental execution unit in the GPU. It represents a group of threads that are executed together in a SIMD (Single Instruction, Multiple Data) fashion on a single GPU core. In CUDA, each group of 32 consecutive threads is called a warp. A Warp is the primary unit of execution in an SM. Once a thread block is allocated to an SM, it will be further divided into a set of warps for execution.\n- **Stream Multiprocessor** (SM)\nThe CUDA architecture is built around a scalable array of multithreaded Streaming Multiprocessors (SMs). When a CUDA program on the host CPU invokes a kernel grid, the blocks of the grid are enumerated and distributed to multiprocessors with available execution capacity. The threads of a thread block execute concurrently on one multiprocessor, and multiple thread blocks can execute concurrently on one multiprocessor. As thread blocks terminate, new blocks are launched on the vacated multiprocessors\nEach SM contains 8 CUDA cores, and at any one time they're executing a single warp of 32 threads - so it takes 4 clock cycles to issue a single instruction for the whole warp. You can assume that threads in any given warp execute in lock-step, but to synchronise across warps, you need to use `__syncthreads()`\n\n## memory\n- shared_memory: https://developer.nvidia.com/blog/using-shared-memory-cuda-cc/\n- [Page-Locked Host Memory](https://leimao.github.io/blog/Page-Locked-Host-Memory-Data-Transfer/)\n\n## stream\nA sequence of operations that execute in issue-order on the GPU\n- CUDA operations in different streams may run concurrently\n- CUDA operations from different streams may be interleaved\n![cuda stream example](images/cuda/cuda_stream_example.png)\n\n## event\nevents are synchronization mechanisms used to measure and manage time, particularly for timing purposes and performance profiling. \n- event creation\n```cpp\n#include \n\ncudaEvent_t startEvent, stopEvent;\ncudaEventCreate(&startEvent);\ncudaEventCreate(&stopEvent);\n```\n- record event\n```cpp\ncudaEventRecord(startEvent, 0);\n// ... GPU code to be timed ...\ncudaEventRecord(stopEvent, 0);\n```\nHere, `cudaEventRecord` records the current time in the events startEvent and stopEvent. The second argument, `0`, specifies the stream in which to record the event (0 means the default stream).\n- synchonize events\n```cpp\ncudaEventSynchronize(stopEvent);\n```\nThis function call makes the CPU wait until the GPU has completed all tasks associated with the stopEvent.\n- Calculate Elapsed Time\n```cpp\nfloat milliseconds = 0;\ncudaEventElapsedTime(&milliseconds, startEvent, stopEvent);\n```\n- destroy event\n```cpp\ncudaEventDestroy(startEvent);\ncudaEventDestroy(stopEvent);\n```\n## keywords\n- __global__ functions that are executed on the GPU, called from CPU\n- __device__ functions that are executed on the GPU, called within GPU\n- __shared__ \n- __align__ is a compiler directive that can be used to specify alignment requirements for variables in device code.\n\n- `__launch_bounds__`\nthe __launch_bounds__ attribute is used to specify launch bounds for a kernel function. The launch bounds control the number of threads per block and the maximum number of blocks per multiprocessor for a specific kernel.\n```cpp\n__launch_bounds__(maxThreadsPerBlock, minBlocksPerMultiprocessor)\n```\n\n## macros\n- __CUDACC__ is a predefined macro in CUDA C/C++ that is used to determine whether the code is being compiled by the NVIDIA CUDA C/C++ compiler (nvcc). \n- __NVCC__ is a predefined macro that is automatically defined by the NVIDIA CUDA compiler (nvcc). It allows you to conditionally compile code based on whether the code is being processed by the CUDA compiler or a regular C++ compiler\n\n## usage\nthe triple-chevron notation `<<<...>>>` is used to specify the execution configuration of a CUDA kernel launch. \n```cpp\nkernel<<>>(...);\n```\n- `gridDim`: Specifies the dimensions of the grid (number of blocks). It can be a 1D, 2D, or 3D configuration, specified as (gridDimX, gridDimY, gridDimZ).\n\n- `blockDim`: Specifies the dimensions of each block (number of threads per block). It can also be a 1D, 2D, or 3D configuration, specified as (blockDimX, blockDimY, blockDimZ).\n\n- `sharedMem`: Optional parameter specifying the dynamic shared memory per block in bytes. If not specified, it defaults to 0.\n\n- `stream`: Optional parameter specifying the CUDA stream in which the kernel should be executed. If not specified, it defaults to the default stream (0).","source":"_posts/cuda/concepts_and_keywords.md","raw":"---\ntitle: key concepts in cuda\ndate: 2023-11-22 14:47:28\ntags: [cuda]\n---\n\n## key concepts\n- **Warps**: In CUDA, a warp is a fundamental execution unit in the GPU. It represents a group of threads that are executed together in a SIMD (Single Instruction, Multiple Data) fashion on a single GPU core. In CUDA, each group of 32 consecutive threads is called a warp. A Warp is the primary unit of execution in an SM. Once a thread block is allocated to an SM, it will be further divided into a set of warps for execution.\n- **Stream Multiprocessor** (SM)\nThe CUDA architecture is built around a scalable array of multithreaded Streaming Multiprocessors (SMs). When a CUDA program on the host CPU invokes a kernel grid, the blocks of the grid are enumerated and distributed to multiprocessors with available execution capacity. The threads of a thread block execute concurrently on one multiprocessor, and multiple thread blocks can execute concurrently on one multiprocessor. As thread blocks terminate, new blocks are launched on the vacated multiprocessors\nEach SM contains 8 CUDA cores, and at any one time they're executing a single warp of 32 threads - so it takes 4 clock cycles to issue a single instruction for the whole warp. You can assume that threads in any given warp execute in lock-step, but to synchronise across warps, you need to use `__syncthreads()`\n\n## memory\n- shared_memory: https://developer.nvidia.com/blog/using-shared-memory-cuda-cc/\n- [Page-Locked Host Memory](https://leimao.github.io/blog/Page-Locked-Host-Memory-Data-Transfer/)\n\n## stream\nA sequence of operations that execute in issue-order on the GPU\n- CUDA operations in different streams may run concurrently\n- CUDA operations from different streams may be interleaved\n![cuda stream example](images/cuda/cuda_stream_example.png)\n\n## event\nevents are synchronization mechanisms used to measure and manage time, particularly for timing purposes and performance profiling. \n- event creation\n```cpp\n#include \n\ncudaEvent_t startEvent, stopEvent;\ncudaEventCreate(&startEvent);\ncudaEventCreate(&stopEvent);\n```\n- record event\n```cpp\ncudaEventRecord(startEvent, 0);\n// ... GPU code to be timed ...\ncudaEventRecord(stopEvent, 0);\n```\nHere, `cudaEventRecord` records the current time in the events startEvent and stopEvent. The second argument, `0`, specifies the stream in which to record the event (0 means the default stream).\n- synchonize events\n```cpp\ncudaEventSynchronize(stopEvent);\n```\nThis function call makes the CPU wait until the GPU has completed all tasks associated with the stopEvent.\n- Calculate Elapsed Time\n```cpp\nfloat milliseconds = 0;\ncudaEventElapsedTime(&milliseconds, startEvent, stopEvent);\n```\n- destroy event\n```cpp\ncudaEventDestroy(startEvent);\ncudaEventDestroy(stopEvent);\n```\n## keywords\n- __global__ functions that are executed on the GPU, called from CPU\n- __device__ functions that are executed on the GPU, called within GPU\n- __shared__ \n- __align__ is a compiler directive that can be used to specify alignment requirements for variables in device code.\n\n- `__launch_bounds__`\nthe __launch_bounds__ attribute is used to specify launch bounds for a kernel function. The launch bounds control the number of threads per block and the maximum number of blocks per multiprocessor for a specific kernel.\n```cpp\n__launch_bounds__(maxThreadsPerBlock, minBlocksPerMultiprocessor)\n```\n\n## macros\n- __CUDACC__ is a predefined macro in CUDA C/C++ that is used to determine whether the code is being compiled by the NVIDIA CUDA C/C++ compiler (nvcc). \n- __NVCC__ is a predefined macro that is automatically defined by the NVIDIA CUDA compiler (nvcc). It allows you to conditionally compile code based on whether the code is being processed by the CUDA compiler or a regular C++ compiler\n\n## usage\nthe triple-chevron notation `<<<...>>>` is used to specify the execution configuration of a CUDA kernel launch. \n```cpp\nkernel<<>>(...);\n```\n- `gridDim`: Specifies the dimensions of the grid (number of blocks). It can be a 1D, 2D, or 3D configuration, specified as (gridDimX, gridDimY, gridDimZ).\n\n- `blockDim`: Specifies the dimensions of each block (number of threads per block). It can also be a 1D, 2D, or 3D configuration, specified as (blockDimX, blockDimY, blockDimZ).\n\n- `sharedMem`: Optional parameter specifying the dynamic shared memory per block in bytes. If not specified, it defaults to 0.\n\n- `stream`: Optional parameter specifying the CUDA stream in which the kernel should be executed. If not specified, it defaults to the default stream (0).","slug":"cuda/concepts_and_keywords","published":1,"updated":"2023-11-23T07:30:03.676Z","_id":"clp95py9p0000e37u3ivq1as7","comments":1,"layout":"post","photos":[],"link":"","content":"

key concepts

    \n
  • Warps: In CUDA, a warp is a fundamental execution unit in the GPU. It represents a group of threads that are executed together in a SIMD (Single Instruction, Multiple Data) fashion on a single GPU core. In CUDA, each group of 32 consecutive threads is called a warp. A Warp is the primary unit of execution in an SM. Once a thread block is allocated to an SM, it will be further divided into a set of warps for execution.
    • \n
  • Stream Multiprocessor (SM)
    The CUDA architecture is built around a scalable array of multithreaded Streaming Multiprocessors (SMs). When a CUDA program on the host CPU invokes a kernel grid, the blocks of the grid are enumerated and distributed to multiprocessors with available execution capacity. The threads of a thread block execute concurrently on one multiprocessor, and multiple thread blocks can execute concurrently on one multiprocessor. As thread blocks terminate, new blocks are launched on the vacated multiprocessors
    Each SM contains 8 CUDA cores, and at any one time they’re executing a single warp of 32 threads - so it takes 4 clock cycles to issue a single instruction for the whole warp. You can assume that threads in any given warp execute in lock-step, but to synchronise across warps, you need to use __syncthreads()
    • \n
\n

memory

\n

stream

A sequence of operations that execute in issue-order on the GPU

\n
    \n
  • CUDA operations in different streams may run concurrently
    • \n
  • CUDA operations from different streams may be interleaved
    \"cuda
    • \n
\n

event

events are synchronization mechanisms used to measure and manage time, particularly for timing purposes and performance profiling.

\n
    \n
  • event creation
    1
    2
    3
    4
    5
    #include <cuda_runtime.h>

    cudaEvent_t startEvent, stopEvent;
    cudaEventCreate(&startEvent);
    cudaEventCreate(&stopEvent);
    • \n
  • record event
    1
    2
    3
    cudaEventRecord(startEvent, 0);
    // ... GPU code to be timed ...
    cudaEventRecord(stopEvent, 0);
    \nHere, cudaEventRecord records the current time in the events startEvent and stopEvent. The second argument, 0, specifies the stream in which to record the event (0 means the default stream).
    • \n
  • synchonize events
    1
    cudaEventSynchronize(stopEvent);
    \nThis function call makes the CPU wait until the GPU has completed all tasks associated with the stopEvent.
    • \n
  • Calculate Elapsed Time
    1
    2
    float milliseconds = 0;
    cudaEventElapsedTime(&milliseconds, startEvent, stopEvent);
    • \n
  • destroy event
    1
    2
    cudaEventDestroy(startEvent);
    cudaEventDestroy(stopEvent);
    • \n
\n

keywords

    \n

  • global functions that are executed on the GPU, called from CPU

    \n
    • \n

  • device functions that are executed on the GPU, called within GPU

    \n
    • \n

  • shared

    \n
    • \n

  • align is a compiler directive that can be used to specify alignment requirements for variables in device code.

    \n
    • \n

  • __launch_bounds__
    the launch_bounds attribute is used to specify launch bounds for a kernel function. The launch bounds control the number of threads per block and the maximum number of blocks per multiprocessor for a specific kernel.

    \n
    1
    __launch_bounds__(maxThreadsPerBlock, minBlocksPerMultiprocessor)
    • \n
\n

macros

    \n
  • CUDACC is a predefined macro in CUDA C/C++ that is used to determine whether the code is being compiled by the NVIDIA CUDA C/C++ compiler (nvcc).
    • \n
  • NVCC is a predefined macro that is automatically defined by the NVIDIA CUDA compiler (nvcc). It allows you to conditionally compile code based on whether the code is being processed by the CUDA compiler or a regular C++ compiler
    • \n
\n

usage

the triple-chevron notation <<<...>>> is used to specify the execution configuration of a CUDA kernel launch.

\n
1
kernel<<<gridDim, blockDim, sharedMem, stream>>>(...);
\n
    \n

  • gridDim: Specifies the dimensions of the grid (number of blocks). It can be a 1D, 2D, or 3D configuration, specified as (gridDimX, gridDimY, gridDimZ).

    \n
    • \n

  • blockDim: Specifies the dimensions of each block (number of threads per block). It can also be a 1D, 2D, or 3D configuration, specified as (blockDimX, blockDimY, blockDimZ).

    \n
    • \n

  • sharedMem: Optional parameter specifying the dynamic shared memory per block in bytes. If not specified, it defaults to 0.

    \n
    • \n

  • stream: Optional parameter specifying the CUDA stream in which the kernel should be executed. If not specified, it defaults to the default stream (0).

    \n
    • \n
\n","site":{"data":{}},"excerpt":"","more":"

key concepts

    \n
  • Warps: In CUDA, a warp is a fundamental execution unit in the GPU. It represents a group of threads that are executed together in a SIMD (Single Instruction, Multiple Data) fashion on a single GPU core. In CUDA, each group of 32 consecutive threads is called a warp. A Warp is the primary unit of execution in an SM. Once a thread block is allocated to an SM, it will be further divided into a set of warps for execution.
    • \n
  • Stream Multiprocessor (SM)
    The CUDA architecture is built around a scalable array of multithreaded Streaming Multiprocessors (SMs). When a CUDA program on the host CPU invokes a kernel grid, the blocks of the grid are enumerated and distributed to multiprocessors with available execution capacity. The threads of a thread block execute concurrently on one multiprocessor, and multiple thread blocks can execute concurrently on one multiprocessor. As thread blocks terminate, new blocks are launched on the vacated multiprocessors
    Each SM contains 8 CUDA cores, and at any one time they’re executing a single warp of 32 threads - so it takes 4 clock cycles to issue a single instruction for the whole warp. You can assume that threads in any given warp execute in lock-step, but to synchronise across warps, you need to use __syncthreads()
    • \n
\n

memory

\n

stream

A sequence of operations that execute in issue-order on the GPU

\n
    \n
  • CUDA operations in different streams may run concurrently
    • \n
  • CUDA operations from different streams may be interleaved
    \"cuda
    • \n
\n

event

events are synchronization mechanisms used to measure and manage time, particularly for timing purposes and performance profiling.

\n
    \n
  • event creation
    1
    2
    3
    4
    5
    #include <cuda_runtime.h>

    cudaEvent_t startEvent, stopEvent;
    cudaEventCreate(&startEvent);
    cudaEventCreate(&stopEvent);
    • \n
  • record event
    1
    2
    3
    cudaEventRecord(startEvent, 0);
    // ... GPU code to be timed ...
    cudaEventRecord(stopEvent, 0);
    \nHere, cudaEventRecord records the current time in the events startEvent and stopEvent. The second argument, 0, specifies the stream in which to record the event (0 means the default stream).
    • \n
  • synchonize events
    1
    cudaEventSynchronize(stopEvent);
    \nThis function call makes the CPU wait until the GPU has completed all tasks associated with the stopEvent.
    • \n
  • Calculate Elapsed Time
    1
    2
    float milliseconds = 0;
    cudaEventElapsedTime(&milliseconds, startEvent, stopEvent);
    • \n
  • destroy event
    1
    2
    cudaEventDestroy(startEvent);
    cudaEventDestroy(stopEvent);
    • \n
\n

keywords

    \n

  • global functions that are executed on the GPU, called from CPU

    \n
    • \n

  • device functions that are executed on the GPU, called within GPU

    \n
    • \n

  • shared

    \n
    • \n

  • align is a compiler directive that can be used to specify alignment requirements for variables in device code.

    \n
    • \n

  • __launch_bounds__
    the launch_bounds attribute is used to specify launch bounds for a kernel function. The launch bounds control the number of threads per block and the maximum number of blocks per multiprocessor for a specific kernel.

    \n
    1
    __launch_bounds__(maxThreadsPerBlock, minBlocksPerMultiprocessor)
    • \n
\n

macros

    \n
  • CUDACC is a predefined macro in CUDA C/C++ that is used to determine whether the code is being compiled by the NVIDIA CUDA C/C++ compiler (nvcc).
    • \n
  • NVCC is a predefined macro that is automatically defined by the NVIDIA CUDA compiler (nvcc). It allows you to conditionally compile code based on whether the code is being processed by the CUDA compiler or a regular C++ compiler
    • \n
\n

usage

the triple-chevron notation <<<...>>> is used to specify the execution configuration of a CUDA kernel launch.

\n
1
kernel<<<gridDim, blockDim, sharedMem, stream>>>(...);
\n
    \n

  • gridDim: Specifies the dimensions of the grid (number of blocks). It can be a 1D, 2D, or 3D configuration, specified as (gridDimX, gridDimY, gridDimZ).

    \n
    • \n

  • blockDim: Specifies the dimensions of each block (number of threads per block). It can also be a 1D, 2D, or 3D configuration, specified as (blockDimX, blockDimY, blockDimZ).

    \n
    • \n

  • sharedMem: Optional parameter specifying the dynamic shared memory per block in bytes. If not specified, it defaults to 0.

    \n
    • \n

  • stream: Optional parameter specifying the CUDA stream in which the kernel should be executed. If not specified, it defaults to the default stream (0).

    \n
    • \n
\n"},{"title":"cuda standard api & tools","date":"2023-11-22T06:47:28.000Z","_content":"\n# API\n- `__syncthreads()`: A thread’s execution can only proceed past a __syncthreads() after all threads in its block have executed the __syncthreads()\n- `__syncwarp()` is an intrinsic function that provides a warp-level synchronization point within a warp. A warp is a group of threads in CUDA that execute instructions in lockstep.\n\n\n# Tool\n## cmd\n- nvidia-smi\n\n\n\n## references\n- [cuda runtime api](https://docs.nvidia.com/cuda/cuda-runtime-api/index.html)","source":"_posts/cuda/cuda_api_and_tools.md","raw":"---\ntitle: cuda standard api & tools\ndate: 2023-11-22 14:47:28\ntags: [cuda]\n---\n\n# API\n- `__syncthreads()`: A thread’s execution can only proceed past a __syncthreads() after all threads in its block have executed the __syncthreads()\n- `__syncwarp()` is an intrinsic function that provides a warp-level synchronization point within a warp. A warp is a group of threads in CUDA that execute instructions in lockstep.\n\n\n# Tool\n## cmd\n- nvidia-smi\n\n\n\n## references\n- [cuda runtime api](https://docs.nvidia.com/cuda/cuda-runtime-api/index.html)","slug":"cuda/cuda_api_and_tools","published":1,"updated":"2023-11-23T07:41:25.289Z","_id":"clp95py9w0003e37uhwx0baye","comments":1,"layout":"post","photos":[],"link":"","content":"

API

    \n
  • __syncthreads(): A thread’s execution can only proceed past a __syncthreads() after all threads in its block have executed the __syncthreads()
    • \n
  • __syncwarp() is an intrinsic function that provides a warp-level synchronization point within a warp. A warp is a group of threads in CUDA that execute instructions in lockstep.
    • \n
\n

Tool

cmd

    \n
  • nvidia-smi
    • \n
\n

references

\n","site":{"data":{}},"excerpt":"","more":"

API

    \n
  • __syncthreads(): A thread’s execution can only proceed past a __syncthreads() after all threads in its block have executed the __syncthreads()
    • \n
  • __syncwarp() is an intrinsic function that provides a warp-level synchronization point within a warp. A warp is a group of threads in CUDA that execute instructions in lockstep.
    • \n
\n

Tool

cmd

    \n
  • nvidia-smi
    • \n
\n

references

\n"},{"title":"cryptography arithmetic tricks","date":"2023-11-22T06:47:28.000Z","_content":"\n\n\n\n- get the next m*SIZE\n```cpp\n# define WARP_SZ 32\nsize_t n = (nelems+WARP_SZ-1) & ((size_t)0-WARP_SZ)\n```\n0-WARP_SIZE = 0xffffffffffffffe0\neo = 0b11100000\nit will select multiples of WARP_SZ\nnelems+WARP_SZ-1 will counter for the removed numbers by 00000\n\nn will be [32, 64, 96, ...]","source":"_posts/arithmatic/tricks.md","raw":"---\ntitle: cryptography arithmetic tricks\ndate: 2023-11-22 14:47:28\ntags: [arithmatic]\n---\n\n\n\n\n- get the next m*SIZE\n```cpp\n# define WARP_SZ 32\nsize_t n = (nelems+WARP_SZ-1) & ((size_t)0-WARP_SZ)\n```\n0-WARP_SIZE = 0xffffffffffffffe0\neo = 0b11100000\nit will select multiples of WARP_SZ\nnelems+WARP_SZ-1 will counter for the removed numbers by 00000\n\nn will be [32, 64, 96, ...]","slug":"arithmatic/tricks","published":1,"updated":"2023-12-08T06:18:33.813Z","_id":"clp96plbb0006e37uei4tcdkw","comments":1,"layout":"post","photos":[],"link":"","content":"\n\n\n
    \n
  • get the next m*SIZE
    1
    2
    # define WARP_SZ 32
    size_t n = (nelems+WARP_SZ-1) & ((size_t)0-WARP_SZ)
    \n0-WARP_SIZE = 0xffffffffffffffe0
    eo = 0b11100000
    it will select multiples of WARP_SZ
    nelems+WARP_SZ-1 will counter for the removed numbers by 00000
    • \n
\n

n will be [32, 64, 96, …]

\n","site":{"data":{}},"excerpt":"","more":"\n\n\n
    \n
  • get the next m*SIZE
    1
    2
    # define WARP_SZ 32
    size_t n = (nelems+WARP_SZ-1) & ((size_t)0-WARP_SZ)
    \n0-WARP_SIZE = 0xffffffffffffffe0
    eo = 0b11100000
    it will select multiples of WARP_SZ
    nelems+WARP_SZ-1 will counter for the removed numbers by 00000
    • \n
\n

n will be [32, 64, 96, …]

\n"},{"title":"classic implementations of arithematic of multi precision big integers","date":"2023-11-22T06:47:28.000Z","_content":"\n\n\n\n\n## multiple-precision integer arithmetic\nIf \\\\(b ≥ 2\\\\) is an integer, then any positive integer \\\\(a\\\\) can be expressed uniquely as \\\\(a = a_n \\cdot b^n+a_{n−1}\\cdot b^{n−1}+···+a_1 \\cdot b+a_0\\\\), where \\\\(a_i\\\\) is an integer with \\\\(0≤a_i \n\n\n\n## multiple-precision integer arithmetic\nIf \\\\(b ≥ 2\\\\) is an integer, then any positive integer \\\\(a\\\\) can be expressed uniquely as \\\\(a = a_n \\cdot b^n+a_{n−1}\\cdot b^{n−1}+···+a_1 \\cdot b+a_0\\\\), where \\\\(a_i\\\\) is an integer with \\\\(0≤a_i \n\n\n\n

multiple-precision integer arithmetic

If \\(b ≥ 2\\) is an integer, then any positive integer \\(a\\) can be expressed uniquely as \\(a = a_n \\cdot b^n+a_{n−1}\\cdot b^{n−1}+···+a_1 \\cdot b+a_0\\), where \\(a_i\\) is an integer with \\(0≤a_i <b \\) for \\(0≤ i ≤ n \\), and \\(a_n \\ne 0\\).

\n

The representation of a positive integer \\(a\\) as a sum of multiples of powers of \\(b\\), as given above, is called the base \\(b\\) or \\(radix b\\) representation of \\(a\\), which is usually written as
\\(a = (a_n a_{n−1} ···a_1 a_0)_b\\)

\n

If \\( (a_n a_{n−1} ···a_1 a_0)_b\\) is the base b representation of \\(a\\) and \\(a_n \\ne 0\\), then the precision
or length of a is n + 1. If n = 0, then a is called a single-precision integer; otherwise, a is a multiple-precision integer.

\n

The division algorithm for integers provides an efficient method for determining the base b representation of a non-negative integer
\"radix_b_repesentation\"

\n

if \\( (a_{n} a_{n−1} ···a_1 a_0)_b \\) is the base b representation of a and k is a positive integer, then

\n

\\( (u_{l} u_{l−1} ···u_1 u_0)_{b^k} \\) is the base \\( b^k \\) representation of a, where \\( l =\\lceil (n+1)/k \\rceil -1 \\),

\n

\\( u_i = \\sum_{j=0}^{k-1} a_{ik+j}b^j \\) for \\( 0 \\le i \\le l-1 \\), and
\\( u_l = \\sum_{j=0}^{n-lk} a_{lk+j}b^j\\)

\n

Representing negative numbers

complement representation.

\n

addition and subtraction

\"multiple_precision_addition\"
subraction is quite similar to addition. (omitted here)

\n

multiplication

Let x and y be integers expressed in radix b representation:
\\( x = (x_n x_{n−1} ··· x_1 x_0)_b \\) and

\n

\\( y = (y_t y_{t−1} ··· y_1 y_0)_b \\). The product x · y will have at most (n + t + 2) base b digits.

\n

modular reduction

Barrett

Mongtgomery

references

\n","site":{"data":{}},"excerpt":"","more":"\n\n\n\n

multiple-precision integer arithmetic

If \\(b ≥ 2\\) is an integer, then any positive integer \\(a\\) can be expressed uniquely as \\(a = a_n \\cdot b^n+a_{n−1}\\cdot b^{n−1}+···+a_1 \\cdot b+a_0\\), where \\(a_i\\) is an integer with \\(0≤a_i <b \\) for \\(0≤ i ≤ n \\), and \\(a_n \\ne 0\\).

\n

The representation of a positive integer \\(a\\) as a sum of multiples of powers of \\(b\\), as given above, is called the base \\(b\\) or \\(radix b\\) representation of \\(a\\), which is usually written as
\\(a = (a_n a_{n−1} ···a_1 a_0)_b\\)

\n

If \\( (a_n a_{n−1} ···a_1 a_0)_b\\) is the base b representation of \\(a\\) and \\(a_n \\ne 0\\), then the precision
or length of a is n + 1. If n = 0, then a is called a single-precision integer; otherwise, a is a multiple-precision integer.

\n

The division algorithm for integers provides an efficient method for determining the base b representation of a non-negative integer
\"radix_b_repesentation\"

\n

if \\( (a_{n} a_{n−1} ···a_1 a_0)_b \\) is the base b representation of a and k is a positive integer, then

\n

\\( (u_{l} u_{l−1} ···u_1 u_0)_{b^k} \\) is the base \\( b^k \\) representation of a, where \\( l =\\lceil (n+1)/k \\rceil -1 \\),

\n

\\( u_i = \\sum_{j=0}^{k-1} a_{ik+j}b^j \\) for \\( 0 \\le i \\le l-1 \\), and
\\( u_l = \\sum_{j=0}^{n-lk} a_{lk+j}b^j\\)

\n

Representing negative numbers

complement representation.

\n

addition and subtraction

\"multiple_precision_addition\"
subraction is quite similar to addition. (omitted here)

\n

multiplication

Let x and y be integers expressed in radix b representation:
\\( x = (x_n x_{n−1} ··· x_1 x_0)_b \\) and

\n

\\( y = (y_t y_{t−1} ··· y_1 y_0)_b \\). The product x · y will have at most (n + t + 2) base b digits.

\n

modular reduction

Barrett

Mongtgomery

references

\n"},{"title":"montgomery multiplication","date":"2023-12-07T11:10:13.000Z","_content":"\n\n\n\n## montgomery space\nMontgomery multiplication works by first transforming the multipliers into Montgomery space, where modular multiplication can be performed cheaply, and then transforming them back when their actual values are needed. Unlike general integer division methods, Montgomery multiplication is not efficient for performing just one modular reduction and only becomes worthwhile when there is a chain of modular operations. \n\nThe space is defined by the modulo `n` and a positive integer `r ≥ n` coprime to `n`. The algorithm involves modulo and division by `r`, so in practice, `r` is chosen to be `2^32` or `2^64`, so that these operations can be done with a right-shift and a bitwise `AND` respectively.\n\n\nDefinition. the representative \\\\(\\bar{x}\\\\) of a number \\\\(x\\\\) in the Montgomery space is defined as \n\\\\[ \\bar{x} = x \\cdot r \\mod n \\\\]\n\nComputing this transformation involves a multiplication and a modulo — an expensive operation that we wanted to optimize away in the first place — which is why we only use this method when the overhead of transforming numbers to and from the Montgomery space is worth it and not for general modular multiplication.\n\nInside the Montgomery space, addition, substraction, and checking for equality is performed as usual:\n\n\n\\\\[ x \\cdot r + y \\cdot r \\equiv (x+y) \\cdot r \\mod n \\\\]\n\nHowever, this is not the case for multiplication. Denoting multiplication in the Montgomery space as \\\\(∗\\\\) and the “normal” multiplication as \\\\(\\cdot\\\\), we expect the result to be:\n\\\\[ \\bar{x} * \\bar{y} = \\overline{x\\cdot y} = (x\\cdot y)\\cdot r \\mod n \\\\]\n\nBut the normal multiplication in the Montgomery space yields:\n\\\\[ \\bar{x} \\cdot \\bar{y}=(x\\cdot y)\\cdot r \\cdot r \\mod n \\\\]\n\nTherefore, the multiplication in the Montgomery space is defined as\n\\\\[ \\bar{x} * \\bar{y} = \\bar{x} \\cdot \\bar{y} \\cdot r^{-1} \\mod n \\\\]\n\nThis means that, after we normally multiply two numbers in the Montgomery space, we need to reduce the result by multiplying it by \\\\(r^{-1}\\\\) and taking the modulo — and there is an efficent way to do this particular operation.\n\n\n## Montgomery reduction\nassume that \\\\(r=2^{32}\\\\), the modulo \\\\(n\\\\) is 32-bit, and the number \\\\(x\\\\) we need to reduce is 64-bit(the product of two 32-bit numbers). The goal is to calculate \\\\(y=x\\cdot r^{-1} \\mod n\\\\).\n\nSince \\\\(r\\\\) is coprime with \\\\(n\\\\), we know that there are two numbers \\\\(r^{-1}\\\\) and \\\\(n'\\\\) in the \\\\([0, n)\\\\) range such that\n\\\\[ r \\cdot r^{-1} + n \\cdot n' = 1 \\\\]\nboth \\\\(r^{-1} and \\quad n'\\\\) can be computed using extended Euclidean algorithm **(\\\\(n' = n^{-1} \\mod r\\\\))**.\nUsing this identiy, we can express \\\\(r \\cdot r^{-1}\\\\) as \\\\(1-n\\cdot n'\\\\) and write \\\\(x \\cdot r^{-1}\\\\) as\n\n\\\\[\n \\displaylines{\n x\\cdot r^{-1} = \\frac{x \\cdot r \\cdot r^{-1}}{r} \\\\\\\\\n = \\frac{x \\cdot (1-n\\cdot n')}{r} \\\\\\\\\n = \\frac{x - x \\cdot n \\cdot n'}{r} \\\\\\\\\n = \\frac{x - x \\cdot n \\cdot n' + k\\cdot r \\cdot n}{r} \\quad (\\mod n) (for \\hspace{0.2cm}any\\hspace{0.2cm} integer\\hspace{0.2cm} k) \\\\\\\\\n = \\frac{x - (x\\cdot n' - k\\cdot r)\\cdot n}{r}\n}\n \\\\]\n\nNow, if we choose \\\\(k\\\\) to be \\\\(\\lfloor x\\cdot n'/r\\rfloor\\\\) (the upper 64 bits of \\\\(x\\cdot n'\\\\), giving \\\\(x\\\\) is 64 bits and \\\\(n'\\\\) is 32 bits ), it will cancel out, and \\\\(k\\cdot r - x\\cdot n'\\\\) will simply be equal to \\\\(x \\cdot n' \\mod r\\\\) (the lower 32 bits of \\\\(x\\cdot n'\\\\)), implying:\n\\\\[ x\\cdot r^{-1} = (x - (x\\cdot n' \\mod r )\\cdot n)/r\\\\]\n\nthe algorithm itself just evaluates this formula, performing two multiplications to calculate \\\\(q = x\\cdot n' \\mod r\\\\) and \\\\(m = q\\cdot n\\\\), and then subtracts it from \\\\(x\\\\) and right shifts the result to divide it by r.\n\nThe only remaining thing to handle is that the result may not be in the \\\\([0,n)\\\\) range; but since\n\\\\[x < n\\cdot n < r\\cdot n => x/r < n \\\\]\nand\n\\\\[m = q\\cdot n < r\\cdot n => m/r < n\\\\]\nit is guaranted that \n\\\\[ -n < (x-m)/r < n \\\\]\n\nTherefore, we can simply check if the result is negative and in that case, add \\\\(n\\\\) to it, giving the following algorithm:\n\n```cpp\ntypedef __uint32_t u32;\ntypedef __uint64_t u64;\n\nconst u32 n = 1e9 + 7, nr = inverse(n, 1ull << 32); // nr is n' in the above equations, r is 1<<32\n\nu32 reduce(u64 x) {\n u32 q = u32(x) * nr; // q = x * n' mod r\n u64 m = (u64) q * n; // m = q * n\n u32 y = (x - m) >> 32; // y = (x - m) / r\n return x < m ? y + n : y; // if y < 0, add n to make it be in the [0, n) range\n}\n```\nThis last check is relatively cheap, but it is still on the critical path. If we are fine with the result being in the \\\\([0,2\\cdot n−2]\\\\) range instead of \\\\([0, n)\\\\), we can remove it and add \\\\(n\\\\) to the result unconditionally:\n\n```cpp\nu32 reduce(u64 x) {\n u32 q = u32(x) * nr;\n u64 m = (u64) q * n;\n u32 y = (x - m) >> 32;\n return y + n\n}\n```\n\nWe can also move the `>> 32` operation one step earlier in the computation graph and compute \\\\(\\lfloor x/r \\rfloor - \\lfloor m/r \\rfloor\\\\) instead of \\\\( (x-m)/r\\\\). This is correct because the lower 32 bits of \\\\(x\\\\) and \\\\(m\\\\) are equal anyway since \n\\\\[ m = x\\cdot n' \\cdot n = x \\mod r\\\\] (mod r is same as taking the lower 32 bits).\n\nBut why would we voluntarily choose to perfom two right-shifts instead of just one? This is beneficial because for `((u64) q * n) >> 32` we need to do a 32-by-32 multiplication and take the upper 32 bits of the result (which the x86 mul instruction already writes in a separate register, so it doesn’t cost anything), and the other right-shift `x >> 32` is not on the critical path.\n```cpp\nu32 reduce(u64 x) {\n u32 q = u32(x) * nr;\n u32 m = ((u64) q * n) >> 32;\n return (x >> 32) + n - m;\n}\n```\nif to reduce a u128 number（u64 * u64). it is as below\n```cpp\ntypedef __uint128_t u128;\n\nu64 reduce(u128 x) const {\n u64 q = u64(x) * nr;\n u64 m = ((u128) q * n) >> 64;\n return (x >> 64) + n - m;\n}\n```\n\n# Faster Inverse\nMontgomery multiplication itself is fast, but it requires some precomputation:\n- inverting \\\\(n\\\\) modulo \\\\(r\\\\) to compute \\\\(n'\\\\)\n- transforming a number to the Montgomery space,\n- transforming a number from the Montgomery space.\n\nThe last operation is already efficiently performed with the reduce procedure we just implemented, but the first `inverting` can be slightly optimized.\nComputing the inverse \\\\(n' = n^{-1} \\mod r\\\\) can be done faster than with the extended Euclidean algorith by taking advantage of the fact that \\\\(r\\\\) is a power of two and using the following identity:\n\\\\[ a\\cdot x = 1 \\mod 2^k => a\\cdot x\\cdot (2-a\\cdot x) = 1 \\mod 2^{2k} \\\\]\nProof: \n \\\\[\n \\displaylines{\n a\\cdot x \\cdot (2-a\\cdot x) = 2\\cdot a\\cdot x - (a\\cdot x)^2 \\\\\\\\\n = 2\\cdot(1+m\\cdot 2^k) - (1+m\\cdot 2^k)^2 \\\\\\\\\n = 2 + 2\\cdot m\\cdot 2^k - 1-2\\cdot m \\cdot 2^k -m^2\\cdot 2^{2k} \\\\\\\\\n = 1-m^2\\cdot 2^{2k} \\\\\\\\\n = 1 \\mod 2^{2k}\n}\n \\\\]\n**note**\n- \\\\(a\\cdot x \\\\) coudl be represented by \\\\(1+m\\cdot 2^k\\\\) as \\\\(a\\cdot x = 1 \\mod 2^k\\\\)\n\n\nas for our case, \\\\(x\\\\) is \\\\(nr\\\\), \\\\(a\\\\) is \\\\(n\\\\). i.e \\\\(n\\cdot nr = 1 \\mod 2^k\\\\). Since \\\\(n \\\\) is an odd number, we can start with \\\\(nr=1\\\\) as the inverse of \\\\(n\\\\) modulo \\\\(2^1\\\\). then applying this identity one time, i.e \\\\(n\\cdot 1 \\cdot (2-n\\cdot 1) = 1 \\mod 2^2\\\\). Let \\\\(nr = 1\\cdot(2-n\\cdot 1) \\\\), we apply this identity again, and get \\\\(n\\cdot nr \\cdot (2-a\\cdot nr) = 1 \\mod 2^4\\\\). Following this approach, and apply this identity exactly \\\\(log_2^m\\\\) times, where \\\\(m\\\\) is the bit length of \\\\(r\\\\)\n\n## Complement Implementation\n```cpp\n\n\n```\n## references\n- https://en.algorithmica.org/hpc/number-theory/montgomery/\n- [csdn blog on montgomery reduction](https://blog.csdn.net/mutourend/article/details/95613967)\n- [nayuki montgomery reduction](https://www.nayuki.io/page/montgomery-reduction-algorithm)\n","source":"_posts/arithmatic/montgomery-multiplication.md","raw":"---\ntitle: montgomery multiplication\ndate: 2023-12-07 19:10:13\ntags: [number_theory]\n---\n\n\n\n\n## montgomery space\nMontgomery multiplication works by first transforming the multipliers into Montgomery space, where modular multiplication can be performed cheaply, and then transforming them back when their actual values are needed. Unlike general integer division methods, Montgomery multiplication is not efficient for performing just one modular reduction and only becomes worthwhile when there is a chain of modular operations. \n\nThe space is defined by the modulo `n` and a positive integer `r ≥ n` coprime to `n`. The algorithm involves modulo and division by `r`, so in practice, `r` is chosen to be `2^32` or `2^64`, so that these operations can be done with a right-shift and a bitwise `AND` respectively.\n\n\nDefinition. the representative \\\\(\\bar{x}\\\\) of a number \\\\(x\\\\) in the Montgomery space is defined as \n\\\\[ \\bar{x} = x \\cdot r \\mod n \\\\]\n\nComputing this transformation involves a multiplication and a modulo — an expensive operation that we wanted to optimize away in the first place — which is why we only use this method when the overhead of transforming numbers to and from the Montgomery space is worth it and not for general modular multiplication.\n\nInside the Montgomery space, addition, substraction, and checking for equality is performed as usual:\n\n\n\\\\[ x \\cdot r + y \\cdot r \\equiv (x+y) \\cdot r \\mod n \\\\]\n\nHowever, this is not the case for multiplication. Denoting multiplication in the Montgomery space as \\\\(∗\\\\) and the “normal” multiplication as \\\\(\\cdot\\\\), we expect the result to be:\n\\\\[ \\bar{x} * \\bar{y} = \\overline{x\\cdot y} = (x\\cdot y)\\cdot r \\mod n \\\\]\n\nBut the normal multiplication in the Montgomery space yields:\n\\\\[ \\bar{x} \\cdot \\bar{y}=(x\\cdot y)\\cdot r \\cdot r \\mod n \\\\]\n\nTherefore, the multiplication in the Montgomery space is defined as\n\\\\[ \\bar{x} * \\bar{y} = \\bar{x} \\cdot \\bar{y} \\cdot r^{-1} \\mod n \\\\]\n\nThis means that, after we normally multiply two numbers in the Montgomery space, we need to reduce the result by multiplying it by \\\\(r^{-1}\\\\) and taking the modulo — and there is an efficent way to do this particular operation.\n\n\n## Montgomery reduction\nassume that \\\\(r=2^{32}\\\\), the modulo \\\\(n\\\\) is 32-bit, and the number \\\\(x\\\\) we need to reduce is 64-bit(the product of two 32-bit numbers). The goal is to calculate \\\\(y=x\\cdot r^{-1} \\mod n\\\\).\n\nSince \\\\(r\\\\) is coprime with \\\\(n\\\\), we know that there are two numbers \\\\(r^{-1}\\\\) and \\\\(n'\\\\) in the \\\\([0, n)\\\\) range such that\n\\\\[ r \\cdot r^{-1} + n \\cdot n' = 1 \\\\]\nboth \\\\(r^{-1} and \\quad n'\\\\) can be computed using extended Euclidean algorithm **(\\\\(n' = n^{-1} \\mod r\\\\))**.\nUsing this identiy, we can express \\\\(r \\cdot r^{-1}\\\\) as \\\\(1-n\\cdot n'\\\\) and write \\\\(x \\cdot r^{-1}\\\\) as\n\n\\\\[\n \\displaylines{\n x\\cdot r^{-1} = \\frac{x \\cdot r \\cdot r^{-1}}{r} \\\\\\\\\n = \\frac{x \\cdot (1-n\\cdot n')}{r} \\\\\\\\\n = \\frac{x - x \\cdot n \\cdot n'}{r} \\\\\\\\\n = \\frac{x - x \\cdot n \\cdot n' + k\\cdot r \\cdot n}{r} \\quad (\\mod n) (for \\hspace{0.2cm}any\\hspace{0.2cm} integer\\hspace{0.2cm} k) \\\\\\\\\n = \\frac{x - (x\\cdot n' - k\\cdot r)\\cdot n}{r}\n}\n \\\\]\n\nNow, if we choose \\\\(k\\\\) to be \\\\(\\lfloor x\\cdot n'/r\\rfloor\\\\) (the upper 64 bits of \\\\(x\\cdot n'\\\\), giving \\\\(x\\\\) is 64 bits and \\\\(n'\\\\) is 32 bits ), it will cancel out, and \\\\(k\\cdot r - x\\cdot n'\\\\) will simply be equal to \\\\(x \\cdot n' \\mod r\\\\) (the lower 32 bits of \\\\(x\\cdot n'\\\\)), implying:\n\\\\[ x\\cdot r^{-1} = (x - (x\\cdot n' \\mod r )\\cdot n)/r\\\\]\n\nthe algorithm itself just evaluates this formula, performing two multiplications to calculate \\\\(q = x\\cdot n' \\mod r\\\\) and \\\\(m = q\\cdot n\\\\), and then subtracts it from \\\\(x\\\\) and right shifts the result to divide it by r.\n\nThe only remaining thing to handle is that the result may not be in the \\\\([0,n)\\\\) range; but since\n\\\\[x < n\\cdot n < r\\cdot n => x/r < n \\\\]\nand\n\\\\[m = q\\cdot n < r\\cdot n => m/r < n\\\\]\nit is guaranted that \n\\\\[ -n < (x-m)/r < n \\\\]\n\nTherefore, we can simply check if the result is negative and in that case, add \\\\(n\\\\) to it, giving the following algorithm:\n\n```cpp\ntypedef __uint32_t u32;\ntypedef __uint64_t u64;\n\nconst u32 n = 1e9 + 7, nr = inverse(n, 1ull << 32); // nr is n' in the above equations, r is 1<<32\n\nu32 reduce(u64 x) {\n u32 q = u32(x) * nr; // q = x * n' mod r\n u64 m = (u64) q * n; // m = q * n\n u32 y = (x - m) >> 32; // y = (x - m) / r\n return x < m ? y + n : y; // if y < 0, add n to make it be in the [0, n) range\n}\n```\nThis last check is relatively cheap, but it is still on the critical path. If we are fine with the result being in the \\\\([0,2\\cdot n−2]\\\\) range instead of \\\\([0, n)\\\\), we can remove it and add \\\\(n\\\\) to the result unconditionally:\n\n```cpp\nu32 reduce(u64 x) {\n u32 q = u32(x) * nr;\n u64 m = (u64) q * n;\n u32 y = (x - m) >> 32;\n return y + n\n}\n```\n\nWe can also move the `>> 32` operation one step earlier in the computation graph and compute \\\\(\\lfloor x/r \\rfloor - \\lfloor m/r \\rfloor\\\\) instead of \\\\( (x-m)/r\\\\). This is correct because the lower 32 bits of \\\\(x\\\\) and \\\\(m\\\\) are equal anyway since \n\\\\[ m = x\\cdot n' \\cdot n = x \\mod r\\\\] (mod r is same as taking the lower 32 bits).\n\nBut why would we voluntarily choose to perfom two right-shifts instead of just one? This is beneficial because for `((u64) q * n) >> 32` we need to do a 32-by-32 multiplication and take the upper 32 bits of the result (which the x86 mul instruction already writes in a separate register, so it doesn’t cost anything), and the other right-shift `x >> 32` is not on the critical path.\n```cpp\nu32 reduce(u64 x) {\n u32 q = u32(x) * nr;\n u32 m = ((u64) q * n) >> 32;\n return (x >> 32) + n - m;\n}\n```\nif to reduce a u128 number（u64 * u64). it is as below\n```cpp\ntypedef __uint128_t u128;\n\nu64 reduce(u128 x) const {\n u64 q = u64(x) * nr;\n u64 m = ((u128) q * n) >> 64;\n return (x >> 64) + n - m;\n}\n```\n\n# Faster Inverse\nMontgomery multiplication itself is fast, but it requires some precomputation:\n- inverting \\\\(n\\\\) modulo \\\\(r\\\\) to compute \\\\(n'\\\\)\n- transforming a number to the Montgomery space,\n- transforming a number from the Montgomery space.\n\nThe last operation is already efficiently performed with the reduce procedure we just implemented, but the first `inverting` can be slightly optimized.\nComputing the inverse \\\\(n' = n^{-1} \\mod r\\\\) can be done faster than with the extended Euclidean algorith by taking advantage of the fact that \\\\(r\\\\) is a power of two and using the following identity:\n\\\\[ a\\cdot x = 1 \\mod 2^k => a\\cdot x\\cdot (2-a\\cdot x) = 1 \\mod 2^{2k} \\\\]\nProof: \n \\\\[\n \\displaylines{\n a\\cdot x \\cdot (2-a\\cdot x) = 2\\cdot a\\cdot x - (a\\cdot x)^2 \\\\\\\\\n = 2\\cdot(1+m\\cdot 2^k) - (1+m\\cdot 2^k)^2 \\\\\\\\\n = 2 + 2\\cdot m\\cdot 2^k - 1-2\\cdot m \\cdot 2^k -m^2\\cdot 2^{2k} \\\\\\\\\n = 1-m^2\\cdot 2^{2k} \\\\\\\\\n = 1 \\mod 2^{2k}\n}\n \\\\]\n**note**\n- \\\\(a\\cdot x \\\\) coudl be represented by \\\\(1+m\\cdot 2^k\\\\) as \\\\(a\\cdot x = 1 \\mod 2^k\\\\)\n\n\nas for our case, \\\\(x\\\\) is \\\\(nr\\\\), \\\\(a\\\\) is \\\\(n\\\\). i.e \\\\(n\\cdot nr = 1 \\mod 2^k\\\\). Since \\\\(n \\\\) is an odd number, we can start with \\\\(nr=1\\\\) as the inverse of \\\\(n\\\\) modulo \\\\(2^1\\\\). then applying this identity one time, i.e \\\\(n\\cdot 1 \\cdot (2-n\\cdot 1) = 1 \\mod 2^2\\\\). Let \\\\(nr = 1\\cdot(2-n\\cdot 1) \\\\), we apply this identity again, and get \\\\(n\\cdot nr \\cdot (2-a\\cdot nr) = 1 \\mod 2^4\\\\). Following this approach, and apply this identity exactly \\\\(log_2^m\\\\) times, where \\\\(m\\\\) is the bit length of \\\\(r\\\\)\n\n## Complement Implementation\n```cpp\n\n\n```\n## references\n- https://en.algorithmica.org/hpc/number-theory/montgomery/\n- [csdn blog on montgomery reduction](https://blog.csdn.net/mutourend/article/details/95613967)\n- [nayuki montgomery reduction](https://www.nayuki.io/page/montgomery-reduction-algorithm)\n","slug":"arithmatic/montgomery-multiplication","published":1,"updated":"2023-12-08T02:45:44.046Z","_id":"clphpvdof0000aj7u9gaz5qfy","comments":1,"layout":"post","photos":[],"link":"","content":"\n\n\n

montgomery space

Montgomery multiplication works by first transforming the multipliers into Montgomery space, where modular multiplication can be performed cheaply, and then transforming them back when their actual values are needed. Unlike general integer division methods, Montgomery multiplication is not efficient for performing just one modular reduction and only becomes worthwhile when there is a chain of modular operations.

\n

The space is defined by the modulo n and a positive integer r ≥ n coprime to n. The algorithm involves modulo and division by r, so in practice, r is chosen to be 2^32 or 2^64, so that these operations can be done with a right-shift and a bitwise AND respectively.

\n

Definition. the representative \\(\\bar{x}\\) of a number \\(x\\) in the Montgomery space is defined as
\\[ \\bar{x} = x \\cdot r \\mod n \\]

\n

Computing this transformation involves a multiplication and a modulo — an expensive operation that we wanted to optimize away in the first place — which is why we only use this method when the overhead of transforming numbers to and from the Montgomery space is worth it and not for general modular multiplication.

\n

Inside the Montgomery space, addition, substraction, and checking for equality is performed as usual:

\n

\\[ x \\cdot r + y \\cdot r \\equiv (x+y) \\cdot r \\mod n \\]

\n

However, this is not the case for multiplication. Denoting multiplication in the Montgomery space as \\(∗\\) and the “normal” multiplication as \\(\\cdot\\), we expect the result to be:
\\[ \\bar{x} * \\bar{y} = \\overline{x\\cdot y} = (x\\cdot y)\\cdot r \\mod n \\]

\n

But the normal multiplication in the Montgomery space yields:
\\[ \\bar{x} \\cdot \\bar{y}=(x\\cdot y)\\cdot r \\cdot r \\mod n \\]

\n

Therefore, the multiplication in the Montgomery space is defined as
\\[ \\bar{x} * \\bar{y} = \\bar{x} \\cdot \\bar{y} \\cdot r^{-1} \\mod n \\]

\n

This means that, after we normally multiply two numbers in the Montgomery space, we need to reduce the result by multiplying it by \\(r^{-1}\\) and taking the modulo — and there is an efficent way to do this particular operation.

\n

Montgomery reduction

assume that \\(r=2^{32}\\), the modulo \\(n\\) is 32-bit, and the number \\(x\\) we need to reduce is 64-bit(the product of two 32-bit numbers). The goal is to calculate \\(y=x\\cdot r^{-1} \\mod n\\).

\n

Since \\(r\\) is coprime with \\(n\\), we know that there are two numbers \\(r^{-1}\\) and \\(n’\\) in the \\([0, n)\\) range such that
\\[ r \\cdot r^{-1} + n \\cdot n’ = 1 \\]
both \\(r^{-1} and \\quad n’\\) can be computed using extended Euclidean algorithm (\\(n’ = n^{-1} \\mod r\\)).
Using this identiy, we can express \\(r \\cdot r^{-1}\\) as \\(1-n\\cdot n’\\) and write \\(x \\cdot r^{-1}\\) as

\n

\\[
\\displaylines{
x\\cdot r^{-1} = \\frac{x \\cdot r \\cdot r^{-1}}{r} \\\\
= \\frac{x \\cdot (1-n\\cdot n’)}{r} \\\\
= \\frac{x - x \\cdot n \\cdot n’}{r} \\\\
= \\frac{x - x \\cdot n \\cdot n’ + k\\cdot r \\cdot n}{r} \\quad (\\mod n) (for \\hspace{0.2cm}any\\hspace{0.2cm} integer\\hspace{0.2cm} k) \\\\
= \\frac{x - (x\\cdot n’ - k\\cdot r)\\cdot n}{r}
}
\\]

\n

Now, if we choose \\(k\\) to be \\(\\lfloor x\\cdot n’/r\\rfloor\\) (the upper 64 bits of \\(x\\cdot n’\\), giving \\(x\\) is 64 bits and \\(n’\\) is 32 bits ), it will cancel out, and \\(k\\cdot r - x\\cdot n’\\) will simply be equal to \\(x \\cdot n’ \\mod r\\) (the lower 32 bits of \\(x\\cdot n’\\)), implying:
\\[ x\\cdot r^{-1} = (x - (x\\cdot n’ \\mod r )\\cdot n)/r\\]

\n

the algorithm itself just evaluates this formula, performing two multiplications to calculate \\(q = x\\cdot n’ \\mod r\\) and \\(m = q\\cdot n\\), and then subtracts it from \\(x\\) and right shifts the result to divide it by r.

\n

The only remaining thing to handle is that the result may not be in the \\([0,n)\\) range; but since
\\[x < n\\cdot n < r\\cdot n => x/r < n \\]
and
\\[m = q\\cdot n < r\\cdot n => m/r < n\\]
it is guaranted that
\\[ -n < (x-m)/r < n \\]

\n

Therefore, we can simply check if the result is negative and in that case, add \\(n\\) to it, giving the following algorithm:

\n
1
2
3
4
5
6
7
8
9
10
11
typedef __uint32_t u32;
typedef __uint64_t u64;

const u32 n = 1e9 + 7, nr = inverse(n, 1ull << 32); // nr is n' in the above equations, r is 1<<32

u32 reduce(u64 x) {
    u32 q = u32(x) * nr;      // q = x * n' mod r
    u64 m = (u64) q * n;      // m = q * n
    u32 y = (x - m) >> 32;    // y = (x - m) / r
    return x < m ? y + n : y; // if y < 0, add n to make it be in the [0, n) range
}
\n

This last check is relatively cheap, but it is still on the critical path. If we are fine with the result being in the \\([0,2\\cdot n−2]\\) range instead of \\([0, n)\\), we can remove it and add \\(n\\) to the result unconditionally:

\n
1
2
3
4
5
6
u32 reduce(u64 x) {
    u32 q = u32(x) * nr;
    u64 m = (u64) q * n;
    u32 y = (x - m) >> 32;
    return y + n
}
\n\n

We can also move the >> 32 operation one step earlier in the computation graph and compute \\(\\lfloor x/r \\rfloor - \\lfloor m/r \\rfloor\\) instead of \\( (x-m)/r\\). This is correct because the lower 32 bits of \\(x\\) and \\(m\\) are equal anyway since
\\[ m = x\\cdot n’ \\cdot n = x \\mod r\\] (mod r is same as taking the lower 32 bits).

\n

But why would we voluntarily choose to perfom two right-shifts instead of just one? This is beneficial because for ((u64) q * n) >> 32 we need to do a 32-by-32 multiplication and take the upper 32 bits of the result (which the x86 mul instruction already writes in a separate register, so it doesn’t cost anything), and the other right-shift x >> 32 is not on the critical path.

\n
1
2
3
4
5
u32 reduce(u64 x) {
    u32 q = u32(x) * nr;
    u32 m = ((u64) q * n) >> 32;
    return (x >> 32) + n - m;
}
\n

if to reduce a u128 number（u64 * u64). it is as below

\n
1
2
3
4
5
6
7
typedef __uint128_t u128;

u64 reduce(u128 x) const {
    u64 q = u64(x) * nr;
    u64 m = ((u128) q * n) >> 64;
    return (x >> 64) + n - m;
}
\n\n

Faster Inverse

Montgomery multiplication itself is fast, but it requires some precomputation:

\n
    \n
  • inverting \\(n\\) modulo \\(r\\) to compute \\(n’\\)
    • \n
  • transforming a number to the Montgomery space,
    • \n
  • transforming a number from the Montgomery space.
    • \n
\n

The last operation is already efficiently performed with the reduce procedure we just implemented, but the first inverting can be slightly optimized.
Computing the inverse \\(n’ = n^{-1} \\mod r\\) can be done faster than with the extended Euclidean algorith by taking advantage of the fact that \\(r\\) is a power of two and using the following identity:
\\[ a\\cdot x = 1 \\mod 2^k => a\\cdot x\\cdot (2-a\\cdot x) = 1 \\mod 2^{2k} \\]
Proof:
\\[
\\displaylines{
a\\cdot x \\cdot (2-a\\cdot x) = 2\\cdot a\\cdot x - (a\\cdot x)^2 \\\\
= 2\\cdot(1+m\\cdot 2^k) - (1+m\\cdot 2^k)^2 \\\\
= 2 + 2\\cdot m\\cdot 2^k - 1-2\\cdot m \\cdot 2^k -m^2\\cdot 2^{2k} \\\\
= 1-m^2\\cdot 2^{2k} \\\\
= 1 \\mod 2^{2k}
}
\\]
note

\n
    \n
  • \\(a\\cdot x \\) coudl be represented by \\(1+m\\cdot 2^k\\) as \\(a\\cdot x = 1 \\mod 2^k\\)
    • \n
\n

as for our case, \\(x\\) is \\(nr\\), \\(a\\) is \\(n\\). i.e \\(n\\cdot nr = 1 \\mod 2^k\\). Since \\(n \\) is an odd number, we can start with \\(nr=1\\) as the inverse of \\(n\\) modulo \\(2^1\\). then applying this identity one time, i.e \\(n\\cdot 1 \\cdot (2-n\\cdot 1) = 1 \\mod 2^2\\). Let \\(nr = 1\\cdot(2-n\\cdot 1) \\), we apply this identity again, and get \\(n\\cdot nr \\cdot (2-a\\cdot nr) = 1 \\mod 2^4\\). Following this approach, and apply this identity exactly \\(log_2^m\\) times, where \\(m\\) is the bit length of \\(r\\)

\n

Complement Implementation

1
2


\n

references

\n","site":{"data":{}},"excerpt":"","more":"\n\n\n

montgomery space

Montgomery multiplication works by first transforming the multipliers into Montgomery space, where modular multiplication can be performed cheaply, and then transforming them back when their actual values are needed. Unlike general integer division methods, Montgomery multiplication is not efficient for performing just one modular reduction and only becomes worthwhile when there is a chain of modular operations.

\n

The space is defined by the modulo n and a positive integer r ≥ n coprime to n. The algorithm involves modulo and division by r, so in practice, r is chosen to be 2^32 or 2^64, so that these operations can be done with a right-shift and a bitwise AND respectively.

\n

Definition. the representative \\(\\bar{x}\\) of a number \\(x\\) in the Montgomery space is defined as
\\[ \\bar{x} = x \\cdot r \\mod n \\]

\n

Computing this transformation involves a multiplication and a modulo — an expensive operation that we wanted to optimize away in the first place — which is why we only use this method when the overhead of transforming numbers to and from the Montgomery space is worth it and not for general modular multiplication.

\n

Inside the Montgomery space, addition, substraction, and checking for equality is performed as usual:

\n

\\[ x \\cdot r + y \\cdot r \\equiv (x+y) \\cdot r \\mod n \\]

\n

However, this is not the case for multiplication. Denoting multiplication in the Montgomery space as \\(∗\\) and the “normal” multiplication as \\(\\cdot\\), we expect the result to be:
\\[ \\bar{x} * \\bar{y} = \\overline{x\\cdot y} = (x\\cdot y)\\cdot r \\mod n \\]

\n

But the normal multiplication in the Montgomery space yields:
\\[ \\bar{x} \\cdot \\bar{y}=(x\\cdot y)\\cdot r \\cdot r \\mod n \\]

\n

Therefore, the multiplication in the Montgomery space is defined as
\\[ \\bar{x} * \\bar{y} = \\bar{x} \\cdot \\bar{y} \\cdot r^{-1} \\mod n \\]

\n

This means that, after we normally multiply two numbers in the Montgomery space, we need to reduce the result by multiplying it by \\(r^{-1}\\) and taking the modulo — and there is an efficent way to do this particular operation.

\n

Montgomery reduction

assume that \\(r=2^{32}\\), the modulo \\(n\\) is 32-bit, and the number \\(x\\) we need to reduce is 64-bit(the product of two 32-bit numbers). The goal is to calculate \\(y=x\\cdot r^{-1} \\mod n\\).

\n

Since \\(r\\) is coprime with \\(n\\), we know that there are two numbers \\(r^{-1}\\) and \\(n’\\) in the \\([0, n)\\) range such that
\\[ r \\cdot r^{-1} + n \\cdot n’ = 1 \\]
both \\(r^{-1} and \\quad n’\\) can be computed using extended Euclidean algorithm (\\(n’ = n^{-1} \\mod r\\)).
Using this identiy, we can express \\(r \\cdot r^{-1}\\) as \\(1-n\\cdot n’\\) and write \\(x \\cdot r^{-1}\\) as

\n

\\[
\\displaylines{
x\\cdot r^{-1} = \\frac{x \\cdot r \\cdot r^{-1}}{r} \\\\
= \\frac{x \\cdot (1-n\\cdot n’)}{r} \\\\
= \\frac{x - x \\cdot n \\cdot n’}{r} \\\\
= \\frac{x - x \\cdot n \\cdot n’ + k\\cdot r \\cdot n}{r} \\quad (\\mod n) (for \\hspace{0.2cm}any\\hspace{0.2cm} integer\\hspace{0.2cm} k) \\\\
= \\frac{x - (x\\cdot n’ - k\\cdot r)\\cdot n}{r}
}
\\]

\n

Now, if we choose \\(k\\) to be \\(\\lfloor x\\cdot n’/r\\rfloor\\) (the upper 64 bits of \\(x\\cdot n’\\), giving \\(x\\) is 64 bits and \\(n’\\) is 32 bits ), it will cancel out, and \\(k\\cdot r - x\\cdot n’\\) will simply be equal to \\(x \\cdot n’ \\mod r\\) (the lower 32 bits of \\(x\\cdot n’\\)), implying:
\\[ x\\cdot r^{-1} = (x - (x\\cdot n’ \\mod r )\\cdot n)/r\\]

\n

the algorithm itself just evaluates this formula, performing two multiplications to calculate \\(q = x\\cdot n’ \\mod r\\) and \\(m = q\\cdot n\\), and then subtracts it from \\(x\\) and right shifts the result to divide it by r.

\n

The only remaining thing to handle is that the result may not be in the \\([0,n)\\) range; but since
\\[x < n\\cdot n < r\\cdot n => x/r < n \\]
and
\\[m = q\\cdot n < r\\cdot n => m/r < n\\]
it is guaranted that
\\[ -n < (x-m)/r < n \\]

\n

Therefore, we can simply check if the result is negative and in that case, add \\(n\\) to it, giving the following algorithm:

\n
1
2
3
4
5
6
7
8
9
10
11
typedef __uint32_t u32;
typedef __uint64_t u64;

const u32 n = 1e9 + 7, nr = inverse(n, 1ull << 32); // nr is n' in the above equations, r is 1<<32

u32 reduce(u64 x) {
    u32 q = u32(x) * nr;      // q = x * n' mod r
    u64 m = (u64) q * n;      // m = q * n
    u32 y = (x - m) >> 32;    // y = (x - m) / r
    return x < m ? y + n : y; // if y < 0, add n to make it be in the [0, n) range
}
\n

This last check is relatively cheap, but it is still on the critical path. If we are fine with the result being in the \\([0,2\\cdot n−2]\\) range instead of \\([0, n)\\), we can remove it and add \\(n\\) to the result unconditionally:

\n
1
2
3
4
5
6
u32 reduce(u64 x) {
    u32 q = u32(x) * nr;
    u64 m = (u64) q * n;
    u32 y = (x - m) >> 32;
    return y + n
}
\n\n

We can also move the >> 32 operation one step earlier in the computation graph and compute \\(\\lfloor x/r \\rfloor - \\lfloor m/r \\rfloor\\) instead of \\( (x-m)/r\\). This is correct because the lower 32 bits of \\(x\\) and \\(m\\) are equal anyway since
\\[ m = x\\cdot n’ \\cdot n = x \\mod r\\] (mod r is same as taking the lower 32 bits).

\n

But why would we voluntarily choose to perfom two right-shifts instead of just one? This is beneficial because for ((u64) q * n) >> 32 we need to do a 32-by-32 multiplication and take the upper 32 bits of the result (which the x86 mul instruction already writes in a separate register, so it doesn’t cost anything), and the other right-shift x >> 32 is not on the critical path.

\n
1
2
3
4
5
u32 reduce(u64 x) {
    u32 q = u32(x) * nr;
    u32 m = ((u64) q * n) >> 32;
    return (x >> 32) + n - m;
}
\n

if to reduce a u128 number（u64 * u64). it is as below

\n
1
2
3
4
5
6
7
typedef __uint128_t u128;

u64 reduce(u128 x) const {
    u64 q = u64(x) * nr;
    u64 m = ((u128) q * n) >> 64;
    return (x >> 64) + n - m;
}
\n\n

Faster Inverse

Montgomery multiplication itself is fast, but it requires some precomputation:

\n
    \n
  • inverting \\(n\\) modulo \\(r\\) to compute \\(n’\\)
    • \n
  • transforming a number to the Montgomery space,
    • \n
  • transforming a number from the Montgomery space.
    • \n
\n

The last operation is already efficiently performed with the reduce procedure we just implemented, but the first inverting can be slightly optimized.
Computing the inverse \\(n’ = n^{-1} \\mod r\\) can be done faster than with the extended Euclidean algorith by taking advantage of the fact that \\(r\\) is a power of two and using the following identity:
\\[ a\\cdot x = 1 \\mod 2^k => a\\cdot x\\cdot (2-a\\cdot x) = 1 \\mod 2^{2k} \\]
Proof:
\\[
\\displaylines{
a\\cdot x \\cdot (2-a\\cdot x) = 2\\cdot a\\cdot x - (a\\cdot x)^2 \\\\
= 2\\cdot(1+m\\cdot 2^k) - (1+m\\cdot 2^k)^2 \\\\
= 2 + 2\\cdot m\\cdot 2^k - 1-2\\cdot m \\cdot 2^k -m^2\\cdot 2^{2k} \\\\
= 1-m^2\\cdot 2^{2k} \\\\
= 1 \\mod 2^{2k}
}
\\]
note

\n
    \n
  • \\(a\\cdot x \\) coudl be represented by \\(1+m\\cdot 2^k\\) as \\(a\\cdot x = 1 \\mod 2^k\\)
    • \n
\n

as for our case, \\(x\\) is \\(nr\\), \\(a\\) is \\(n\\). i.e \\(n\\cdot nr = 1 \\mod 2^k\\). Since \\(n \\) is an odd number, we can start with \\(nr=1\\) as the inverse of \\(n\\) modulo \\(2^1\\). then applying this identity one time, i.e \\(n\\cdot 1 \\cdot (2-n\\cdot 1) = 1 \\mod 2^2\\). Let \\(nr = 1\\cdot(2-n\\cdot 1) \\), we apply this identity again, and get \\(n\\cdot nr \\cdot (2-a\\cdot nr) = 1 \\mod 2^4\\). Following this approach, and apply this identity exactly \\(log_2^m\\) times, where \\(m\\) is the bit length of \\(r\\)

\n

Complement Implementation

1
2


\n

references

\n"},{"title":"sgx multi prover","date":"2023-11-24T06:29:26.000Z","_content":"\n\n\n## references \n- https://blog.csdn.net/mutourend/article/details/133586549?spm=1001.2014.3001.5502","source":"_posts/cryptography/zkp/sgx_multi_prover.md","raw":"---\ntitle: sgx multi prover\ndate: 2023-11-24 14:29:26\ntags: [cryptography,zkp]\n---\n\n\n\n## references \n- https://blog.csdn.net/mutourend/article/details/133586549?spm=1001.2014.3001.5502","slug":"cryptography/zkp/sgx_multi_prover","published":1,"updated":"2023-11-24T14:05:15.647Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clphpvdoj0003aj7ugctear78","content":"\n\n\n

references

\n","site":{"data":{}},"excerpt":"","more":"\n\n\n

references

\n"},{"title":"stark 101","date":"2023-11-03T09:07:33.000Z","_content":"\n\n\n## trace and low degree extension\nthe objective is to develop a STARK prover for the FibonacciSq sequence over a finite field. The FibonacciSq sequence is defined by the recurrence relation\n\\\\[ a_{n+2} = a_{n+1} ^2 + a_n ^2 \\\\]\n\nthe statement is: **I know a FieldElement \\\\(X\\in \\mathbb{F}\\\\) such that the 1023rd element of the FibonacciSq sequence starting with \\\\(1, X\\\\) is \\\\(2338775057\\\\)**\n\nThe underlying field of this class is \\\\(\\mathbb{F}_{3221225473}\\\\) (\\\\(3221225473 = 3 \\cdot 2^{30} + 1\\\\)), so all operations are done modulo 3221225473.\n\n\n### FibonacciSq Trace\n let's construct a list `a` of length 1023, whose first two elements will be FieldElement objects representing 1 and 3141592, respectively. The next 1021 elements will be the FibonacciSq sequence induced by these two elements. `a` is called the **trace** of FibonacciSq, or, when the context is clear, the trace.\n\n### Thinking of Polynomials\nWe now want to think of the sequence as the evaluation of some polynomial \\\\(f\\\\) of degree 1022.\nWe will choose the domain to be some subgroup \\\\(G \\subseteq \\mathbb{F}^\\times\\\\) of size 1024, for reasons that will become clear later.\n\n(Recall that \\\\(\\mathbb{F}^\\times\\\\) denotes the multiplicative group of \\\\(\\mathbb{F}\\\\), which we get from \\\\(\\mathbb{F}\\\\) by omitting the zero element with the induced multiplication from the field. A subgroup of size 1024 exists because \\\\(\\mathbb{F}^\\times\\\\) is a cyclic group of size \\\\(3\\cdot 2^{30}\\\\), so it contains a subgroup of size \\\\(2^i\\\\) for any \\\\(0 \\leq i \\leq 30\\\\)).\n#### Find a Group of Size 1024\nIf we find an element \\\\(g \\in \\mathbb{F}\\\\) whose (multiplicative) order is 1024, then \\\\(g\\\\) will generate such a group. Create a list called `G` with all the elements of \\\\(G\\\\), such that \\\\(G[i] := g^i\\\\).\n\n\n### Evaluating on a Larger Domain\nthen, interpolating `G` over `a` we get a polynomial `f`. The trace, viewed as evaluations of a polynomial \\\\(f\\\\) on \\\\(G\\\\), can now be extended by evaluating \\\\(f\\\\) over a larger domain, thereby creating a Reed-Solomon error correction code.\n\n#### Cosets\nTo that end, we must decide on a larger domain on which \\\\(f\\\\) will be evaluated. We will work with a domain that is 8 times larger than \\\\(G\\\\).
A natural choice for such a domain is to take some group \\\\(H\\\\) of size 8192 (which exists because 8192 divides \\\\(|\\mathbb{F}^\\times|\\\\)), and shift it by the generator of \\\\(\\mathbb{F}^\\times\\\\), thereby obtaining a [coset](https://en.wikipedia.org/wiki/Coset) of \\\\(H\\\\).\n\nCreate a list called `H` of the elements of \\\\(H\\\\), and multiply each of them by the generator of \\\\(\\mathbb{F}^\\times\\\\) to obtain a list called `eval_domain`. In other words, `eval_domain` = \\\\(\\\\{w\\cdot h^i | 0 \\leq i <8192 \\\\}\\\\) for \\\\(h\\\\) the generator of \\\\(H\\\\) and \\\\(w\\\\) the generator of \\\\(\\mathbb{F}^\\times\\\\).\n\n\n#### Evaluate on a Coset\n```python\nf = interpolate_poly(G[:-1], a)\nf_eval = [f(d) for d in eval_domain]\n```\n\n### Commitments\nWe will use [Sha256](https://en.wikipedia.org/wiki/SHA-2)-based [Merkle Trees](https://en.wikipedia.org/wiki/Merkle_tree) as our commitment scheme.\n```python\nfrom merkle import MerkleTree\nf_merkle = MerkleTree(f_eval)\n```\n\n### Channel\nTheoretically, a STARK proof system is a protocol for interaction between two parties - a prover and a verifier. In practice, we convert this interactive protocol into a non-interactive proof using the [Fiat-Shamir Heuristic](https://en.wikipedia.org/wiki/Fiat%E2%80%93Shamir_heuristic). In this tutorial you will use the `Channel` class, which implements this transformation. This channel replaces the verifier in the sense that the prover (which you are writing) will send data, and receive random numbers or random `FieldElement` instances.\n\n\n\n\n## constraints\nIn this part, we are going to create a set of constraints over the trace `a`. \n### Step 1 - FibonacciSq Constraints\nFor `a` to be a correct trace of a FibonacciSq sequence that proves our claim:\n1. The first element has to be 1, namely \\\\(a[0] = 1\\\\).\n2. The last element has to be 2338775057, namely \\\\(a[1022] = 2338775057\\\\).\n3. The FibonacciSq rule must apply, that is - for every \\\\(i<1021\\\\), \\\\(a[i+2]=a[i+1]^2+a[i]^2\\\\).\n\n\n### Step 2 - Polynomial Constraints\nRecall that `f` is a polynomial over the trace domain, that evaluates exactly to `a` over \\\\(G \\setminus \\{g^{1023}\\}\\\\) where \\\\(G=\\{g^i : 0\\leq i\\leq 1023\\}\\\\) is the \"small\" group generated by \\\\(g\\\\).
\n\nWe now rewrite the above three constraints in a form of polynomial constraints over `f`:\n1. \\\\(a[0] = 1\\\\) is translated to the polynomial \\\\(f(x) - 1\\\\), which evalutes to 0 for \\\\(x = g^0\\\\) (note that \\\\(g^0\\\\) is \\\\(1\\\\)).
\n2. \\\\(a[1022] = 2338775057\\\\) is translated to the polynomial \\\\(f(x) - 2338775057\\\\), which evalutes to 0 for \\\\(x = g^{1022}\\\\).
\n3. \\\\(a[i+2]=a[i+1]^2+a[i]^2\\\\) for every \\\\(i<1021\\\\) is translated to the polynomial \\\\(f(g^2 \\cdot x) - (f(g \\cdot x))^2 - (f(x))^2\\\\), which evaluates to 0 for \\\\(x \\in G \\backslash \\{g^{1021}, g^{1022}, g^{1023}\\}\\\\).

\n\n### Step 3 - Rational Functions (That are in Fact Polynomials)\n\nEach of the constraints above is represented by a polynomial \\\\(u(x)\\\\) that supposedly evaluates to \\\\(0\\\\) on certain elements of the group \\\\(G\\\\). That is, for some \\\\(x_0, \\ldots, x_k \\in G\\\\), we claim that\n\n\\\\[u(x_0) = \\ldots = u(x_k) = 0\\\\]\n\n(note that for the first two constaints, \\\\(k=0\\\\) because they only refer to one point and for the third \\\\(k=1021\\\\)).\n\nThis is equivalent to saying that \\\\(u(x)\\\\) is divisible, as a polynomial, by all of \\\\(\\{(x-x_i)\\}_{i=0}^k\\\\), or, equivalently, by\n\n\\\\[\\prod_{i=0}^k (x-x_i)\\\\]\n\nTherefore, each of the three constraints above can be written as a rational function of the form:\n\n\\\\[\\frac{u(x)}{\\prod_{i=0}^k (x-x_i)}\\\\]\n\nfor the corresponding \\\\(u(x)\\\\) and \\\\(\\{x_i\\}_{i=0}^k\\\\). In this step we will construct these three rational functions and show that they are indeed polynomials.\n\n### The First Constraint:\n\nIn the first constraint, \\\\(f(x) - 1\\\\) and \\\\(\\{x_i\\} = \\{1\\}\\\\).\n\nWe will now construct the **polynomial** \\\\(p_0(x)=\\frac{f(x) - 1}{x - 1}\\\\), making sure that \\\\(f(x) - 1\\\\) is indeed divisible by \\\\((x-1)\\\\).\n\n### The Second Constraint\n\nConstruct the polynomial `p1` representing the second constraint, \\\\(p_1(x)= \\frac{f(x) - 2338775057}{x - g^{1022}}\\\\), similarly:
\n\n### The Third Constraint - Succinctness\n\nThe last constraint's rational function is slightly more complicated:
\n\n\n\\\\[p_2(x) = \\frac{f(g^2 \\cdot x) - (f(g \\cdot x))^2 - (f(x))^2}{\\prod\\limits_{i=0}^{1020} (x-g^i)}\\\\]\n\nwhose denominator can be rewritten, so that the entire expression is easier to compute:
\n\n$$\\frac{f(g^2 \\cdot x) - (f(g \\cdot x))^2 - (f(x))^2}{\\frac{x^{1024} - 1}{(x-g^{1021})(x-g^{1022})(x-g^{1023})}}$$
\n\nThis follows from the equality\n\n$$\\prod\\limits_{i=0}^{1023} (x-g^i) = x^{1024} - 1$$\n\n### Step 4 - Composition Polynomial\nRecall that we're translating a problem of checking the validity of three polynomial constraints to checking that each of the rational functions \\\\(p_0, p_1, p_2\\\\) are polynomials.
\n\nOur protocol uses an algorithm called [FRI](https://eccc.weizmann.ac.il/report/2017/134/) to do so, which will be discussed in the next part.
\nIn order for the proof to be succinct (short), we prefer to work with just one rational function instead of three. For that, we take a random linear combination of \\\\(p_0, p_1, p_2\\\\) called the **compostion polynomial** (CP for short):\n\n$$CP(x) = \\alpha_0 \\cdot p_0(x) + \\alpha_1 \\cdot p_1(x) + \\alpha_2 \\cdot p_2(x)$$
\n\nwhere \\\\(\\alpha_0, \\alpha_1, \\alpha_2 \\\\) are random field elements obtained from the verifier, or in our case - from the channel.\n\nProving that (the rational function) \\\\(CP\\\\) is a polynomial guarantess, with high probability, that each of \\\\(p_0, p_1, p_2\\\\) are themselves polynomials.\n\n### Commit on the Composition Polynomial\nLastly, we evaluate $cp$ over the evaluation domain (`eval_domain`), build a Merkle tree on top of that and send its root over the channel. This is similar to commiting on the LDE trace, as we did at the end of part 1.\n\n![trace to cp](images/zkp/stark/trace_to_CP.png)\n\n## FRI Commitments\n### FRI Folding\n\nOur goal in this part is to construct the FRI layers and commit on them. \n
To obtain each layer we need:\n1. To generate a domain for the layer (from the previous layer's domain).\n2. To generate a polynomial for the layer (from the previous layer's polynomial and domain).\n3. To evaluate said polynomial on said domain - **this is the next FRI layer**.\n\n#### Domain Generation\n\nThe first FRI domain is simply the `eval_domain` that you already generated in Part 1, namely a coset of a group of order 8192. Each subsequent FRI domain is obtained by taking the first half of the previous FRI domain (dropping the second half), and squaring each of its elements.
\n\nFormally - we got `eval_domain` by taking:
\n$$w, w\\cdot h, w\\cdot h^2, ..., w\\cdot h^{8191}$$\n\nThe next layer will therefore be:
\n$$w^2, (w\\cdot h)^2, (w\\cdot h^2)^2, ..., (w\\cdot h^{4095})^2$$\n\nNote that taking the squares of the second half of each elements in `eval_domain` yields exactly\nthe same result as taking the squares of the first half. This is true for the next layers as well.\n\nSimilarly, the domain of the third layer will be:
\n$$w^4, (w\\cdot h)^4, (w\\cdot h^2)^4, ..., (w\\cdot h^{2047})^4$$\n\nAnd so on.\n\n#### FRI Folding Operator\nThe first FRI polynomial is simply the composition polynomial, i.e., `cp`.
\nEach subsequent FRI polynomial is obtained by:\n1. Getting a random field element \\\\(\\beta\\\\) (by calling `Channel.receive_random_field_element`).\n2. Multiplying the odd coefficients of the previous polynomial by \\\\(\\beta\\\\).\n3. Summing together consecutive pairs (even-odd) of coefficients.\n\nFormally, let's say that the k-th polynomial is of degree \\\\(< m\\\\) (for some \\\\(m\\\\) which is a power of 2):\n\n$$p_{k}(x) := \\sum _{i=0} ^{m-1} c_i x^i$$\n\n\nThen the (k+1)-th polynomial, whose degree is \\\\(< \\frac m 2 \\\\) will be:\n\n\\\\[ p_{k+1}(x) := \\sum_{i=0} ^{ m / 2 - 1 } (c_{2i} + \\beta \\cdot c_{2i + 1}) x^i \\\\]
\n\n![fri](images/zkp/stark/fri.png)\n\n![fri example](images/zkp/stark/fri_example.png)\n\n### Generating FRI Commitments\n\nWe have now developed the tools to write the `FriCommit` method, that contains the main FRI commitment loop.
\n\nIt takes the following 5 arguments:\n1. The composition polynomial, that is also the first FRI polynomial, that is - `cp`.\n2. The coset of order 8192 that is also the first FRI domain, that is - `eval_domain`.\n3. The evaluation of the former over the latter, which is also the first FRI layer , that is - `cp_eval`.\n4. The first Merkle tree (we will have one for each FRI layer) constructed from these evaluations, that is - `cp_merkle`.\n5. A channel object, that is `channel`.\n\nThe method accordingly returns 4 lists:\n1. The FRI polynomials.\n2. The FRI domains.\n3. The FRI layers.\n4. The FRI Merkle trees.\n\nThe method contains a loop, in each iteration of which we extend these four lists, using the last element in each.\nThe iteration should stop once the last FRI polynomial is of degree 0, that is - when the last FRI polynomial is just a constant. It should then send over the channel this constant (i.e. - the polynomial's free term).\nThe `Channel` class only supports sending strings, so make sure you convert anything you wish to send over the channel to a string before sending.\n\n![commitment](images/zkp/stark/commitment.png)\n\n## Query Phase\nGet q random elements, provide a valdiation data for each\n\n![decommitment](images/zkp/stark/decommitment.png)\n\n# references\n- https://github.com/starkware-industries/stark101\n- [startk_math](https://medium.com/starkware/tagged/stark-math)\n- [starkEx deep dive](https://medium.com/starkware/starkdex-deep-dive-introduction-7b4ef0dedba8)\n- [coset] https://en.wikipedia.org/wiki/Coset\n- [starkware math](https://medium.com/starkware/arithmetization-ii-403c3b3f4355)","source":"_posts/cryptography/zkp/stark-101.md","raw":"---\ntitle: stark 101\ndate: 2023-11-03 17:07:33\ntags: [cryptography,zkp]\n---\n\n\n\n## trace and low degree extension\nthe objective is to develop a STARK prover for the FibonacciSq sequence over a finite field. The FibonacciSq sequence is defined by the recurrence relation\n\\\\[ a_{n+2} = a_{n+1} ^2 + a_n ^2 \\\\]\n\nthe statement is: **I know a FieldElement \\\\(X\\in \\mathbb{F}\\\\) such that the 1023rd element of the FibonacciSq sequence starting with \\\\(1, X\\\\) is \\\\(2338775057\\\\)**\n\nThe underlying field of this class is \\\\(\\mathbb{F}_{3221225473}\\\\) (\\\\(3221225473 = 3 \\cdot 2^{30} + 1\\\\)), so all operations are done modulo 3221225473.\n\n\n### FibonacciSq Trace\n let's construct a list `a` of length 1023, whose first two elements will be FieldElement objects representing 1 and 3141592, respectively. The next 1021 elements will be the FibonacciSq sequence induced by these two elements. `a` is called the **trace** of FibonacciSq, or, when the context is clear, the trace.\n\n### Thinking of Polynomials\nWe now want to think of the sequence as the evaluation of some polynomial \\\\(f\\\\) of degree 1022.\nWe will choose the domain to be some subgroup \\\\(G \\subseteq \\mathbb{F}^\\times\\\\) of size 1024, for reasons that will become clear later.\n\n(Recall that \\\\(\\mathbb{F}^\\times\\\\) denotes the multiplicative group of \\\\(\\mathbb{F}\\\\), which we get from \\\\(\\mathbb{F}\\\\) by omitting the zero element with the induced multiplication from the field. A subgroup of size 1024 exists because \\\\(\\mathbb{F}^\\times\\\\) is a cyclic group of size \\\\(3\\cdot 2^{30}\\\\), so it contains a subgroup of size \\\\(2^i\\\\) for any \\\\(0 \\leq i \\leq 30\\\\)).\n#### Find a Group of Size 1024\nIf we find an element \\\\(g \\in \\mathbb{F}\\\\) whose (multiplicative) order is 1024, then \\\\(g\\\\) will generate such a group. Create a list called `G` with all the elements of \\\\(G\\\\), such that \\\\(G[i] := g^i\\\\).\n\n\n### Evaluating on a Larger Domain\nthen, interpolating `G` over `a` we get a polynomial `f`. The trace, viewed as evaluations of a polynomial \\\\(f\\\\) on \\\\(G\\\\), can now be extended by evaluating \\\\(f\\\\) over a larger domain, thereby creating a Reed-Solomon error correction code.\n\n#### Cosets\nTo that end, we must decide on a larger domain on which \\\\(f\\\\) will be evaluated. We will work with a domain that is 8 times larger than \\\\(G\\\\).
A natural choice for such a domain is to take some group \\\\(H\\\\) of size 8192 (which exists because 8192 divides \\\\(|\\mathbb{F}^\\times|\\\\)), and shift it by the generator of \\\\(\\mathbb{F}^\\times\\\\), thereby obtaining a [coset](https://en.wikipedia.org/wiki/Coset) of \\\\(H\\\\).\n\nCreate a list called `H` of the elements of \\\\(H\\\\), and multiply each of them by the generator of \\\\(\\mathbb{F}^\\times\\\\) to obtain a list called `eval_domain`. In other words, `eval_domain` = \\\\(\\\\{w\\cdot h^i | 0 \\leq i <8192 \\\\}\\\\) for \\\\(h\\\\) the generator of \\\\(H\\\\) and \\\\(w\\\\) the generator of \\\\(\\mathbb{F}^\\times\\\\).\n\n\n#### Evaluate on a Coset\n```python\nf = interpolate_poly(G[:-1], a)\nf_eval = [f(d) for d in eval_domain]\n```\n\n### Commitments\nWe will use [Sha256](https://en.wikipedia.org/wiki/SHA-2)-based [Merkle Trees](https://en.wikipedia.org/wiki/Merkle_tree) as our commitment scheme.\n```python\nfrom merkle import MerkleTree\nf_merkle = MerkleTree(f_eval)\n```\n\n### Channel\nTheoretically, a STARK proof system is a protocol for interaction between two parties - a prover and a verifier. In practice, we convert this interactive protocol into a non-interactive proof using the [Fiat-Shamir Heuristic](https://en.wikipedia.org/wiki/Fiat%E2%80%93Shamir_heuristic). In this tutorial you will use the `Channel` class, which implements this transformation. This channel replaces the verifier in the sense that the prover (which you are writing) will send data, and receive random numbers or random `FieldElement` instances.\n\n\n\n\n## constraints\nIn this part, we are going to create a set of constraints over the trace `a`. \n### Step 1 - FibonacciSq Constraints\nFor `a` to be a correct trace of a FibonacciSq sequence that proves our claim:\n1. The first element has to be 1, namely \\\\(a[0] = 1\\\\).\n2. The last element has to be 2338775057, namely \\\\(a[1022] = 2338775057\\\\).\n3. The FibonacciSq rule must apply, that is - for every \\\\(i<1021\\\\), \\\\(a[i+2]=a[i+1]^2+a[i]^2\\\\).\n\n\n### Step 2 - Polynomial Constraints\nRecall that `f` is a polynomial over the trace domain, that evaluates exactly to `a` over \\\\(G \\setminus \\{g^{1023}\\}\\\\) where \\\\(G=\\{g^i : 0\\leq i\\leq 1023\\}\\\\) is the \"small\" group generated by \\\\(g\\\\).
\n\nWe now rewrite the above three constraints in a form of polynomial constraints over `f`:\n1. \\\\(a[0] = 1\\\\) is translated to the polynomial \\\\(f(x) - 1\\\\), which evalutes to 0 for \\\\(x = g^0\\\\) (note that \\\\(g^0\\\\) is \\\\(1\\\\)).
\n2. \\\\(a[1022] = 2338775057\\\\) is translated to the polynomial \\\\(f(x) - 2338775057\\\\), which evalutes to 0 for \\\\(x = g^{1022}\\\\).
\n3. \\\\(a[i+2]=a[i+1]^2+a[i]^2\\\\) for every \\\\(i<1021\\\\) is translated to the polynomial \\\\(f(g^2 \\cdot x) - (f(g \\cdot x))^2 - (f(x))^2\\\\), which evaluates to 0 for \\\\(x \\in G \\backslash \\{g^{1021}, g^{1022}, g^{1023}\\}\\\\).

\n\n### Step 3 - Rational Functions (That are in Fact Polynomials)\n\nEach of the constraints above is represented by a polynomial \\\\(u(x)\\\\) that supposedly evaluates to \\\\(0\\\\) on certain elements of the group \\\\(G\\\\). That is, for some \\\\(x_0, \\ldots, x_k \\in G\\\\), we claim that\n\n\\\\[u(x_0) = \\ldots = u(x_k) = 0\\\\]\n\n(note that for the first two constaints, \\\\(k=0\\\\) because they only refer to one point and for the third \\\\(k=1021\\\\)).\n\nThis is equivalent to saying that \\\\(u(x)\\\\) is divisible, as a polynomial, by all of \\\\(\\{(x-x_i)\\}_{i=0}^k\\\\), or, equivalently, by\n\n\\\\[\\prod_{i=0}^k (x-x_i)\\\\]\n\nTherefore, each of the three constraints above can be written as a rational function of the form:\n\n\\\\[\\frac{u(x)}{\\prod_{i=0}^k (x-x_i)}\\\\]\n\nfor the corresponding \\\\(u(x)\\\\) and \\\\(\\{x_i\\}_{i=0}^k\\\\). In this step we will construct these three rational functions and show that they are indeed polynomials.\n\n### The First Constraint:\n\nIn the first constraint, \\\\(f(x) - 1\\\\) and \\\\(\\{x_i\\} = \\{1\\}\\\\).\n\nWe will now construct the **polynomial** \\\\(p_0(x)=\\frac{f(x) - 1}{x - 1}\\\\), making sure that \\\\(f(x) - 1\\\\) is indeed divisible by \\\\((x-1)\\\\).\n\n### The Second Constraint\n\nConstruct the polynomial `p1` representing the second constraint, \\\\(p_1(x)= \\frac{f(x) - 2338775057}{x - g^{1022}}\\\\), similarly:
\n\n### The Third Constraint - Succinctness\n\nThe last constraint's rational function is slightly more complicated:
\n\n\n\\\\[p_2(x) = \\frac{f(g^2 \\cdot x) - (f(g \\cdot x))^2 - (f(x))^2}{\\prod\\limits_{i=0}^{1020} (x-g^i)}\\\\]\n\nwhose denominator can be rewritten, so that the entire expression is easier to compute:
\n\n$$\\frac{f(g^2 \\cdot x) - (f(g \\cdot x))^2 - (f(x))^2}{\\frac{x^{1024} - 1}{(x-g^{1021})(x-g^{1022})(x-g^{1023})}}$$
\n\nThis follows from the equality\n\n$$\\prod\\limits_{i=0}^{1023} (x-g^i) = x^{1024} - 1$$\n\n### Step 4 - Composition Polynomial\nRecall that we're translating a problem of checking the validity of three polynomial constraints to checking that each of the rational functions \\\\(p_0, p_1, p_2\\\\) are polynomials.
\n\nOur protocol uses an algorithm called [FRI](https://eccc.weizmann.ac.il/report/2017/134/) to do so, which will be discussed in the next part.
\nIn order for the proof to be succinct (short), we prefer to work with just one rational function instead of three. For that, we take a random linear combination of \\\\(p_0, p_1, p_2\\\\) called the **compostion polynomial** (CP for short):\n\n$$CP(x) = \\alpha_0 \\cdot p_0(x) + \\alpha_1 \\cdot p_1(x) + \\alpha_2 \\cdot p_2(x)$$
\n\nwhere \\\\(\\alpha_0, \\alpha_1, \\alpha_2 \\\\) are random field elements obtained from the verifier, or in our case - from the channel.\n\nProving that (the rational function) \\\\(CP\\\\) is a polynomial guarantess, with high probability, that each of \\\\(p_0, p_1, p_2\\\\) are themselves polynomials.\n\n### Commit on the Composition Polynomial\nLastly, we evaluate $cp$ over the evaluation domain (`eval_domain`), build a Merkle tree on top of that and send its root over the channel. This is similar to commiting on the LDE trace, as we did at the end of part 1.\n\n![trace to cp](images/zkp/stark/trace_to_CP.png)\n\n## FRI Commitments\n### FRI Folding\n\nOur goal in this part is to construct the FRI layers and commit on them. \n
To obtain each layer we need:\n1. To generate a domain for the layer (from the previous layer's domain).\n2. To generate a polynomial for the layer (from the previous layer's polynomial and domain).\n3. To evaluate said polynomial on said domain - **this is the next FRI layer**.\n\n#### Domain Generation\n\nThe first FRI domain is simply the `eval_domain` that you already generated in Part 1, namely a coset of a group of order 8192. Each subsequent FRI domain is obtained by taking the first half of the previous FRI domain (dropping the second half), and squaring each of its elements.
\n\nFormally - we got `eval_domain` by taking:
\n$$w, w\\cdot h, w\\cdot h^2, ..., w\\cdot h^{8191}$$\n\nThe next layer will therefore be:
\n$$w^2, (w\\cdot h)^2, (w\\cdot h^2)^2, ..., (w\\cdot h^{4095})^2$$\n\nNote that taking the squares of the second half of each elements in `eval_domain` yields exactly\nthe same result as taking the squares of the first half. This is true for the next layers as well.\n\nSimilarly, the domain of the third layer will be:
\n$$w^4, (w\\cdot h)^4, (w\\cdot h^2)^4, ..., (w\\cdot h^{2047})^4$$\n\nAnd so on.\n\n#### FRI Folding Operator\nThe first FRI polynomial is simply the composition polynomial, i.e., `cp`.
\nEach subsequent FRI polynomial is obtained by:\n1. Getting a random field element \\\\(\\beta\\\\) (by calling `Channel.receive_random_field_element`).\n2. Multiplying the odd coefficients of the previous polynomial by \\\\(\\beta\\\\).\n3. Summing together consecutive pairs (even-odd) of coefficients.\n\nFormally, let's say that the k-th polynomial is of degree \\\\(< m\\\\) (for some \\\\(m\\\\) which is a power of 2):\n\n$$p_{k}(x) := \\sum _{i=0} ^{m-1} c_i x^i$$\n\n\nThen the (k+1)-th polynomial, whose degree is \\\\(< \\frac m 2 \\\\) will be:\n\n\\\\[ p_{k+1}(x) := \\sum_{i=0} ^{ m / 2 - 1 } (c_{2i} + \\beta \\cdot c_{2i + 1}) x^i \\\\]
\n\n![fri](images/zkp/stark/fri.png)\n\n![fri example](images/zkp/stark/fri_example.png)\n\n### Generating FRI Commitments\n\nWe have now developed the tools to write the `FriCommit` method, that contains the main FRI commitment loop.
\n\nIt takes the following 5 arguments:\n1. The composition polynomial, that is also the first FRI polynomial, that is - `cp`.\n2. The coset of order 8192 that is also the first FRI domain, that is - `eval_domain`.\n3. The evaluation of the former over the latter, which is also the first FRI layer , that is - `cp_eval`.\n4. The first Merkle tree (we will have one for each FRI layer) constructed from these evaluations, that is - `cp_merkle`.\n5. A channel object, that is `channel`.\n\nThe method accordingly returns 4 lists:\n1. The FRI polynomials.\n2. The FRI domains.\n3. The FRI layers.\n4. The FRI Merkle trees.\n\nThe method contains a loop, in each iteration of which we extend these four lists, using the last element in each.\nThe iteration should stop once the last FRI polynomial is of degree 0, that is - when the last FRI polynomial is just a constant. It should then send over the channel this constant (i.e. - the polynomial's free term).\nThe `Channel` class only supports sending strings, so make sure you convert anything you wish to send over the channel to a string before sending.\n\n![commitment](images/zkp/stark/commitment.png)\n\n## Query Phase\nGet q random elements, provide a valdiation data for each\n\n![decommitment](images/zkp/stark/decommitment.png)\n\n# references\n- https://github.com/starkware-industries/stark101\n- [startk_math](https://medium.com/starkware/tagged/stark-math)\n- [starkEx deep dive](https://medium.com/starkware/starkdex-deep-dive-introduction-7b4ef0dedba8)\n- [coset] https://en.wikipedia.org/wiki/Coset\n- [starkware math](https://medium.com/starkware/arithmetization-ii-403c3b3f4355)","slug":"cryptography/zkp/stark-101","published":1,"updated":"2023-11-05T04:18:25.566Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clphpvdoj0005aj7ug6vbhr5l","content":"\n\n\n

trace and low degree extension

the objective is to develop a STARK prover for the FibonacciSq sequence over a finite field. The FibonacciSq sequence is defined by the recurrence relation
\\[ a_{n+2} = a_{n+1} ^2 + a_n ^2 \\]

\n

the statement is: I know a FieldElement \\(X\\in \\mathbb{F}\\) such that the 1023rd element of the FibonacciSq sequence starting with \\(1, X\\) is \\(2338775057\\)

\n

The underlying field of this class is \\(\\mathbb{F}_{3221225473}\\) (\\(3221225473 = 3 \\cdot 2^{30} + 1\\)), so all operations are done modulo 3221225473.

\n

FibonacciSq Trace

let’s construct a list a of length 1023, whose first two elements will be FieldElement objects representing 1 and 3141592, respectively. The next 1021 elements will be the FibonacciSq sequence induced by these two elements. a is called the trace of FibonacciSq, or, when the context is clear, the trace.

\n

Thinking of Polynomials

We now want to think of the sequence as the evaluation of some polynomial \\(f\\) of degree 1022.
We will choose the domain to be some subgroup \\(G \\subseteq \\mathbb{F}^\\times\\) of size 1024, for reasons that will become clear later.

\n

(Recall that \\(\\mathbb{F}^\\times\\) denotes the multiplicative group of \\(\\mathbb{F}\\), which we get from \\(\\mathbb{F}\\) by omitting the zero element with the induced multiplication from the field. A subgroup of size 1024 exists because \\(\\mathbb{F}^\\times\\) is a cyclic group of size \\(3\\cdot 2^{30}\\), so it contains a subgroup of size \\(2^i\\) for any \\(0 \\leq i \\leq 30\\)).

\n

Find a Group of Size 1024

If we find an element \\(g \\in \\mathbb{F}\\) whose (multiplicative) order is 1024, then \\(g\\) will generate such a group. Create a list called G with all the elements of \\(G\\), such that \\(G[i] := g^i\\).

\n

Evaluating on a Larger Domain

then, interpolating G over a we get a polynomial f. The trace, viewed as evaluations of a polynomial \\(f\\) on \\(G\\), can now be extended by evaluating \\(f\\) over a larger domain, thereby creating a Reed-Solomon error correction code.

\n

Cosets

To that end, we must decide on a larger domain on which \\(f\\) will be evaluated. We will work with a domain that is 8 times larger than \\(G\\).
A natural choice for such a domain is to take some group \\(H\\) of size 8192 (which exists because 8192 divides \\(|\\mathbb{F}^\\times|\\)), and shift it by the generator of \\(\\mathbb{F}^\\times\\), thereby obtaining a coset of \\(H\\).

\n

Create a list called H of the elements of \\(H\\), and multiply each of them by the generator of \\(\\mathbb{F}^\\times\\) to obtain a list called eval_domain. In other words, eval_domain = \\(\\{w\\cdot h^i | 0 \\leq i <8192 \\}\\) for \\(h\\) the generator of \\(H\\) and \\(w\\) the generator of \\(\\mathbb{F}^\\times\\).

\n

Evaluate on a Coset

1
2
f = interpolate_poly(G[:-1], a)
f_eval = [f(d) for d in eval_domain]
\n\n

Commitments

We will use Sha256-based Merkle Trees as our commitment scheme.

\n
1
2
from merkle import MerkleTree
f_merkle = MerkleTree(f_eval)
\n\n

Channel

Theoretically, a STARK proof system is a protocol for interaction between two parties - a prover and a verifier. In practice, we convert this interactive protocol into a non-interactive proof using the Fiat-Shamir Heuristic. In this tutorial you will use the Channel class, which implements this transformation. This channel replaces the verifier in the sense that the prover (which you are writing) will send data, and receive random numbers or random FieldElement instances.

\n

constraints

In this part, we are going to create a set of constraints over the trace a.

\n

Step 1 - FibonacciSq Constraints

For a to be a correct trace of a FibonacciSq sequence that proves our claim:

\n
    \n
  1. The first element has to be 1, namely \\(a[0] = 1\\).
    2. \n
  2. The last element has to be 2338775057, namely \\(a[1022] = 2338775057\\).
    3. \n
  3. The FibonacciSq rule must apply, that is - for every \\(i<1021\\), \\(a[i+2]=a[i+1]^2+a[i]^2\\).
    4. \n
\n

Step 2 - Polynomial Constraints

Recall that f is a polynomial over the trace domain, that evaluates exactly to a over \\(G \\setminus {g^{1023}}\\) where \\(G={g^i : 0\\leq i\\leq 1023}\\) is the “small” group generated by \\(g\\).

\n

We now rewrite the above three constraints in a form of polynomial constraints over f:

\n
    \n
  1. \\(a[0] = 1\\) is translated to the polynomial \\(f(x) - 1\\), which evalutes to 0 for \\(x = g^0\\) (note that \\(g^0\\) is \\(1\\)).
    2. \n
  2. \\(a[1022] = 2338775057\\) is translated to the polynomial \\(f(x) - 2338775057\\), which evalutes to 0 for \\(x = g^{1022}\\).
    3. \n
  3. \\(a[i+2]=a[i+1]^2+a[i]^2\\) for every \\(i<1021\\) is translated to the polynomial \\(f(g^2 \\cdot x) - (f(g \\cdot x))^2 - (f(x))^2\\), which evaluates to 0 for \\(x \\in G \\backslash {g^{1021}, g^{1022}, g^{1023}}\\).

    4. \n
\n

Step 3 - Rational Functions (That are in Fact Polynomials)

Each of the constraints above is represented by a polynomial \\(u(x)\\) that supposedly evaluates to \\(0\\) on certain elements of the group \\(G\\). That is, for some \\(x_0, \\ldots, x_k \\in G\\), we claim that

\n

\\[u(x_0) = \\ldots = u(x_k) = 0\\]

\n

(note that for the first two constaints, \\(k=0\\) because they only refer to one point and for the third \\(k=1021\\)).

\n

This is equivalent to saying that \\(u(x)\\) is divisible, as a polynomial, by all of \\({(x-x_i)}_{i=0}^k\\), or, equivalently, by

\n

\\[\\prod_{i=0}^k (x-x_i)\\]

\n

Therefore, each of the three constraints above can be written as a rational function of the form:

\n

\\[\\frac{u(x)}{\\prod_{i=0}^k (x-x_i)}\\]

\n

for the corresponding \\(u(x)\\) and \\({x_i}_{i=0}^k\\). In this step we will construct these three rational functions and show that they are indeed polynomials.

\n

The First Constraint:

In the first constraint, \\(f(x) - 1\\) and \\({x_i} = {1}\\).

\n

We will now construct the polynomial \\(p_0(x)=\\frac{f(x) - 1}{x - 1}\\), making sure that \\(f(x) - 1\\) is indeed divisible by \\((x-1)\\).

\n

The Second Constraint

Construct the polynomial p1 representing the second constraint, \\(p_1(x)= \\frac{f(x) - 2338775057}{x - g^{1022}}\\), similarly:

\n

The Third Constraint - Succinctness

The last constraint’s rational function is slightly more complicated:

\n

\\[p_2(x) = \\frac{f(g^2 \\cdot x) - (f(g \\cdot x))^2 - (f(x))^2}{\\prod\\limits_{i=0}^{1020} (x-g^i)}\\]

\n

whose denominator can be rewritten, so that the entire expression is easier to compute:

\n

$$\\frac{f(g^2 \\cdot x) - (f(g \\cdot x))^2 - (f(x))^2}{\\frac{x^{1024} - 1}{(x-g^{1021})(x-g^{1022})(x-g^{1023})}}$$

\n

This follows from the equality

\n

$$\\prod\\limits_{i=0}^{1023} (x-g^i) = x^{1024} - 1$$

\n

Step 4 - Composition Polynomial

Recall that we’re translating a problem of checking the validity of three polynomial constraints to checking that each of the rational functions \\(p_0, p_1, p_2\\) are polynomials.

\n

Our protocol uses an algorithm called FRI to do so, which will be discussed in the next part.

In order for the proof to be succinct (short), we prefer to work with just one rational function instead of three. For that, we take a random linear combination of \\(p_0, p_1, p_2\\) called the compostion polynomial (CP for short):

\n

$$CP(x) = \\alpha_0 \\cdot p_0(x) + \\alpha_1 \\cdot p_1(x) + \\alpha_2 \\cdot p_2(x)$$

\n

where \\(\\alpha_0, \\alpha_1, \\alpha_2 \\) are random field elements obtained from the verifier, or in our case - from the channel.

\n

Proving that (the rational function) \\(CP\\) is a polynomial guarantess, with high probability, that each of \\(p_0, p_1, p_2\\) are themselves polynomials.

\n

Commit on the Composition Polynomial

Lastly, we evaluate $cp$ over the evaluation domain (eval_domain), build a Merkle tree on top of that and send its root over the channel. This is similar to commiting on the LDE trace, as we did at the end of part 1.

\n

\"trace

\n

FRI Commitments

FRI Folding

Our goal in this part is to construct the FRI layers and commit on them.

To obtain each layer we need:

\n
    \n
  1. To generate a domain for the layer (from the previous layer’s domain).
    2. \n
  2. To generate a polynomial for the layer (from the previous layer’s polynomial and domain).
    3. \n
  3. To evaluate said polynomial on said domain - this is the next FRI layer.
    4. \n
\n

Domain Generation

The first FRI domain is simply the eval_domain that you already generated in Part 1, namely a coset of a group of order 8192. Each subsequent FRI domain is obtained by taking the first half of the previous FRI domain (dropping the second half), and squaring each of its elements.

\n

Formally - we got eval_domain by taking:

$$w, w\\cdot h, w\\cdot h^2, …, w\\cdot h^{8191}$$

\n

The next layer will therefore be:

$$w^2, (w\\cdot h)^2, (w\\cdot h^2)^2, …, (w\\cdot h^{4095})^2$$

\n

Note that taking the squares of the second half of each elements in eval_domain yields exactly
the same result as taking the squares of the first half. This is true for the next layers as well.

\n

Similarly, the domain of the third layer will be:

$$w^4, (w\\cdot h)^4, (w\\cdot h^2)^4, …, (w\\cdot h^{2047})^4$$

\n

And so on.

\n

FRI Folding Operator

The first FRI polynomial is simply the composition polynomial, i.e., cp.

Each subsequent FRI polynomial is obtained by:

\n
    \n
  1. Getting a random field element \\(\\beta\\) (by calling Channel.receive_random_field_element).
    2. \n
  2. Multiplying the odd coefficients of the previous polynomial by \\(\\beta\\).
    3. \n
  3. Summing together consecutive pairs (even-odd) of coefficients.
    4. \n
\n

Formally, let’s say that the k-th polynomial is of degree \\(< m\\) (for some \\(m\\) which is a power of 2):

\n

$$p_{k}(x) := \\sum _{i=0} ^{m-1} c_i x^i$$

\n

Then the (k+1)-th polynomial, whose degree is \\(< \\frac m 2 \\) will be:

\n

\\[ p_{k+1}(x) := \\sum_{i=0} ^{ m / 2 - 1 } (c_{2i} + \\beta \\cdot c_{2i + 1}) x^i \\]

\n

\"fri\"

\n

\"fri

\n

Generating FRI Commitments

We have now developed the tools to write the FriCommit method, that contains the main FRI commitment loop.

\n

It takes the following 5 arguments:

\n
    \n
  1. The composition polynomial, that is also the first FRI polynomial, that is - cp.
    2. \n
  2. The coset of order 8192 that is also the first FRI domain, that is - eval_domain.
    3. \n
  3. The evaluation of the former over the latter, which is also the first FRI layer , that is - cp_eval.
    4. \n
  4. The first Merkle tree (we will have one for each FRI layer) constructed from these evaluations, that is - cp_merkle.
    5. \n
  5. A channel object, that is channel.
    6. \n
\n

The method accordingly returns 4 lists:

\n
    \n
  1. The FRI polynomials.
    2. \n
  2. The FRI domains.
    3. \n
  3. The FRI layers.
    4. \n
  4. The FRI Merkle trees.
    5. \n
\n

The method contains a loop, in each iteration of which we extend these four lists, using the last element in each.
The iteration should stop once the last FRI polynomial is of degree 0, that is - when the last FRI polynomial is just a constant. It should then send over the channel this constant (i.e. - the polynomial’s free term).
The Channel class only supports sending strings, so make sure you convert anything you wish to send over the channel to a string before sending.

\n

\"commitment\"

\n

Query Phase

Get q random elements, provide a valdiation data for each

\n

\"decommitment\"

\n

references

\n","site":{"data":{}},"excerpt":"","more":"\n\n\n

trace and low degree extension

the objective is to develop a STARK prover for the FibonacciSq sequence over a finite field. The FibonacciSq sequence is defined by the recurrence relation
\\[ a_{n+2} = a_{n+1} ^2 + a_n ^2 \\]

\n

the statement is: I know a FieldElement \\(X\\in \\mathbb{F}\\) such that the 1023rd element of the FibonacciSq sequence starting with \\(1, X\\) is \\(2338775057\\)

\n

The underlying field of this class is \\(\\mathbb{F}_{3221225473}\\) (\\(3221225473 = 3 \\cdot 2^{30} + 1\\)), so all operations are done modulo 3221225473.

\n

FibonacciSq Trace

let’s construct a list a of length 1023, whose first two elements will be FieldElement objects representing 1 and 3141592, respectively. The next 1021 elements will be the FibonacciSq sequence induced by these two elements. a is called the trace of FibonacciSq, or, when the context is clear, the trace.

\n

Thinking of Polynomials

We now want to think of the sequence as the evaluation of some polynomial \\(f\\) of degree 1022.
We will choose the domain to be some subgroup \\(G \\subseteq \\mathbb{F}^\\times\\) of size 1024, for reasons that will become clear later.

\n

(Recall that \\(\\mathbb{F}^\\times\\) denotes the multiplicative group of \\(\\mathbb{F}\\), which we get from \\(\\mathbb{F}\\) by omitting the zero element with the induced multiplication from the field. A subgroup of size 1024 exists because \\(\\mathbb{F}^\\times\\) is a cyclic group of size \\(3\\cdot 2^{30}\\), so it contains a subgroup of size \\(2^i\\) for any \\(0 \\leq i \\leq 30\\)).

\n

Find a Group of Size 1024

If we find an element \\(g \\in \\mathbb{F}\\) whose (multiplicative) order is 1024, then \\(g\\) will generate such a group. Create a list called G with all the elements of \\(G\\), such that \\(G[i] := g^i\\).

\n

Evaluating on a Larger Domain

then, interpolating G over a we get a polynomial f. The trace, viewed as evaluations of a polynomial \\(f\\) on \\(G\\), can now be extended by evaluating \\(f\\) over a larger domain, thereby creating a Reed-Solomon error correction code.

\n

Cosets

To that end, we must decide on a larger domain on which \\(f\\) will be evaluated. We will work with a domain that is 8 times larger than \\(G\\).
A natural choice for such a domain is to take some group \\(H\\) of size 8192 (which exists because 8192 divides \\(|\\mathbb{F}^\\times|\\)), and shift it by the generator of \\(\\mathbb{F}^\\times\\), thereby obtaining a coset of \\(H\\).

\n

Create a list called H of the elements of \\(H\\), and multiply each of them by the generator of \\(\\mathbb{F}^\\times\\) to obtain a list called eval_domain. In other words, eval_domain = \\(\\{w\\cdot h^i | 0 \\leq i <8192 \\}\\) for \\(h\\) the generator of \\(H\\) and \\(w\\) the generator of \\(\\mathbb{F}^\\times\\).

\n

Evaluate on a Coset

1
2
f = interpolate_poly(G[:-1], a)
f_eval = [f(d) for d in eval_domain]
\n\n

Commitments

We will use Sha256-based Merkle Trees as our commitment scheme.

\n
1
2
from merkle import MerkleTree
f_merkle = MerkleTree(f_eval)
\n\n

Channel

Theoretically, a STARK proof system is a protocol for interaction between two parties - a prover and a verifier. In practice, we convert this interactive protocol into a non-interactive proof using the Fiat-Shamir Heuristic. In this tutorial you will use the Channel class, which implements this transformation. This channel replaces the verifier in the sense that the prover (which you are writing) will send data, and receive random numbers or random FieldElement instances.

\n

constraints

In this part, we are going to create a set of constraints over the trace a.

\n

Step 1 - FibonacciSq Constraints

For a to be a correct trace of a FibonacciSq sequence that proves our claim:

\n
    \n
  1. The first element has to be 1, namely \\(a[0] = 1\\).
    2. \n
  2. The last element has to be 2338775057, namely \\(a[1022] = 2338775057\\).
    3. \n
  3. The FibonacciSq rule must apply, that is - for every \\(i<1021\\), \\(a[i+2]=a[i+1]^2+a[i]^2\\).
    4. \n
\n

Step 2 - Polynomial Constraints

Recall that f is a polynomial over the trace domain, that evaluates exactly to a over \\(G \\setminus {g^{1023}}\\) where \\(G={g^i : 0\\leq i\\leq 1023}\\) is the “small” group generated by \\(g\\).

\n

We now rewrite the above three constraints in a form of polynomial constraints over f:

\n
    \n
  1. \\(a[0] = 1\\) is translated to the polynomial \\(f(x) - 1\\), which evalutes to 0 for \\(x = g^0\\) (note that \\(g^0\\) is \\(1\\)).
    2. \n
  2. \\(a[1022] = 2338775057\\) is translated to the polynomial \\(f(x) - 2338775057\\), which evalutes to 0 for \\(x = g^{1022}\\).
    3. \n
  3. \\(a[i+2]=a[i+1]^2+a[i]^2\\) for every \\(i<1021\\) is translated to the polynomial \\(f(g^2 \\cdot x) - (f(g \\cdot x))^2 - (f(x))^2\\), which evaluates to 0 for \\(x \\in G \\backslash {g^{1021}, g^{1022}, g^{1023}}\\).

    4. \n
\n

Step 3 - Rational Functions (That are in Fact Polynomials)

Each of the constraints above is represented by a polynomial \\(u(x)\\) that supposedly evaluates to \\(0\\) on certain elements of the group \\(G\\). That is, for some \\(x_0, \\ldots, x_k \\in G\\), we claim that

\n

\\[u(x_0) = \\ldots = u(x_k) = 0\\]

\n

(note that for the first two constaints, \\(k=0\\) because they only refer to one point and for the third \\(k=1021\\)).

\n

This is equivalent to saying that \\(u(x)\\) is divisible, as a polynomial, by all of \\({(x-x_i)}_{i=0}^k\\), or, equivalently, by

\n

\\[\\prod_{i=0}^k (x-x_i)\\]

\n

Therefore, each of the three constraints above can be written as a rational function of the form:

\n

\\[\\frac{u(x)}{\\prod_{i=0}^k (x-x_i)}\\]

\n

for the corresponding \\(u(x)\\) and \\({x_i}_{i=0}^k\\). In this step we will construct these three rational functions and show that they are indeed polynomials.

\n

The First Constraint:

In the first constraint, \\(f(x) - 1\\) and \\({x_i} = {1}\\).

\n

We will now construct the polynomial \\(p_0(x)=\\frac{f(x) - 1}{x - 1}\\), making sure that \\(f(x) - 1\\) is indeed divisible by \\((x-1)\\).

\n

The Second Constraint

Construct the polynomial p1 representing the second constraint, \\(p_1(x)= \\frac{f(x) - 2338775057}{x - g^{1022}}\\), similarly:

\n

The Third Constraint - Succinctness

The last constraint’s rational function is slightly more complicated:

\n

\\[p_2(x) = \\frac{f(g^2 \\cdot x) - (f(g \\cdot x))^2 - (f(x))^2}{\\prod\\limits_{i=0}^{1020} (x-g^i)}\\]

\n

whose denominator can be rewritten, so that the entire expression is easier to compute:

\n

$$\\frac{f(g^2 \\cdot x) - (f(g \\cdot x))^2 - (f(x))^2}{\\frac{x^{1024} - 1}{(x-g^{1021})(x-g^{1022})(x-g^{1023})}}$$

\n

This follows from the equality

\n

$$\\prod\\limits_{i=0}^{1023} (x-g^i) = x^{1024} - 1$$

\n

Step 4 - Composition Polynomial

Recall that we’re translating a problem of checking the validity of three polynomial constraints to checking that each of the rational functions \\(p_0, p_1, p_2\\) are polynomials.

\n

Our protocol uses an algorithm called FRI to do so, which will be discussed in the next part.

In order for the proof to be succinct (short), we prefer to work with just one rational function instead of three. For that, we take a random linear combination of \\(p_0, p_1, p_2\\) called the compostion polynomial (CP for short):

\n

$$CP(x) = \\alpha_0 \\cdot p_0(x) + \\alpha_1 \\cdot p_1(x) + \\alpha_2 \\cdot p_2(x)$$

\n

where \\(\\alpha_0, \\alpha_1, \\alpha_2 \\) are random field elements obtained from the verifier, or in our case - from the channel.

\n

Proving that (the rational function) \\(CP\\) is a polynomial guarantess, with high probability, that each of \\(p_0, p_1, p_2\\) are themselves polynomials.

\n

Commit on the Composition Polynomial

Lastly, we evaluate $cp$ over the evaluation domain (eval_domain), build a Merkle tree on top of that and send its root over the channel. This is similar to commiting on the LDE trace, as we did at the end of part 1.

\n

\"trace

\n

FRI Commitments

FRI Folding

Our goal in this part is to construct the FRI layers and commit on them.

To obtain each layer we need:

\n
    \n
  1. To generate a domain for the layer (from the previous layer’s domain).
    2. \n
  2. To generate a polynomial for the layer (from the previous layer’s polynomial and domain).
    3. \n
  3. To evaluate said polynomial on said domain - this is the next FRI layer.
    4. \n
\n

Domain Generation

The first FRI domain is simply the eval_domain that you already generated in Part 1, namely a coset of a group of order 8192. Each subsequent FRI domain is obtained by taking the first half of the previous FRI domain (dropping the second half), and squaring each of its elements.

\n

Formally - we got eval_domain by taking:

$$w, w\\cdot h, w\\cdot h^2, …, w\\cdot h^{8191}$$

\n

The next layer will therefore be:

$$w^2, (w\\cdot h)^2, (w\\cdot h^2)^2, …, (w\\cdot h^{4095})^2$$

\n

Note that taking the squares of the second half of each elements in eval_domain yields exactly
the same result as taking the squares of the first half. This is true for the next layers as well.

\n

Similarly, the domain of the third layer will be:

$$w^4, (w\\cdot h)^4, (w\\cdot h^2)^4, …, (w\\cdot h^{2047})^4$$

\n

And so on.

\n

FRI Folding Operator

The first FRI polynomial is simply the composition polynomial, i.e., cp.

Each subsequent FRI polynomial is obtained by:

\n
    \n
  1. Getting a random field element \\(\\beta\\) (by calling Channel.receive_random_field_element).
    2. \n
  2. Multiplying the odd coefficients of the previous polynomial by \\(\\beta\\).
    3. \n
  3. Summing together consecutive pairs (even-odd) of coefficients.
    4. \n
\n

Formally, let’s say that the k-th polynomial is of degree \\(< m\\) (for some \\(m\\) which is a power of 2):

\n

$$p_{k}(x) := \\sum _{i=0} ^{m-1} c_i x^i$$

\n

Then the (k+1)-th polynomial, whose degree is \\(< \\frac m 2 \\) will be:

\n

\\[ p_{k+1}(x) := \\sum_{i=0} ^{ m / 2 - 1 } (c_{2i} + \\beta \\cdot c_{2i + 1}) x^i \\]

\n

\"fri\"

\n

\"fri

\n

Generating FRI Commitments

We have now developed the tools to write the FriCommit method, that contains the main FRI commitment loop.

\n

It takes the following 5 arguments:

\n
    \n
  1. The composition polynomial, that is also the first FRI polynomial, that is - cp.
    2. \n
  2. The coset of order 8192 that is also the first FRI domain, that is - eval_domain.
    3. \n
  3. The evaluation of the former over the latter, which is also the first FRI layer , that is - cp_eval.
    4. \n
  4. The first Merkle tree (we will have one for each FRI layer) constructed from these evaluations, that is - cp_merkle.
    5. \n
  5. A channel object, that is channel.
    6. \n
\n

The method accordingly returns 4 lists:

\n
    \n
  1. The FRI polynomials.
    2. \n
  2. The FRI domains.
    3. \n
  3. The FRI layers.
    4. \n
  4. The FRI Merkle trees.
    5. \n
\n

The method contains a loop, in each iteration of which we extend these four lists, using the last element in each.
The iteration should stop once the last FRI polynomial is of degree 0, that is - when the last FRI polynomial is just a constant. It should then send over the channel this constant (i.e. - the polynomial’s free term).
The Channel class only supports sending strings, so make sure you convert anything you wish to send over the channel to a string before sending.

\n

\"commitment\"

\n

Query Phase

Get q random elements, provide a valdiation data for each

\n

\"decommitment\"

\n

references

\n"},{"title":"understanding plonk","date":"2023-11-01T09:19:04.000Z","_content":"\n\n\n\n# introduction\n[PLONK](https://eprint.iacr.org/2019/953), standing for the unwieldy quasi-backronym \"Permutations over Lagrange-bases for Oecumenical Noninteractive arguments of Knowledge\". PLONK still requires a \"universal and updateable\" trusted setup, meaning each program share the same setup. Secondly, there is a way for multiple parties to participate in the trusted setup such that it is secure as long as any one of them is honest.\n\n\n# preliminary\n## observation 1\na key fact: for non-zero \\\\( f \\in \\mathbb{F_p}^{\\le d}[X]\\\\)
\nfor \\\\( r \\leftarrow \\mathbb{F_p}: Pr[f(r) = 0] \\le d/p\\\\)\nunderstanding: polynomial \\\\(f\\\\) contains at most \\\\(d\\\\) roots of zeros.
\nsuppose \\\\( p \\approx 2^{256}\\\\) and \\\\( d \\approx 2^{40}\\\\) then \\\\(d/p\\\\) is negligible.
\nTherefore, for random \\\\( r \\leftarrow \\mathbb{F_p}\\\\) if \\\\(f(r) = 0\\\\) then \\\\(f\\\\) is identically zero w.h.p (with high probability)\n\n=> a simple zero test for a committed polynomial\n**Note** SZDL lemma: this also holds for multivariate polynomials (where d is total degree of f) \n\n## observation 2\nlet \\\\(f,g \\in \\mathbb{F_p}^{\\le d}[X]\\\\).\nfor \\\\( r \\leftarrow \\mathbb{F_p}\\\\), if \\\\(f(r) = g(r)\\\\), then \\\\(f = g\\\\) w.h.p
\n\\\\(f(r)-g(r)=0\\\\) => \\\\(f-g=0\\\\) w.h.p
\n\n=> a simple equality test for two committed polynomials\n\n# useful proof gadgets\n## 1. zero test\nlet \\\\( \\omega \\in \\mathbb{F_p} \\\\) be a primitive k-th root of unity \\\\(( \\omega ^{k} = 1)\\\\)\nset \\\\( H:= \\\\{1,\\omega,\\omega^2,\\omega^3,...,\\omega^{k-1}\\\\} \\subseteq \\mathbb{F_p} \\\\)\nlet \\\\( f \\in \\mathbb{F_p}^{\\le d}[X]\\\\) and \\\\( b, c \\in \\mathbb{F_p}\\\\) \\\\((d \\ge k)\\\\)\n\ntask: prove that \\\\(f\\\\) is identically zero on \\\\(H\\\\)\n![zero test](images/zkp/plonk/zero_test.png)\n\n**info** the box of \\\\(f\\\\) means the commitment of polynomial \\\\(f\\\\), i.e \\\\(com_f\\\\)\n\n## 2. product check\nproduct check on \\\\(\\Omega: \\quad \\prod_{a\\in \\Omega} f(a) = 1\\\\)\nSet \\\\( t \\in \\mathbb{F_p}^{\\le k}[X]\\\\) to be the degree-k polynomial:\n\\\\[ t(1) = f(\\omega^0) = f(1), \\quad t(\\omega^s) = \\prod_{i=0}^{s}f(\\omega^{i}) \\quad for \\quad s = 1,..., k-1\\\\]\nThen \n\\\\(t(\\omega) = f(1) \\cdot f(\\omega), \\quad t(\\omega^2) = f(1) \\cdot f(\\omega) \\cdot f(\\omega^2), ... \\\\)\n\\\\(t( \\omega^{k-1}) = \\prod_{a \\in \\Omega}f(a) = 1\\\\)\nand \\\\( t(\\omega \\cdot x) = t(x) \\cdot f(\\omega \\cdot x) \\\\) for all \\\\(x \\in \\Omega \\\\) (including at \\\\(x = \\omega^{k-1}\\\\))\n![prod_check_lemma](/images/zkp/plonk/prod_check_lemma.png)\n![prod_check_prove_and_verify](/images/zkp/plonk/prod_check_prove_verify.png)\n\nSame works for rational functions: \\\\( \\prod_{a \\in \\Omega}{f/g}(a) =1 \\\\)\nThe proof is similar\n\n## 3. permutation check\nlet \\\\(f,g\\\\) be polynomials in \\\\(\\mathbb{F_p}^{\\le d}[X]\\\\). Verifier has \\\\(com_f, com_g\\\\).\nProver wants to prove that \\\\((f(1),f(\\omega^1),f(\\omega^2),...,f(\\omega^{k-1})) \\in \\mathbb{F_p}^{\\le k}[X]\\\\) is a permutaion of \\\\((g(1),g(\\omega^1),g(\\omega^2),...,g(\\omega^{k-1})) \\in \\mathbb{F_p}^{\\le k}[X]\\\\)\n\n![permutation check](/images/zkp/plonk/permutation_check.png)\n\n## 4. prescribed permutation check\n![](/images/zkp/plonk/prescribed_perm_check_problem.png)\n![](/images/zkp/plonk/prescribed_perm_check_problem_quadratic.png)\n![](/images/zkp/plonk/prescribed_perm_check_problem_reduce.png)\n![](/images/zkp/plonk/prescribed_perm_check_problem_prove_verify.png)\n![](/images/zkp/plonk/prescribed_perm_check_problem_complete.png)\n\n# PLONK: a poly-IOP for a general circuit C(x,w)\n## step 1: compile circuit to a computation trace (gate fan-in = 2)\n![circuit to trace](images/zkp/plonk/plonk_circuit_to_trace.png)\n\nand encode the trace as polynomial\nlet \\\\(|C|\\\\) be the total number of gates, \\\\(|I| := |I_x| + |I_w|\\\\) be total number of inputs, where \\\\(|I_x|\\\\) is the number of public inputs, \\\\(I_w\\\\) is the number of private inputs.\nLet \\\\(d=3*|C|+|I|\\\\) and \\\\( \\Omega=\\\\{1,\\omega,\\omega^2,\\omega^3,...,\\omega^{d-1}\\\\} \\\\)\n\nprover interpolates \\\\( T \\in \\mathbb{F_p}^{\\le d}[X]\\\\) such that\n- **T encodes all inputs**: \\\\( T(\\omega^{-j}) \\\\)= input #j, for j = 1,...,|I|\n- **T encodes all wires**: \n\\\\(LeftInput=f(\\omega^{3l})\\\\), \\\\( RightInput=f(\\omega^{3l+1})\\\\), \\\\(Output=f(\\omega^{3l+2})\\\\), for \\\\(l = 0,1,..., |C| -1\\\\)\nFor the example,\n**inputs**\n\\\\(x_1= 5 = T(\\omega^9)\\\\), \\\\(x_2= 6 = T(\\omega^{10})\\\\), and \\\\(w_1 = 1=T(\\omega^{11})\\\\)\n**wires**\n\\\\(5=T(\\omega^0)\\\\), \\\\(6=T(\\omega^{1})\\\\), and \\\\(11=T(\\omega^{2})\\\\)\n\\\\(6=T(\\omega^3)\\\\), \\\\(1=T(\\omega^{4})\\\\), and \\\\(7=T(\\omega^{5})\\\\)\n\\\\(11=T(\\omega^6)\\\\), \\\\(7=T(\\omega^{7})\\\\), and \\\\(77=T(\\omega^{8})\\\\)\n\n\n## step 2: proving validity of T\nProver needs to prove 4 things\n1. **\\\\(T(x)\\\\) encodes the correct public inputs**\nBoth prover and verifier interpolate a polynomial \\\\( v(x) \\in \\mathbb{F_p}^{\\le |I_x|}[X]\\\\)\nthat encodes the \\\\(x\\\\)-inputs to the circuit:\n\\\\(v(\\omega^{-j}) =\\\\) input #j, for \\\\(j = 1, ..., |I_x|\\\\)\nIn our example, \\\\(v(\\omega^{-1} = 5), v(\\omega^{-2} = 6)\\\\)\nLet \\\\( \\Omega_{inp}=\\\\{\\omega^{-1},\\omega^{-2},...,\\omega^{-|I_x|}\\\\} \\\\)\nProver proves by using a **ZeroTest** on \\\\(\\Omega_inp\\\\) to prove that\n\\\\[T(y) - v(y) =0 \\quad \\forall y \\in \\Omega_{inp}\\\\]\n2. **every gate is evaluated correctly**\n**Idea** encode gate types using a selector polynomial \\\\(S(X)\\\\)\ndefine \\\\(S(X) \\in \\mathbb{F_p}^{\\le d}[X]\\\\) such that \\\\( \\forall l = 0, ..., |C| -1\\\\):\n- \\\\(S(\\omega^{3l}) =1\\\\) if gate #l is an addition gate\n- \\\\(S(\\omega^{3l}) =0\\\\) if gate #l is a multiplication gate\n\nThen, \\\\( \\forall y \\in \\Omega_{gates} : = \\\\{1,\\omega^{3},\\omega^{6},...,\\omega^{3(|C|-1)}\\\\} \\\\)\n\\\\(S(y) \\cdot [T(y) + T(\\omega y)] + (1-S(y))\\cdot T(y) \\cdot T(\\omega y) = T(\\omega^2 y)\\\\)\n![gate_evaluation_zero_test](/images/zkp/plonk/gate_evaluation_zero_test.png)\n\n3. **the wiring is implemented correctly (coppy constraint)**\n![](/images/zkp/plonk/copy_constraint_example.png)\n\n \\\\(T(\\omega^9,\\omega^0)=\\sigma(\\omega^0,\\omega^9)\\\\)\n \\\\(T(\\omega^{10},\\omega^1,\\omega^3)=\\sigma(\\omega^1,\\omega^3,\\omega^{10})\\\\)\n \\\\(T(\\omega^2,\\omega^6)=\\sigma(\\omega^6,\\omega^2)\\\\)\n \\\\(T(\\omega^{11},\\omega^4)=\\sigma(\\omega^4,\\omega^{11})\\\\)\n \\\\(T(\\omega^{5},\\omega^7)=\\sigma(\\omega^7,\\omega^{5})\\\\)\n**note**: 9 is actually -1, 10 is -2, 11 is -3\nDefine a polynomial \\\\(W: \\Omega -> \\Omega\\\\) that implemnets a rotation\n\\\\( W(\\omega^{10}, \\omega^1, \\omega^3) =(\\omega^1, \\omega^3, \\omega^{10}) \\\\), \\\\(W(\\omega^{9}, \\omega^0)=(\\omega^0, \\omega^{9})\\\\), ...\n\n**Lemma**: \\\\(\\forall y \\in \\Omega: T(y) = T(W(y))\\\\) => wire constraints are satisfied\nThis could be proved using a prescribed permutation check\n\n4. **the output of last gate is 0**\nthis is to prove \\\\(T(\\omega^8) -77 = 0\\\\)\n\n# custom gate\n![](/images/zkp/plonk/custom_gate.png)\n\\\\(u, v, w, t, r\\\\) are polynomials represent input variables (row number is the gate number). in the `Add`, `Mul` only circuits, there are only two inputs, namely `LeftInput` and `RightInput`. Hoever, here there are multiple inputs for custom gate. \n\nIn the above example, it is a constraint for \\\\( v_4 + w_3 \\cdot t_3 - t_4 = 0 \\\\)\n\n\n# plonkup\nplonkup is to ensure some values are in a pre-defined list. for example\n\n| x1 | x2 | x3 | Output |\n| --- | --- | --- | --- |\n| \\\\(a_{1,1}\\\\) | \\\\(a_{1,2}\\\\) | \\\\(a_{1,3}\\\\) | \\\\(a_{1,4}\\\\) |\n| \\\\(a_{2,1}\\\\) | \\\\(a_{2,2}\\\\) | \\\\(a_{2,3}\\\\) | \\\\(a_{2,4}\\\\) |\n| ... | ... | ... | ... |\n| \\\\(a_{n,1}\\\\) | \\\\(a_{n,2}\\\\) | \\\\(a_{n,3}\\\\) | \\\\(a_{n,4}\\\\) |\n\n\n\n\\\\(n\\\\) is gate number. the task is to prove a vector \n## references\n- https://hackmd.io/@learn-zkp/note-plonk-family\n- [ZKP MOOC Lecture 5: The Plonk SNARK](https://www.youtube.com/watch?v=A0oZVEXav24)\n- [CS251.stanford lecture](https://cs251.stanford.edu/lectures/lecture15.pdf)\n\n\n","source":"_posts/cryptography/zkp/understanding-plonk.md","raw":"---\ntitle: understanding plonk\ndate: 2023-11-01 17:19:04\ntags: [cryptography,zkp]\n---\n\n\n\n\n# introduction\n[PLONK](https://eprint.iacr.org/2019/953), standing for the unwieldy quasi-backronym \"Permutations over Lagrange-bases for Oecumenical Noninteractive arguments of Knowledge\". PLONK still requires a \"universal and updateable\" trusted setup, meaning each program share the same setup. Secondly, there is a way for multiple parties to participate in the trusted setup such that it is secure as long as any one of them is honest.\n\n\n# preliminary\n## observation 1\na key fact: for non-zero \\\\( f \\in \\mathbb{F_p}^{\\le d}[X]\\\\)
\nfor \\\\( r \\leftarrow \\mathbb{F_p}: Pr[f(r) = 0] \\le d/p\\\\)\nunderstanding: polynomial \\\\(f\\\\) contains at most \\\\(d\\\\) roots of zeros.
\nsuppose \\\\( p \\approx 2^{256}\\\\) and \\\\( d \\approx 2^{40}\\\\) then \\\\(d/p\\\\) is negligible.
\nTherefore, for random \\\\( r \\leftarrow \\mathbb{F_p}\\\\) if \\\\(f(r) = 0\\\\) then \\\\(f\\\\) is identically zero w.h.p (with high probability)\n\n=> a simple zero test for a committed polynomial\n**Note** SZDL lemma: this also holds for multivariate polynomials (where d is total degree of f) \n\n## observation 2\nlet \\\\(f,g \\in \\mathbb{F_p}^{\\le d}[X]\\\\).\nfor \\\\( r \\leftarrow \\mathbb{F_p}\\\\), if \\\\(f(r) = g(r)\\\\), then \\\\(f = g\\\\) w.h.p
\n\\\\(f(r)-g(r)=0\\\\) => \\\\(f-g=0\\\\) w.h.p
\n\n=> a simple equality test for two committed polynomials\n\n# useful proof gadgets\n## 1. zero test\nlet \\\\( \\omega \\in \\mathbb{F_p} \\\\) be a primitive k-th root of unity \\\\(( \\omega ^{k} = 1)\\\\)\nset \\\\( H:= \\\\{1,\\omega,\\omega^2,\\omega^3,...,\\omega^{k-1}\\\\} \\subseteq \\mathbb{F_p} \\\\)\nlet \\\\( f \\in \\mathbb{F_p}^{\\le d}[X]\\\\) and \\\\( b, c \\in \\mathbb{F_p}\\\\) \\\\((d \\ge k)\\\\)\n\ntask: prove that \\\\(f\\\\) is identically zero on \\\\(H\\\\)\n![zero test](images/zkp/plonk/zero_test.png)\n\n**info** the box of \\\\(f\\\\) means the commitment of polynomial \\\\(f\\\\), i.e \\\\(com_f\\\\)\n\n## 2. product check\nproduct check on \\\\(\\Omega: \\quad \\prod_{a\\in \\Omega} f(a) = 1\\\\)\nSet \\\\( t \\in \\mathbb{F_p}^{\\le k}[X]\\\\) to be the degree-k polynomial:\n\\\\[ t(1) = f(\\omega^0) = f(1), \\quad t(\\omega^s) = \\prod_{i=0}^{s}f(\\omega^{i}) \\quad for \\quad s = 1,..., k-1\\\\]\nThen \n\\\\(t(\\omega) = f(1) \\cdot f(\\omega), \\quad t(\\omega^2) = f(1) \\cdot f(\\omega) \\cdot f(\\omega^2), ... \\\\)\n\\\\(t( \\omega^{k-1}) = \\prod_{a \\in \\Omega}f(a) = 1\\\\)\nand \\\\( t(\\omega \\cdot x) = t(x) \\cdot f(\\omega \\cdot x) \\\\) for all \\\\(x \\in \\Omega \\\\) (including at \\\\(x = \\omega^{k-1}\\\\))\n![prod_check_lemma](/images/zkp/plonk/prod_check_lemma.png)\n![prod_check_prove_and_verify](/images/zkp/plonk/prod_check_prove_verify.png)\n\nSame works for rational functions: \\\\( \\prod_{a \\in \\Omega}{f/g}(a) =1 \\\\)\nThe proof is similar\n\n## 3. permutation check\nlet \\\\(f,g\\\\) be polynomials in \\\\(\\mathbb{F_p}^{\\le d}[X]\\\\). Verifier has \\\\(com_f, com_g\\\\).\nProver wants to prove that \\\\((f(1),f(\\omega^1),f(\\omega^2),...,f(\\omega^{k-1})) \\in \\mathbb{F_p}^{\\le k}[X]\\\\) is a permutaion of \\\\((g(1),g(\\omega^1),g(\\omega^2),...,g(\\omega^{k-1})) \\in \\mathbb{F_p}^{\\le k}[X]\\\\)\n\n![permutation check](/images/zkp/plonk/permutation_check.png)\n\n## 4. prescribed permutation check\n![](/images/zkp/plonk/prescribed_perm_check_problem.png)\n![](/images/zkp/plonk/prescribed_perm_check_problem_quadratic.png)\n![](/images/zkp/plonk/prescribed_perm_check_problem_reduce.png)\n![](/images/zkp/plonk/prescribed_perm_check_problem_prove_verify.png)\n![](/images/zkp/plonk/prescribed_perm_check_problem_complete.png)\n\n# PLONK: a poly-IOP for a general circuit C(x,w)\n## step 1: compile circuit to a computation trace (gate fan-in = 2)\n![circuit to trace](images/zkp/plonk/plonk_circuit_to_trace.png)\n\nand encode the trace as polynomial\nlet \\\\(|C|\\\\) be the total number of gates, \\\\(|I| := |I_x| + |I_w|\\\\) be total number of inputs, where \\\\(|I_x|\\\\) is the number of public inputs, \\\\(I_w\\\\) is the number of private inputs.\nLet \\\\(d=3*|C|+|I|\\\\) and \\\\( \\Omega=\\\\{1,\\omega,\\omega^2,\\omega^3,...,\\omega^{d-1}\\\\} \\\\)\n\nprover interpolates \\\\( T \\in \\mathbb{F_p}^{\\le d}[X]\\\\) such that\n- **T encodes all inputs**: \\\\( T(\\omega^{-j}) \\\\)= input #j, for j = 1,...,|I|\n- **T encodes all wires**: \n\\\\(LeftInput=f(\\omega^{3l})\\\\), \\\\( RightInput=f(\\omega^{3l+1})\\\\), \\\\(Output=f(\\omega^{3l+2})\\\\), for \\\\(l = 0,1,..., |C| -1\\\\)\nFor the example,\n**inputs**\n\\\\(x_1= 5 = T(\\omega^9)\\\\), \\\\(x_2= 6 = T(\\omega^{10})\\\\), and \\\\(w_1 = 1=T(\\omega^{11})\\\\)\n**wires**\n\\\\(5=T(\\omega^0)\\\\), \\\\(6=T(\\omega^{1})\\\\), and \\\\(11=T(\\omega^{2})\\\\)\n\\\\(6=T(\\omega^3)\\\\), \\\\(1=T(\\omega^{4})\\\\), and \\\\(7=T(\\omega^{5})\\\\)\n\\\\(11=T(\\omega^6)\\\\), \\\\(7=T(\\omega^{7})\\\\), and \\\\(77=T(\\omega^{8})\\\\)\n\n\n## step 2: proving validity of T\nProver needs to prove 4 things\n1. **\\\\(T(x)\\\\) encodes the correct public inputs**\nBoth prover and verifier interpolate a polynomial \\\\( v(x) \\in \\mathbb{F_p}^{\\le |I_x|}[X]\\\\)\nthat encodes the \\\\(x\\\\)-inputs to the circuit:\n\\\\(v(\\omega^{-j}) =\\\\) input #j, for \\\\(j = 1, ..., |I_x|\\\\)\nIn our example, \\\\(v(\\omega^{-1} = 5), v(\\omega^{-2} = 6)\\\\)\nLet \\\\( \\Omega_{inp}=\\\\{\\omega^{-1},\\omega^{-2},...,\\omega^{-|I_x|}\\\\} \\\\)\nProver proves by using a **ZeroTest** on \\\\(\\Omega_inp\\\\) to prove that\n\\\\[T(y) - v(y) =0 \\quad \\forall y \\in \\Omega_{inp}\\\\]\n2. **every gate is evaluated correctly**\n**Idea** encode gate types using a selector polynomial \\\\(S(X)\\\\)\ndefine \\\\(S(X) \\in \\mathbb{F_p}^{\\le d}[X]\\\\) such that \\\\( \\forall l = 0, ..., |C| -1\\\\):\n- \\\\(S(\\omega^{3l}) =1\\\\) if gate #l is an addition gate\n- \\\\(S(\\omega^{3l}) =0\\\\) if gate #l is a multiplication gate\n\nThen, \\\\( \\forall y \\in \\Omega_{gates} : = \\\\{1,\\omega^{3},\\omega^{6},...,\\omega^{3(|C|-1)}\\\\} \\\\)\n\\\\(S(y) \\cdot [T(y) + T(\\omega y)] + (1-S(y))\\cdot T(y) \\cdot T(\\omega y) = T(\\omega^2 y)\\\\)\n![gate_evaluation_zero_test](/images/zkp/plonk/gate_evaluation_zero_test.png)\n\n3. **the wiring is implemented correctly (coppy constraint)**\n![](/images/zkp/plonk/copy_constraint_example.png)\n\n \\\\(T(\\omega^9,\\omega^0)=\\sigma(\\omega^0,\\omega^9)\\\\)\n \\\\(T(\\omega^{10},\\omega^1,\\omega^3)=\\sigma(\\omega^1,\\omega^3,\\omega^{10})\\\\)\n \\\\(T(\\omega^2,\\omega^6)=\\sigma(\\omega^6,\\omega^2)\\\\)\n \\\\(T(\\omega^{11},\\omega^4)=\\sigma(\\omega^4,\\omega^{11})\\\\)\n \\\\(T(\\omega^{5},\\omega^7)=\\sigma(\\omega^7,\\omega^{5})\\\\)\n**note**: 9 is actually -1, 10 is -2, 11 is -3\nDefine a polynomial \\\\(W: \\Omega -> \\Omega\\\\) that implemnets a rotation\n\\\\( W(\\omega^{10}, \\omega^1, \\omega^3) =(\\omega^1, \\omega^3, \\omega^{10}) \\\\), \\\\(W(\\omega^{9}, \\omega^0)=(\\omega^0, \\omega^{9})\\\\), ...\n\n**Lemma**: \\\\(\\forall y \\in \\Omega: T(y) = T(W(y))\\\\) => wire constraints are satisfied\nThis could be proved using a prescribed permutation check\n\n4. **the output of last gate is 0**\nthis is to prove \\\\(T(\\omega^8) -77 = 0\\\\)\n\n# custom gate\n![](/images/zkp/plonk/custom_gate.png)\n\\\\(u, v, w, t, r\\\\) are polynomials represent input variables (row number is the gate number). in the `Add`, `Mul` only circuits, there are only two inputs, namely `LeftInput` and `RightInput`. Hoever, here there are multiple inputs for custom gate. \n\nIn the above example, it is a constraint for \\\\( v_4 + w_3 \\cdot t_3 - t_4 = 0 \\\\)\n\n\n# plonkup\nplonkup is to ensure some values are in a pre-defined list. for example\n\n| x1 | x2 | x3 | Output |\n| --- | --- | --- | --- |\n| \\\\(a_{1,1}\\\\) | \\\\(a_{1,2}\\\\) | \\\\(a_{1,3}\\\\) | \\\\(a_{1,4}\\\\) |\n| \\\\(a_{2,1}\\\\) | \\\\(a_{2,2}\\\\) | \\\\(a_{2,3}\\\\) | \\\\(a_{2,4}\\\\) |\n| ... | ... | ... | ... |\n| \\\\(a_{n,1}\\\\) | \\\\(a_{n,2}\\\\) | \\\\(a_{n,3}\\\\) | \\\\(a_{n,4}\\\\) |\n\n\n\n\\\\(n\\\\) is gate number. the task is to prove a vector \n## references\n- https://hackmd.io/@learn-zkp/note-plonk-family\n- [ZKP MOOC Lecture 5: The Plonk SNARK](https://www.youtube.com/watch?v=A0oZVEXav24)\n- [CS251.stanford lecture](https://cs251.stanford.edu/lectures/lecture15.pdf)\n\n\n","slug":"cryptography/zkp/understanding-plonk","published":1,"updated":"2023-11-29T03:24:45.714Z","_id":"clphpvdok000baj7u3i3te2rg","comments":1,"layout":"post","photos":[],"link":"","content":"\n\n\n

introduction

PLONK, standing for the unwieldy quasi-backronym “Permutations over Lagrange-bases for Oecumenical Noninteractive arguments of Knowledge”. PLONK still requires a “universal and updateable” trusted setup, meaning each program share the same setup. Secondly, there is a way for multiple parties to participate in the trusted setup such that it is secure as long as any one of them is honest.

\n

preliminary

observation 1

a key fact: for non-zero \\( f \\in \\mathbb{F_p}^{\\le d}[X]\\)

for \\( r \\leftarrow \\mathbb{F_p}: Pr[f(r) = 0] \\le d/p\\)
understanding: polynomial \\(f\\) contains at most \\(d\\) roots of zeros.

suppose \\( p \\approx 2^{256}\\) and \\( d \\approx 2^{40}\\) then \\(d/p\\) is negligible.

Therefore, for random \\( r \\leftarrow \\mathbb{F_p}\\) if \\(f(r) = 0\\) then \\(f\\) is identically zero w.h.p (with high probability)

\n

=> a simple zero test for a committed polynomial
Note SZDL lemma: this also holds for multivariate polynomials (where d is total degree of f)

\n

observation 2

let \\(f,g \\in \\mathbb{F_p}^{\\le d}[X]\\).
for \\( r \\leftarrow \\mathbb{F_p}\\), if \\(f(r) = g(r)\\), then \\(f = g\\) w.h.p

\\(f(r)-g(r)=0\\) => \\(f-g=0\\) w.h.p

\n

=> a simple equality test for two committed polynomials

\n

useful proof gadgets

1. zero test

let \\( \\omega \\in \\mathbb{F_p} \\) be a primitive k-th root of unity \\(( \\omega ^{k} = 1)\\)
set \\( H:= \\{1,\\omega,\\omega^2,\\omega^3,…,\\omega^{k-1}\\} \\subseteq \\mathbb{F_p} \\)
let \\( f \\in \\mathbb{F_p}^{\\le d}[X]\\) and \\( b, c \\in \\mathbb{F_p}\\) \\((d \\ge k)\\)

\n

task: prove that \\(f\\) is identically zero on \\(H\\)
\"zero

\n

info the box of \\(f\\) means the commitment of polynomial \\(f\\), i.e \\(com_f\\)

\n

2. product check

product check on \\(\\Omega: \\quad \\prod_{a\\in \\Omega} f(a) = 1\\)
Set \\( t \\in \\mathbb{F_p}^{\\le k}[X]\\) to be the degree-k polynomial:
\\[ t(1) = f(\\omega^0) = f(1), \\quad t(\\omega^s) = \\prod_{i=0}^{s}f(\\omega^{i}) \\quad for \\quad s = 1,…, k-1\\]
Then
\\(t(\\omega) = f(1) \\cdot f(\\omega), \\quad t(\\omega^2) = f(1) \\cdot f(\\omega) \\cdot f(\\omega^2), … \\)
\\(t( \\omega^{k-1}) = \\prod_{a \\in \\Omega}f(a) = 1\\)
and \\( t(\\omega \\cdot x) = t(x) \\cdot f(\\omega \\cdot x) \\) for all \\(x \\in \\Omega \\) (including at \\(x = \\omega^{k-1}\\))
\"prod_check_lemma\"
\"prod_check_prove_and_verify\"

\n

Same works for rational functions: \\( \\prod_{a \\in \\Omega}{f/g}(a) =1 \\)
The proof is similar

\n

3. permutation check

let \\(f,g\\) be polynomials in \\(\\mathbb{F_p}^{\\le d}[X]\\). Verifier has \\(com_f, com_g\\).
Prover wants to prove that \\((f(1),f(\\omega^1),f(\\omega^2),…,f(\\omega^{k-1})) \\in \\mathbb{F_p}^{\\le k}[X]\\) is a permutaion of \\((g(1),g(\\omega^1),g(\\omega^2),…,g(\\omega^{k-1})) \\in \\mathbb{F_p}^{\\le k}[X]\\)

\n

\"permutation

\n

4. prescribed permutation check





\n

PLONK: a poly-IOP for a general circuit C(x,w)

step 1: compile circuit to a computation trace (gate fan-in = 2)

\"circuit

\n

and encode the trace as polynomial
let \\(|C|\\) be the total number of gates, \\(|I| := |I_x| + |I_w|\\) be total number of inputs, where \\(|I_x|\\) is the number of public inputs, \\(I_w\\) is the number of private inputs.
Let \\(d=3*|C|+|I|\\) and \\( \\Omega=\\{1,\\omega,\\omega^2,\\omega^3,…,\\omega^{d-1}\\} \\)

\n

prover interpolates \\( T \\in \\mathbb{F_p}^{\\le d}[X]\\) such that

\n
    \n
  • T encodes all inputs: \\( T(\\omega^{-j}) \\)= input #j, for j = 1,…,|I|
    • \n
  • T encodes all wires:
    \\(LeftInput=f(\\omega^{3l})\\), \\( RightInput=f(\\omega^{3l+1})\\), \\(Output=f(\\omega^{3l+2})\\), for \\(l = 0,1,…, |C| -1\\)
    For the example,
    inputs
    \\(x_1= 5 = T(\\omega^9)\\), \\(x_2= 6 = T(\\omega^{10})\\), and \\(w_1 = 1=T(\\omega^{11})\\)
    wires
    \\(5=T(\\omega^0)\\), \\(6=T(\\omega^{1})\\), and \\(11=T(\\omega^{2})\\)
    \\(6=T(\\omega^3)\\), \\(1=T(\\omega^{4})\\), and \\(7=T(\\omega^{5})\\)
    \\(11=T(\\omega^6)\\), \\(7=T(\\omega^{7})\\), and \\(77=T(\\omega^{8})\\)
    • \n
\n

step 2: proving validity of T

Prover needs to prove 4 things

\n
    \n
  1. \\(T(x)\\) encodes the correct public inputs
    Both prover and verifier interpolate a polynomial \\( v(x) \\in \\mathbb{F_p}^{\\le |I_x|}[X]\\)
    that encodes the \\(x\\)-inputs to the circuit:
    \\(v(\\omega^{-j}) =\\) input #j, for \\(j = 1, …, |I_x|\\)
    In our example, \\(v(\\omega^{-1} = 5), v(\\omega^{-2} = 6)\\)
    Let \\( \\Omega_{inp}=\\{\\omega^{-1},\\omega^{-2},…,\\omega^{-|I_x|}\\} \\)
    Prover proves by using a ZeroTest on \\(\\Omega_inp\\) to prove that
    \\[T(y) - v(y) =0 \\quad \\forall y \\in \\Omega_{inp}\\]
    2. \n
  2. every gate is evaluated correctly
    Idea encode gate types using a selector polynomial \\(S(X)\\)
    define \\(S(X) \\in \\mathbb{F_p}^{\\le d}[X]\\) such that \\( \\forall l = 0, …, |C| -1\\):
    3. \n
\n
    \n
  • \\(S(\\omega^{3l}) =1\\) if gate #l is an addition gate
    • \n
  • \\(S(\\omega^{3l}) =0\\) if gate #l is a multiplication gate
    • \n
\n

Then, \\( \\forall y \\in \\Omega_{gates} : = \\{1,\\omega^{3},\\omega^{6},…,\\omega^{3(|C|-1)}\\} \\)
\\(S(y) \\cdot [T(y) + T(\\omega y)] + (1-S(y))\\cdot T(y) \\cdot T(\\omega y) = T(\\omega^2 y)\\)
\"gate_evaluation_zero_test\"

\n
    \n
  1. the wiring is implemented correctly (coppy constraint)
    2. \n
\n

\\(T(\\omega^9,\\omega^0)=\\sigma(\\omega^0,\\omega^9)\\)
\\(T(\\omega^{10},\\omega^1,\\omega^3)=\\sigma(\\omega^1,\\omega^3,\\omega^{10})\\)
\\(T(\\omega^2,\\omega^6)=\\sigma(\\omega^6,\\omega^2)\\)
\\(T(\\omega^{11},\\omega^4)=\\sigma(\\omega^4,\\omega^{11})\\)
\\(T(\\omega^{5},\\omega^7)=\\sigma(\\omega^7,\\omega^{5})\\)
note: 9 is actually -1, 10 is -2, 11 is -3
Define a polynomial \\(W: \\Omega -> \\Omega\\) that implemnets a rotation
\\( W(\\omega^{10}, \\omega^1, \\omega^3) =(\\omega^1, \\omega^3, \\omega^{10}) \\), \\(W(\\omega^{9}, \\omega^0)=(\\omega^0, \\omega^{9})\\), …

\n

Lemma: \\(\\forall y \\in \\Omega: T(y) = T(W(y))\\) => wire constraints are satisfied
This could be proved using a prescribed permutation check

\n
    \n
  1. the output of last gate is 0
    this is to prove \\(T(\\omega^8) -77 = 0\\)
    2. \n
\n

custom gate


\\(u, v, w, t, r\\) are polynomials represent input variables (row number is the gate number). in the Add, Mul only circuits, there are only two inputs, namely LeftInput and RightInput. Hoever, here there are multiple inputs for custom gate.

\n

In the above example, it is a constraint for \\( v_4 + w_3 \\cdot t_3 - t_4 = 0 \\)

\n

plonkup

plonkup is to ensure some values are in a pre-defined list. for example

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
x1x2x3Output
\\(a_{1,1}\\)\\(a_{1,2}\\)\\(a_{1,3}\\)\\(a_{1,4}\\)
\\(a_{2,1}\\)\\(a_{2,2}\\)\\(a_{2,3}\\)\\(a_{2,4}\\)
…………
\\(a_{n,1}\\)\\(a_{n,2}\\)\\(a_{n,3}\\)\\(a_{n,4}\\)
\n

\\(n\\) is gate number. the task is to prove a vector

\n

references

\n","site":{"data":{}},"excerpt":"","more":"\n\n\n

introduction

PLONK, standing for the unwieldy quasi-backronym “Permutations over Lagrange-bases for Oecumenical Noninteractive arguments of Knowledge”. PLONK still requires a “universal and updateable” trusted setup, meaning each program share the same setup. Secondly, there is a way for multiple parties to participate in the trusted setup such that it is secure as long as any one of them is honest.

\n

preliminary

observation 1

a key fact: for non-zero \\( f \\in \\mathbb{F_p}^{\\le d}[X]\\)

for \\( r \\leftarrow \\mathbb{F_p}: Pr[f(r) = 0] \\le d/p\\)
understanding: polynomial \\(f\\) contains at most \\(d\\) roots of zeros.

suppose \\( p \\approx 2^{256}\\) and \\( d \\approx 2^{40}\\) then \\(d/p\\) is negligible.

Therefore, for random \\( r \\leftarrow \\mathbb{F_p}\\) if \\(f(r) = 0\\) then \\(f\\) is identically zero w.h.p (with high probability)

\n

=> a simple zero test for a committed polynomial
Note SZDL lemma: this also holds for multivariate polynomials (where d is total degree of f)

\n

observation 2

let \\(f,g \\in \\mathbb{F_p}^{\\le d}[X]\\).
for \\( r \\leftarrow \\mathbb{F_p}\\), if \\(f(r) = g(r)\\), then \\(f = g\\) w.h.p

\\(f(r)-g(r)=0\\) => \\(f-g=0\\) w.h.p

\n

=> a simple equality test for two committed polynomials

\n

useful proof gadgets

1. zero test

let \\( \\omega \\in \\mathbb{F_p} \\) be a primitive k-th root of unity \\(( \\omega ^{k} = 1)\\)
set \\( H:= \\{1,\\omega,\\omega^2,\\omega^3,…,\\omega^{k-1}\\} \\subseteq \\mathbb{F_p} \\)
let \\( f \\in \\mathbb{F_p}^{\\le d}[X]\\) and \\( b, c \\in \\mathbb{F_p}\\) \\((d \\ge k)\\)

\n

task: prove that \\(f\\) is identically zero on \\(H\\)
\"zero

\n

info the box of \\(f\\) means the commitment of polynomial \\(f\\), i.e \\(com_f\\)

\n

2. product check

product check on \\(\\Omega: \\quad \\prod_{a\\in \\Omega} f(a) = 1\\)
Set \\( t \\in \\mathbb{F_p}^{\\le k}[X]\\) to be the degree-k polynomial:
\\[ t(1) = f(\\omega^0) = f(1), \\quad t(\\omega^s) = \\prod_{i=0}^{s}f(\\omega^{i}) \\quad for \\quad s = 1,…, k-1\\]
Then
\\(t(\\omega) = f(1) \\cdot f(\\omega), \\quad t(\\omega^2) = f(1) \\cdot f(\\omega) \\cdot f(\\omega^2), … \\)
\\(t( \\omega^{k-1}) = \\prod_{a \\in \\Omega}f(a) = 1\\)
and \\( t(\\omega \\cdot x) = t(x) \\cdot f(\\omega \\cdot x) \\) for all \\(x \\in \\Omega \\) (including at \\(x = \\omega^{k-1}\\))
\"prod_check_lemma\"
\"prod_check_prove_and_verify\"

\n

Same works for rational functions: \\( \\prod_{a \\in \\Omega}{f/g}(a) =1 \\)
The proof is similar

\n

3. permutation check

let \\(f,g\\) be polynomials in \\(\\mathbb{F_p}^{\\le d}[X]\\). Verifier has \\(com_f, com_g\\).
Prover wants to prove that \\((f(1),f(\\omega^1),f(\\omega^2),…,f(\\omega^{k-1})) \\in \\mathbb{F_p}^{\\le k}[X]\\) is a permutaion of \\((g(1),g(\\omega^1),g(\\omega^2),…,g(\\omega^{k-1})) \\in \\mathbb{F_p}^{\\le k}[X]\\)

\n

\"permutation

\n

4. prescribed permutation check





\n

PLONK: a poly-IOP for a general circuit C(x,w)

step 1: compile circuit to a computation trace (gate fan-in = 2)

\"circuit

\n

and encode the trace as polynomial
let \\(|C|\\) be the total number of gates, \\(|I| := |I_x| + |I_w|\\) be total number of inputs, where \\(|I_x|\\) is the number of public inputs, \\(I_w\\) is the number of private inputs.
Let \\(d=3*|C|+|I|\\) and \\( \\Omega=\\{1,\\omega,\\omega^2,\\omega^3,…,\\omega^{d-1}\\} \\)

\n

prover interpolates \\( T \\in \\mathbb{F_p}^{\\le d}[X]\\) such that

\n
    \n
  • T encodes all inputs: \\( T(\\omega^{-j}) \\)= input #j, for j = 1,…,|I|
    • \n
  • T encodes all wires:
    \\(LeftInput=f(\\omega^{3l})\\), \\( RightInput=f(\\omega^{3l+1})\\), \\(Output=f(\\omega^{3l+2})\\), for \\(l = 0,1,…, |C| -1\\)
    For the example,
    inputs
    \\(x_1= 5 = T(\\omega^9)\\), \\(x_2= 6 = T(\\omega^{10})\\), and \\(w_1 = 1=T(\\omega^{11})\\)
    wires
    \\(5=T(\\omega^0)\\), \\(6=T(\\omega^{1})\\), and \\(11=T(\\omega^{2})\\)
    \\(6=T(\\omega^3)\\), \\(1=T(\\omega^{4})\\), and \\(7=T(\\omega^{5})\\)
    \\(11=T(\\omega^6)\\), \\(7=T(\\omega^{7})\\), and \\(77=T(\\omega^{8})\\)
    • \n
\n

step 2: proving validity of T

Prover needs to prove 4 things

\n
    \n
  1. \\(T(x)\\) encodes the correct public inputs
    Both prover and verifier interpolate a polynomial \\( v(x) \\in \\mathbb{F_p}^{\\le |I_x|}[X]\\)
    that encodes the \\(x\\)-inputs to the circuit:
    \\(v(\\omega^{-j}) =\\) input #j, for \\(j = 1, …, |I_x|\\)
    In our example, \\(v(\\omega^{-1} = 5), v(\\omega^{-2} = 6)\\)
    Let \\( \\Omega_{inp}=\\{\\omega^{-1},\\omega^{-2},…,\\omega^{-|I_x|}\\} \\)
    Prover proves by using a ZeroTest on \\(\\Omega_inp\\) to prove that
    \\[T(y) - v(y) =0 \\quad \\forall y \\in \\Omega_{inp}\\]
    2. \n
  2. every gate is evaluated correctly
    Idea encode gate types using a selector polynomial \\(S(X)\\)
    define \\(S(X) \\in \\mathbb{F_p}^{\\le d}[X]\\) such that \\( \\forall l = 0, …, |C| -1\\):
    3. \n
\n
    \n
  • \\(S(\\omega^{3l}) =1\\) if gate #l is an addition gate
    • \n
  • \\(S(\\omega^{3l}) =0\\) if gate #l is a multiplication gate
    • \n
\n

Then, \\( \\forall y \\in \\Omega_{gates} : = \\{1,\\omega^{3},\\omega^{6},…,\\omega^{3(|C|-1)}\\} \\)
\\(S(y) \\cdot [T(y) + T(\\omega y)] + (1-S(y))\\cdot T(y) \\cdot T(\\omega y) = T(\\omega^2 y)\\)
\"gate_evaluation_zero_test\"

\n
    \n
  1. the wiring is implemented correctly (coppy constraint)
    2. \n
\n

\\(T(\\omega^9,\\omega^0)=\\sigma(\\omega^0,\\omega^9)\\)
\\(T(\\omega^{10},\\omega^1,\\omega^3)=\\sigma(\\omega^1,\\omega^3,\\omega^{10})\\)
\\(T(\\omega^2,\\omega^6)=\\sigma(\\omega^6,\\omega^2)\\)
\\(T(\\omega^{11},\\omega^4)=\\sigma(\\omega^4,\\omega^{11})\\)
\\(T(\\omega^{5},\\omega^7)=\\sigma(\\omega^7,\\omega^{5})\\)
note: 9 is actually -1, 10 is -2, 11 is -3
Define a polynomial \\(W: \\Omega -> \\Omega\\) that implemnets a rotation
\\( W(\\omega^{10}, \\omega^1, \\omega^3) =(\\omega^1, \\omega^3, \\omega^{10}) \\), \\(W(\\omega^{9}, \\omega^0)=(\\omega^0, \\omega^{9})\\), …

\n

Lemma: \\(\\forall y \\in \\Omega: T(y) = T(W(y))\\) => wire constraints are satisfied
This could be proved using a prescribed permutation check

\n
    \n
  1. the output of last gate is 0
    this is to prove \\(T(\\omega^8) -77 = 0\\)
    2. \n
\n

custom gate


\\(u, v, w, t, r\\) are polynomials represent input variables (row number is the gate number). in the Add, Mul only circuits, there are only two inputs, namely LeftInput and RightInput. Hoever, here there are multiple inputs for custom gate.

\n

In the above example, it is a constraint for \\( v_4 + w_3 \\cdot t_3 - t_4 = 0 \\)

\n

plonkup

plonkup is to ensure some values are in a pre-defined list. for example

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
x1x2x3Output
\\(a_{1,1}\\)\\(a_{1,2}\\)\\(a_{1,3}\\)\\(a_{1,4}\\)
\\(a_{2,1}\\)\\(a_{2,2}\\)\\(a_{2,3}\\)\\(a_{2,4}\\)
…………
\\(a_{n,1}\\)\\(a_{n,2}\\)\\(a_{n,3}\\)\\(a_{n,4}\\)
\n

\\(n\\) is gate number. the task is to prove a vector

\n

references

\n"},{"title":"zk bench","date":"2023-11-24T06:29:26.000Z","_content":"\n\n\n\n## references\n- [csdn blog on zkbench](https://blog.csdn.net/mutourend/article/details/134050672?spm=1001.2014.3001.5502)","source":"_posts/cryptography/zkp/zk-bench.md","raw":"---\ntitle: zk bench\ndate: 2023-11-24 14:29:26\ntags: [cryptography,zkp]\n---\n\n\n\n\n## references\n- [csdn blog on zkbench](https://blog.csdn.net/mutourend/article/details/134050672?spm=1001.2014.3001.5502)","slug":"cryptography/zkp/zk-bench","published":1,"updated":"2023-11-24T14:05:26.446Z","comments":1,"layout":"post","photos":[],"link":"","_id":"clphpvdok000caj7ubjq53ldn","content":"\n\n\n\n

references

\n","site":{"data":{}},"excerpt":"","more":"\n\n\n\n

references

\n"},{"title":"goldilocks field","date":"2023-11-18T03:06:53.000Z","_content":"\n\n\n\n\n# properties\n\\\\( p = \\phi^2 - \\phi + 1\\\\)\n\\\\( \\epsilon = \\phi -1\\\\)\n\n## Goldilocks Addition\nlet \\\\( a = a_0 + a_1 \\cdot \\phi, b = b_0 + b_1 \\cdot \\phi\\\\)\n\n`(c, carry) = a + b`;\n- if carry = 0 && c > p, result should be `c -p`\n- if carry = 0 && c < p, result should be `c`\n- if carry = 1 \nlet \\\\( c = c_0 + c_1 \\cdot \\phi + c_2 \\cdot \\phi^2\\\\), and \\\\(c_2 = 1 \\\\). let \\\\( u = c_0 + c_1 \\cdot \\phi \\\\)\ntherefore \\\\(a + b \\mod p\\\\) should be \\\\( a + b - p = c_0 + c_1 \\cdot \\phi + \\phi^2 - (\\phi^2 - \\phi + 1) =c_0 + c_1 \\cdot \\phi + \\phi - 1 = u + \\phi - 1\\\\) \n - if \\\\( u < p\\\\), computationally, \\\\( u - p\\\\) will borrow from \\\\( \\phi^2\\\\), which is equivalent to\n \\\\( \\phi^2 + u - p = u + \\phi - 1 \\\\), which is \\\\( a - b \\mod p\\\\). therefore, the result is just \\\\( u-p \\\\)\n - if \\\\( u> p\\\\), this will unlikely occur, if u >p, given by \\\\( \\phi^2 > p \\\\), so \\\\( u + \\phi^2 > p + \\phi^2 > 2p\\\\) that means a or b is not in canonical form. **does not consider this case for now**\n\n## Goldilocks Subtraction\n\\\\(a - b \\mod MOD\\\\)\nif a < b, a - b is negative, and the binary representation is 2's complement of the negative value. therefore, it's requied to add by MOD and treat the final binary representation as positive value.\n\n## Goldilocks Multiplication\nlet \\\\(a = a_0 + a_1 \\cdot \\phi\\\\), and \\\\(b=b_0 + b_1 \\cdot \\phi\\\\)\nto compute \\\\(c = a \\cdot b\\\\)\n- first, turn it into \\\\( a_0 \\cdot b_0 +(a_1\\cdot b_0 +a_0\\cdot b_1)\\cdot \\phi + a_1 \\cdot b_1 \\phi^2 \\\\)\nit may carry to \\\\( \\phi^2 \\\\). so the results is a `u128` (`[u32;4]`)\n- second, reduce the `[u32;4]` to `u64`, details to be described in the following section.\n## plonky2 Goldilocks Fieldreduce128\n```rust\nfn reduce128(x: u128) -> GoldilocksField {\n let (x_lo, x_hi) = split(x); // This is a no-op\n let x_hi_hi = x_hi >> 32;\n let x_hi_lo = x_hi & EPSILON;\n\n let (mut t0, borrow) = x_lo.overflowing_sub(x_hi_hi);\n if borrow {\n branch_hint(); // A borrow is exceedingly rare. It is faster to branch.\n t0 -= EPSILON; // Cannot underflow.\n }\n let t1 = x_hi_lo * EPSILON;\n let t2 = unsafe { add_no_canonicalize_trashing_input(t0, t1) };\n GoldilocksField(t2)\n}\n```\n\nlet \\\\( n = n_0 + n_1 \\cdot \\phi + n_2 \\cdot \\phi^{2} + n_3 \\cdot \\phi^{3}\\\\)\nsince, \\\\(\\phi^{3} = -1\\\\), \\\\(n\\\\) could be reduced to \\\\( n = n_0 + n_1 \\cdot \\phi + n_2 \\cdot \\phi^{2} - n_3 \\\\),\nwhich is line 6 in the above code snipet. when borrow occurs, it means it borrows \\\\( \\phi^2\\\\) ( subtraction is of u64, so borrow 1 menas add \\\\( \\phi^2\\\\), `1<<64`). therefore, needs to reduce \\\\( \\phi^2\\\\). since \\\\( \\phi^2 = \\phi - 1\\\\), which is reduce by \\\\( \\phi - 1\\\\) (this is `EPSILON`). \n**Note** needs to consider line 23 could underflow\n\nnext step is to reduce \\\\( n_2 \\cdot \\phi^2 \\\\), which is just \\\\( n_2 \\cdot (\\phi -1) \\\\), which is \\\\( n_2 \\cdot EPSILON \\\\), line 11, `x_hi_lo * EPSILON`\n\n\n## Sppark reduce(uint32_t temp[4])\n```cpp\n inline void reduce(uint32_t temp[4])\n {\n uint32_t carry;\n\n asm(\"sub.cc.u32 %0, %0, %3; subc.cc.u32 %1, %1, %4; subc.u32 %2, 0, 0;\"\n : \"+r\"(temp[0]), \"+r\"(temp[1]), \"=r\"(carry)\n : \"r\"(temp[2]), \"r\"(temp[3]));\n asm(\"add.cc.u32 %0, %0, %2; addc.u32 %1, %1, %3;\"\n : \"+r\"(temp[1]), \"+r\"(carry)\n : \"r\"(temp[2]), \"r\"(temp[3]));\n\n asm(\"mad.lo.cc.u32 %0, %3, %4, %0; madc.hi.cc.u32 %1, %3, %4, %1; addc.u32 %2, 0, 0;\"\n : \"+r\"(temp[0]), \"+r\"(temp[1]), \"=r\"(temp[2])\n : \"r\"(carry), \"r\"(gl64_device::W));\n asm(\"mad.lo.cc.u32 %0, %2, %3, %0; madc.hi.u32 %1, %2, %3, %1;\"\n : \"+r\"(temp[0]), \"+r\"(temp[1])\n : \"r\"(temp[2]), \"r\"(gl64_device::W));\n\n\n asm(\"mad.lo.cc.u32 %0, %2, %3, %0; madc.hi.u32 %1, %2, %3, %1;\"\n : \"+r\"(temp[0]), \"+r\"(temp[1])\n : \"r\"(temp[2]), \"r\"(gl64_device::W));\n\n asm(\"mov.b64 %0, {%1, %2};\" : \"=l\"(val) : \"r\"(temp[0]), \"r\"(temp[1]));\n }\n\n```\n**note**: `W` is epsilon\nlet \\\\( n = n_0 + n_1 \\cdot \\phi + n_2 \\cdot \\phi^{2} + n_3 \\cdot \\phi^{3}\\\\) \n\\\\( n = n_0 + n_2\\cdot\\phi^2 + \\phi(n_1+n_3\\cdot \\phi^2)\\\\).\nsince \\\\( \\phi^2 = \\phi - 1\\\\)\n\\\\( n = n_0 + n_2\\cdot(\\phi-1) + \\phi(n_1+n_3\\cdot(\\phi-1))\\\\).\n\\\\( n = n_0 -n_2 +n_2\\phi + \\phi(n_1-n_3 + n_3\\cdot\\phi)\\\\).\n\\\\( n = n_0 -n_2 + \\phi(n_1-n_3 +n_2) + n_3\\cdot\\phi^2\\\\).\nline 5 computes `n0 - n2` and set to temp[0]; & computes `n1-n3` and sets to temp[1]\nline 8 computes `n1- n3 + n2`; add n_3 with carry and put it to carry.\n\nlet \\\\(c\\\\) be the carry, at \\\\(\\phi^2\\\\), it is actually \\\\( c \\cdot \\phi^2\\\\). it could be further reduced to \\\\(c \\cdot (\\phi - 1) = c \\cdot W\\\\)\nline 12 is to reduce the carry part to temp[0], temp[1], and temp[2].\nif temp[2] still exist. line 15 will reduce it to temp[0], temp[1] similar as above.\nat this step. only temp[0] and temp[1] will contains value and return as the result\n\n\n## appendix\n- underflow\n```\nuint64_t tmp;\nuint32_t borrow;\nasm(\"{ .reg.pred %top;\");\nasm(\"sub.cc.u64 %0, %0, %2; subc.u32 %1, 0, 0;\"\n : \"+l\"(val), \"=r\"(borrow)\n : \"l\"(b.val));\n```\nall bits of borrow will be set to 1 if underflow occurs\n\n# references\n- [The Goldilocks Prime by Remco Bloemem](https://xn--2-umb.com/22/goldilocks/)\n- [parameters of goldilocks field](https://cronokirby.com/notes/2022/09/the-goldilocks-field/)\n- [csdn blog](https://blog.csdn.net/mutourend/article/details/126407028)\n- [cuda implementation of GoldilocksNTT by yrrid](https://github.com/yrrid/GoldilocksNTT/tree/main)","source":"_posts/arithmatic/goldilocks-field.md","raw":"---\ntitle: goldilocks field\ndate: 2023-11-18 11:06:53\ntags: [arithmatic]\n---\n\n\n\n\n\n# properties\n\\\\( p = \\phi^2 - \\phi + 1\\\\)\n\\\\( \\epsilon = \\phi -1\\\\)\n\n## Goldilocks Addition\nlet \\\\( a = a_0 + a_1 \\cdot \\phi, b = b_0 + b_1 \\cdot \\phi\\\\)\n\n`(c, carry) = a + b`;\n- if carry = 0 && c > p, result should be `c -p`\n- if carry = 0 && c < p, result should be `c`\n- if carry = 1 \nlet \\\\( c = c_0 + c_1 \\cdot \\phi + c_2 \\cdot \\phi^2\\\\), and \\\\(c_2 = 1 \\\\). let \\\\( u = c_0 + c_1 \\cdot \\phi \\\\)\ntherefore \\\\(a + b \\mod p\\\\) should be \\\\( a + b - p = c_0 + c_1 \\cdot \\phi + \\phi^2 - (\\phi^2 - \\phi + 1) =c_0 + c_1 \\cdot \\phi + \\phi - 1 = u + \\phi - 1\\\\) \n - if \\\\( u < p\\\\), computationally, \\\\( u - p\\\\) will borrow from \\\\( \\phi^2\\\\), which is equivalent to\n \\\\( \\phi^2 + u - p = u + \\phi - 1 \\\\), which is \\\\( a - b \\mod p\\\\). therefore, the result is just \\\\( u-p \\\\)\n - if \\\\( u> p\\\\), this will unlikely occur, if u >p, given by \\\\( \\phi^2 > p \\\\), so \\\\( u + \\phi^2 > p + \\phi^2 > 2p\\\\) that means a or b is not in canonical form. **does not consider this case for now**\n\n## Goldilocks Subtraction\n\\\\(a - b \\mod MOD\\\\)\nif a < b, a - b is negative, and the binary representation is 2's complement of the negative value. therefore, it's requied to add by MOD and treat the final binary representation as positive value.\n\n## Goldilocks Multiplication\nlet \\\\(a = a_0 + a_1 \\cdot \\phi\\\\), and \\\\(b=b_0 + b_1 \\cdot \\phi\\\\)\nto compute \\\\(c = a \\cdot b\\\\)\n- first, turn it into \\\\( a_0 \\cdot b_0 +(a_1\\cdot b_0 +a_0\\cdot b_1)\\cdot \\phi + a_1 \\cdot b_1 \\phi^2 \\\\)\nit may carry to \\\\( \\phi^2 \\\\). so the results is a `u128` (`[u32;4]`)\n- second, reduce the `[u32;4]` to `u64`, details to be described in the following section.\n## plonky2 Goldilocks Fieldreduce128\n```rust\nfn reduce128(x: u128) -> GoldilocksField {\n let (x_lo, x_hi) = split(x); // This is a no-op\n let x_hi_hi = x_hi >> 32;\n let x_hi_lo = x_hi & EPSILON;\n\n let (mut t0, borrow) = x_lo.overflowing_sub(x_hi_hi);\n if borrow {\n branch_hint(); // A borrow is exceedingly rare. It is faster to branch.\n t0 -= EPSILON; // Cannot underflow.\n }\n let t1 = x_hi_lo * EPSILON;\n let t2 = unsafe { add_no_canonicalize_trashing_input(t0, t1) };\n GoldilocksField(t2)\n}\n```\n\nlet \\\\( n = n_0 + n_1 \\cdot \\phi + n_2 \\cdot \\phi^{2} + n_3 \\cdot \\phi^{3}\\\\)\nsince, \\\\(\\phi^{3} = -1\\\\), \\\\(n\\\\) could be reduced to \\\\( n = n_0 + n_1 \\cdot \\phi + n_2 \\cdot \\phi^{2} - n_3 \\\\),\nwhich is line 6 in the above code snipet. when borrow occurs, it means it borrows \\\\( \\phi^2\\\\) ( subtraction is of u64, so borrow 1 menas add \\\\( \\phi^2\\\\), `1<<64`). therefore, needs to reduce \\\\( \\phi^2\\\\). since \\\\( \\phi^2 = \\phi - 1\\\\), which is reduce by \\\\( \\phi - 1\\\\) (this is `EPSILON`). \n**Note** needs to consider line 23 could underflow\n\nnext step is to reduce \\\\( n_2 \\cdot \\phi^2 \\\\), which is just \\\\( n_2 \\cdot (\\phi -1) \\\\), which is \\\\( n_2 \\cdot EPSILON \\\\), line 11, `x_hi_lo * EPSILON`\n\n\n## Sppark reduce(uint32_t temp[4])\n```cpp\n inline void reduce(uint32_t temp[4])\n {\n uint32_t carry;\n\n asm(\"sub.cc.u32 %0, %0, %3; subc.cc.u32 %1, %1, %4; subc.u32 %2, 0, 0;\"\n : \"+r\"(temp[0]), \"+r\"(temp[1]), \"=r\"(carry)\n : \"r\"(temp[2]), \"r\"(temp[3]));\n asm(\"add.cc.u32 %0, %0, %2; addc.u32 %1, %1, %3;\"\n : \"+r\"(temp[1]), \"+r\"(carry)\n : \"r\"(temp[2]), \"r\"(temp[3]));\n\n asm(\"mad.lo.cc.u32 %0, %3, %4, %0; madc.hi.cc.u32 %1, %3, %4, %1; addc.u32 %2, 0, 0;\"\n : \"+r\"(temp[0]), \"+r\"(temp[1]), \"=r\"(temp[2])\n : \"r\"(carry), \"r\"(gl64_device::W));\n asm(\"mad.lo.cc.u32 %0, %2, %3, %0; madc.hi.u32 %1, %2, %3, %1;\"\n : \"+r\"(temp[0]), \"+r\"(temp[1])\n : \"r\"(temp[2]), \"r\"(gl64_device::W));\n\n\n asm(\"mad.lo.cc.u32 %0, %2, %3, %0; madc.hi.u32 %1, %2, %3, %1;\"\n : \"+r\"(temp[0]), \"+r\"(temp[1])\n : \"r\"(temp[2]), \"r\"(gl64_device::W));\n\n asm(\"mov.b64 %0, {%1, %2};\" : \"=l\"(val) : \"r\"(temp[0]), \"r\"(temp[1]));\n }\n\n```\n**note**: `W` is epsilon\nlet \\\\( n = n_0 + n_1 \\cdot \\phi + n_2 \\cdot \\phi^{2} + n_3 \\cdot \\phi^{3}\\\\) \n\\\\( n = n_0 + n_2\\cdot\\phi^2 + \\phi(n_1+n_3\\cdot \\phi^2)\\\\).\nsince \\\\( \\phi^2 = \\phi - 1\\\\)\n\\\\( n = n_0 + n_2\\cdot(\\phi-1) + \\phi(n_1+n_3\\cdot(\\phi-1))\\\\).\n\\\\( n = n_0 -n_2 +n_2\\phi + \\phi(n_1-n_3 + n_3\\cdot\\phi)\\\\).\n\\\\( n = n_0 -n_2 + \\phi(n_1-n_3 +n_2) + n_3\\cdot\\phi^2\\\\).\nline 5 computes `n0 - n2` and set to temp[0]; & computes `n1-n3` and sets to temp[1]\nline 8 computes `n1- n3 + n2`; add n_3 with carry and put it to carry.\n\nlet \\\\(c\\\\) be the carry, at \\\\(\\phi^2\\\\), it is actually \\\\( c \\cdot \\phi^2\\\\). it could be further reduced to \\\\(c \\cdot (\\phi - 1) = c \\cdot W\\\\)\nline 12 is to reduce the carry part to temp[0], temp[1], and temp[2].\nif temp[2] still exist. line 15 will reduce it to temp[0], temp[1] similar as above.\nat this step. only temp[0] and temp[1] will contains value and return as the result\n\n\n## appendix\n- underflow\n```\nuint64_t tmp;\nuint32_t borrow;\nasm(\"{ .reg.pred %top;\");\nasm(\"sub.cc.u64 %0, %0, %2; subc.u32 %1, 0, 0;\"\n : \"+l\"(val), \"=r\"(borrow)\n : \"l\"(b.val));\n```\nall bits of borrow will be set to 1 if underflow occurs\n\n# references\n- [The Goldilocks Prime by Remco Bloemem](https://xn--2-umb.com/22/goldilocks/)\n- [parameters of goldilocks field](https://cronokirby.com/notes/2022/09/the-goldilocks-field/)\n- [csdn blog](https://blog.csdn.net/mutourend/article/details/126407028)\n- [cuda implementation of GoldilocksNTT by yrrid](https://github.com/yrrid/GoldilocksNTT/tree/main)","slug":"arithmatic/goldilocks-field","published":1,"updated":"2023-11-28T09:44:37.469Z","_id":"clpi1wlcr0000067ufpzn63zv","comments":1,"layout":"post","photos":[],"link":"","content":"\n\n\n

properties

\\( p = \\phi^2 - \\phi + 1\\)
\\( \\epsilon = \\phi -1\\)

\n

Goldilocks Addition

let \\( a = a_0 + a_1 \\cdot \\phi, b = b_0 + b_1 \\cdot \\phi\\)

\n

(c, carry) = a + b;

\n
    \n
  • if carry = 0 && c > p, result should be c -p
    • \n
  • if carry = 0 && c < p, result should be c
    • \n
  • if carry = 1
    let \\( c = c_0 + c_1 \\cdot \\phi + c_2 \\cdot \\phi^2\\), and \\(c_2 = 1 \\). let \\( u = c_0 + c_1 \\cdot \\phi \\)
    therefore \\(a + b \\mod p\\) should be \\( a + b - p = c_0 + c_1 \\cdot \\phi + \\phi^2 - (\\phi^2 - \\phi + 1) =c_0 + c_1 \\cdot \\phi + \\phi - 1 = u + \\phi - 1\\)
      \n
    • if \\( u < p\\), computationally, \\( u - p\\) will borrow from \\( \\phi^2\\), which is equivalent to
      \\( \\phi^2 + u - p = u + \\phi - 1 \\), which is \\( a - b \\mod p\\). therefore, the result is just \\( u-p \\)
      • \n
    • if \\( u> p\\), this will unlikely occur, if u >p, given by \\( \\phi^2 > p \\), so \\( u + \\phi^2 > p + \\phi^2 > 2p\\) that means a or b is not in canonical form. does not consider this case for now
      • \n
    \n
    • \n
\n

Goldilocks Subtraction

\\(a - b \\mod MOD\\)
if a < b, a - b is negative, and the binary representation is 2’s complement of the negative value. therefore, it’s requied to add by MOD and treat the final binary representation as positive value.

\n

Goldilocks Multiplication

let \\(a = a_0 + a_1 \\cdot \\phi\\), and \\(b=b_0 + b_1 \\cdot \\phi\\)
to compute \\(c = a \\cdot b\\)

\n
    \n
  • first, turn it into \\( a_0 \\cdot b_0 +(a_1\\cdot b_0 +a_0\\cdot b_1)\\cdot \\phi + a_1 \\cdot b_1 \\phi^2 \\)
    it may carry to \\( \\phi^2 \\). so the results is a u128 ([u32;4])
    • \n
  • second, reduce the [u32;4] to u64, details to be described in the following section.
    • \n
\n

plonky2 Goldilocks Fieldreduce128

1
2
3
4
5
6
7
8
9
10
11
12
13
14
fn reduce128(x: u128) -> GoldilocksField {
    let (x_lo, x_hi) = split(x); // This is a no-op
    let x_hi_hi = x_hi >> 32;
    let x_hi_lo = x_hi & EPSILON;

    let (mut t0, borrow) = x_lo.overflowing_sub(x_hi_hi);
    if borrow {
        branch_hint(); // A borrow is exceedingly rare. It is faster to branch.
        t0 -= EPSILON; // Cannot underflow.
    }
    let t1 = x_hi_lo * EPSILON;
    let t2 = unsafe { add_no_canonicalize_trashing_input(t0, t1) };
    GoldilocksField(t2)
}
\n\n

let \\( n = n_0 + n_1 \\cdot \\phi + n_2 \\cdot \\phi^{2} + n_3 \\cdot \\phi^{3}\\)
since, \\(\\phi^{3} = -1\\), \\(n\\) could be reduced to \\( n = n_0 + n_1 \\cdot \\phi + n_2 \\cdot \\phi^{2} - n_3 \\),
which is line 6 in the above code snipet. when borrow occurs, it means it borrows \\( \\phi^2\\) ( subtraction is of u64, so borrow 1 menas add \\( \\phi^2\\), 1<<64). therefore, needs to reduce \\( \\phi^2\\). since \\( \\phi^2 = \\phi - 1\\), which is reduce by \\( \\phi - 1\\) (this is EPSILON).
Note needs to consider line 23 could underflow

\n

next step is to reduce \\( n_2 \\cdot \\phi^2 \\), which is just \\( n_2 \\cdot (\\phi -1) \\), which is \\( n_2 \\cdot EPSILON \\), line 11, x_hi_lo * EPSILON

\n

Sppark reduce(uint32_t temp[4])

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
inline void reduce(uint32_t temp[4])
{
    uint32_t carry;

    asm("sub.cc.u32 %0, %0, %3; subc.cc.u32 %1, %1, %4; subc.u32 %2, 0, 0;"
        : "+r"(temp[0]), "+r"(temp[1]), "=r"(carry)
        : "r"(temp[2]), "r"(temp[3]));
    asm("add.cc.u32 %0, %0, %2; addc.u32 %1, %1, %3;"
        : "+r"(temp[1]), "+r"(carry)
        : "r"(temp[2]), "r"(temp[3]));

    asm("mad.lo.cc.u32 %0, %3, %4, %0; madc.hi.cc.u32 %1, %3, %4, %1; addc.u32 %2, 0, 0;"
        : "+r"(temp[0]), "+r"(temp[1]), "=r"(temp[2])
        : "r"(carry), "r"(gl64_device::W));
    asm("mad.lo.cc.u32 %0, %2, %3, %0; madc.hi.u32 %1, %2, %3, %1;"
        : "+r"(temp[0]), "+r"(temp[1])
        : "r"(temp[2]), "r"(gl64_device::W));


    asm("mad.lo.cc.u32 %0, %2, %3, %0; madc.hi.u32 %1, %2, %3, %1;"
        : "+r"(temp[0]), "+r"(temp[1])
        : "r"(temp[2]), "r"(gl64_device::W));

    asm("mov.b64 %0, {%1, %2};" : "=l"(val) : "r"(temp[0]), "r"(temp[1]));
}

\n

note: W is epsilon
let \\( n = n_0 + n_1 \\cdot \\phi + n_2 \\cdot \\phi^{2} + n_3 \\cdot \\phi^{3}\\)
\\( n = n_0 + n_2\\cdot\\phi^2 + \\phi(n_1+n_3\\cdot \\phi^2)\\).
since \\( \\phi^2 = \\phi - 1\\)
\\( n = n_0 + n_2\\cdot(\\phi-1) + \\phi(n_1+n_3\\cdot(\\phi-1))\\).
\\( n = n_0 -n_2 +n_2\\phi + \\phi(n_1-n_3 + n_3\\cdot\\phi)\\).
\\( n = n_0 -n_2 + \\phi(n_1-n_3 +n_2) + n_3\\cdot\\phi^2\\).
line 5 computes n0 - n2 and set to temp[0]; & computes n1-n3 and sets to temp[1]
line 8 computes n1- n3 + n2; add n_3 with carry and put it to carry.

\n

let \\(c\\) be the carry, at \\(\\phi^2\\), it is actually \\( c \\cdot \\phi^2\\). it could be further reduced to \\(c \\cdot (\\phi - 1) = c \\cdot W\\)
line 12 is to reduce the carry part to temp[0], temp[1], and temp[2].
if temp[2] still exist. line 15 will reduce it to temp[0], temp[1] similar as above.
at this step. only temp[0] and temp[1] will contains value and return as the result

\n

appendix

    \n
  • underflow
    1
    2
    3
    4
    5
    6
    uint64_t tmp;
    uint32_t borrow;
    asm("{ .reg.pred %top;");
    asm("sub.cc.u64 %0, %0, %2; subc.u32 %1, 0, 0;"
        : "+l"(val), "=r"(borrow)
        : "l"(b.val));
    \nall bits of borrow will be set to 1 if underflow occurs
    • \n
\n

references

\n","site":{"data":{}},"excerpt":"","more":"\n\n\n

properties

\\( p = \\phi^2 - \\phi + 1\\)
\\( \\epsilon = \\phi -1\\)

\n

Goldilocks Addition

let \\( a = a_0 + a_1 \\cdot \\phi, b = b_0 + b_1 \\cdot \\phi\\)

\n

(c, carry) = a + b;

\n
    \n
  • if carry = 0 && c > p, result should be c -p
    • \n
  • if carry = 0 && c < p, result should be c
    • \n
  • if carry = 1
    let \\( c = c_0 + c_1 \\cdot \\phi + c_2 \\cdot \\phi^2\\), and \\(c_2 = 1 \\). let \\( u = c_0 + c_1 \\cdot \\phi \\)
    therefore \\(a + b \\mod p\\) should be \\( a + b - p = c_0 + c_1 \\cdot \\phi + \\phi^2 - (\\phi^2 - \\phi + 1) =c_0 + c_1 \\cdot \\phi + \\phi - 1 = u + \\phi - 1\\)
      \n
    • if \\( u < p\\), computationally, \\( u - p\\) will borrow from \\( \\phi^2\\), which is equivalent to
      \\( \\phi^2 + u - p = u + \\phi - 1 \\), which is \\( a - b \\mod p\\). therefore, the result is just \\( u-p \\)
      • \n
    • if \\( u> p\\), this will unlikely occur, if u >p, given by \\( \\phi^2 > p \\), so \\( u + \\phi^2 > p + \\phi^2 > 2p\\) that means a or b is not in canonical form. does not consider this case for now
      • \n
    \n
    • \n
\n

Goldilocks Subtraction

\\(a - b \\mod MOD\\)
if a < b, a - b is negative, and the binary representation is 2’s complement of the negative value. therefore, it’s requied to add by MOD and treat the final binary representation as positive value.

\n

Goldilocks Multiplication

let \\(a = a_0 + a_1 \\cdot \\phi\\), and \\(b=b_0 + b_1 \\cdot \\phi\\)
to compute \\(c = a \\cdot b\\)

\n
    \n
  • first, turn it into \\( a_0 \\cdot b_0 +(a_1\\cdot b_0 +a_0\\cdot b_1)\\cdot \\phi + a_1 \\cdot b_1 \\phi^2 \\)
    it may carry to \\( \\phi^2 \\). so the results is a u128 ([u32;4])
    • \n
  • second, reduce the [u32;4] to u64, details to be described in the following section.
    • \n
\n

plonky2 Goldilocks Fieldreduce128

1
2
3
4
5
6
7
8
9
10
11
12
13
14
fn reduce128(x: u128) -> GoldilocksField {
    let (x_lo, x_hi) = split(x); // This is a no-op
    let x_hi_hi = x_hi >> 32;
    let x_hi_lo = x_hi & EPSILON;

    let (mut t0, borrow) = x_lo.overflowing_sub(x_hi_hi);
    if borrow {
        branch_hint(); // A borrow is exceedingly rare. It is faster to branch.
        t0 -= EPSILON; // Cannot underflow.
    }
    let t1 = x_hi_lo * EPSILON;
    let t2 = unsafe { add_no_canonicalize_trashing_input(t0, t1) };
    GoldilocksField(t2)
}
\n\n

let \\( n = n_0 + n_1 \\cdot \\phi + n_2 \\cdot \\phi^{2} + n_3 \\cdot \\phi^{3}\\)
since, \\(\\phi^{3} = -1\\), \\(n\\) could be reduced to \\( n = n_0 + n_1 \\cdot \\phi + n_2 \\cdot \\phi^{2} - n_3 \\),
which is line 6 in the above code snipet. when borrow occurs, it means it borrows \\( \\phi^2\\) ( subtraction is of u64, so borrow 1 menas add \\( \\phi^2\\), 1<<64). therefore, needs to reduce \\( \\phi^2\\). since \\( \\phi^2 = \\phi - 1\\), which is reduce by \\( \\phi - 1\\) (this is EPSILON).
Note needs to consider line 23 could underflow

\n

next step is to reduce \\( n_2 \\cdot \\phi^2 \\), which is just \\( n_2 \\cdot (\\phi -1) \\), which is \\( n_2 \\cdot EPSILON \\), line 11, x_hi_lo * EPSILON

\n

Sppark reduce(uint32_t temp[4])

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
inline void reduce(uint32_t temp[4])
{
    uint32_t carry;

    asm("sub.cc.u32 %0, %0, %3; subc.cc.u32 %1, %1, %4; subc.u32 %2, 0, 0;"
        : "+r"(temp[0]), "+r"(temp[1]), "=r"(carry)
        : "r"(temp[2]), "r"(temp[3]));
    asm("add.cc.u32 %0, %0, %2; addc.u32 %1, %1, %3;"
        : "+r"(temp[1]), "+r"(carry)
        : "r"(temp[2]), "r"(temp[3]));

    asm("mad.lo.cc.u32 %0, %3, %4, %0; madc.hi.cc.u32 %1, %3, %4, %1; addc.u32 %2, 0, 0;"
        : "+r"(temp[0]), "+r"(temp[1]), "=r"(temp[2])
        : "r"(carry), "r"(gl64_device::W));
    asm("mad.lo.cc.u32 %0, %2, %3, %0; madc.hi.u32 %1, %2, %3, %1;"
        : "+r"(temp[0]), "+r"(temp[1])
        : "r"(temp[2]), "r"(gl64_device::W));


    asm("mad.lo.cc.u32 %0, %2, %3, %0; madc.hi.u32 %1, %2, %3, %1;"
        : "+r"(temp[0]), "+r"(temp[1])
        : "r"(temp[2]), "r"(gl64_device::W));

    asm("mov.b64 %0, {%1, %2};" : "=l"(val) : "r"(temp[0]), "r"(temp[1]));
}

\n

note: W is epsilon
let \\( n = n_0 + n_1 \\cdot \\phi + n_2 \\cdot \\phi^{2} + n_3 \\cdot \\phi^{3}\\)
\\( n = n_0 + n_2\\cdot\\phi^2 + \\phi(n_1+n_3\\cdot \\phi^2)\\).
since \\( \\phi^2 = \\phi - 1\\)
\\( n = n_0 + n_2\\cdot(\\phi-1) + \\phi(n_1+n_3\\cdot(\\phi-1))\\).
\\( n = n_0 -n_2 +n_2\\phi + \\phi(n_1-n_3 + n_3\\cdot\\phi)\\).
\\( n = n_0 -n_2 + \\phi(n_1-n_3 +n_2) + n_3\\cdot\\phi^2\\).
line 5 computes n0 - n2 and set to temp[0]; & computes n1-n3 and sets to temp[1]
line 8 computes n1- n3 + n2; add n_3 with carry and put it to carry.

\n

let \\(c\\) be the carry, at \\(\\phi^2\\), it is actually \\( c \\cdot \\phi^2\\). it could be further reduced to \\(c \\cdot (\\phi - 1) = c \\cdot W\\)
line 12 is to reduce the carry part to temp[0], temp[1], and temp[2].
if temp[2] still exist. line 15 will reduce it to temp[0], temp[1] similar as above.
at this step. only temp[0] and temp[1] will contains value and return as the result

\n

appendix

    \n
  • underflow
    1
    2
    3
    4
    5
    6
    uint64_t tmp;
    uint32_t borrow;
    asm("{ .reg.pred %top;");
    asm("sub.cc.u64 %0, %0, %2; subc.u32 %1, 0, 0;"
        : "+l"(val), "=r"(borrow)
        : "l"(b.val));
    \nall bits of borrow will be set to 1 if underflow occurs
    • \n
\n

references

\n"},{"title":"bls12_381","date":"2023-12-01T11:09:11.000Z","_content":"\n## introduction\nBLS12-381 is a pairing-friendly elliptic curve. Curve BLS12-381 was designed by [Sean Bowe](https://twitter.com/ebfull) in early 2017 as the foundation for an upgrade to the Zcash protocol. It is both pairing-friendly (making it efficient for digital signatures) and effective for constructing zkSnarks. BLS12-381 is part of a family of curves described by Barreto, Lynn, and Scott (BLS). The 12 is the embedding degree of the curve: neither too low, nor too high. The 381 is the number of bits needed to represent coordinates on the curve: the field modulus, \\\\(q\\\\). 381 is a fairly handy number as we can use 48 bytes per field element, with 3 bits left over for useful flags or arithmetic optimisations. This size of this number is guided both by security requirements and implementation efficiency.\n\n## curve equation and parameters\nThe basic equation of the BLS12-381 curve is \\\\( y^2 = x^3 + 4\\\\)\nThe key parameters for a BLS curve are set using a single parameter \\\\(\\mathbf{x}\\\\) (different from the in the curve equation!). BLS12-381 is derived from the \\\\( k \\equiv 0 \\mod 6\\\\) case of Construction 6.6 in the [taxonomy](https://eprint.iacr.org/2006/372.pdf).\n\nSpecific design goals for BLS12-381 are:\n- \\\\(\\mathbf{x}\\\\) has “low hamming weight”, meaning that it has very few bits set to 1. This is particularly important for the efficiency of the algorithm that calculates pairings (the Miller loop).\n- The field modulus \\\\(q\\\\) mentioned above is prime and has 383 bits or fewer, which makes 64-bit or 32-bit arithmetic on it more efficient.\n- The order \\\\(r\\\\) of the subgroups we use is prime and has 255 bits or fewer, which is good for the same reason as above.\n- To support zkSnark schemes, we want to have a large power of two root of unity in the field. This means we want **\\\\(2^n\\\\) to be a factor of \\\\(r-1\\\\)**, for some biggish \\\\(n\\\\). (Making \\\\(\\mathbf{x}\\\\) a multiple of \\\\(2^{n/2}\\\\) will achieve this.) This property is key to being able to use fast Fourier transforms for interesting things like polynomial multiplication.\n\nThe value \\\\(\\mathbf{x}\\\\) = -0xd201000000010000 (hexadecimal, note that it is negative) gives the largest and the lowest Hamming weight meeting these criteria. With this value we have,\n| parameters |equation|value(hex)|comments|\n|---|:---:|:---:|---:|\n|Field modulus \\\\(q\\\\)|\\\\(\\frac{1}{3}(\\mathbf{x}-1)^2(\\mathbf{x}^4-\\mathbf{x}^2+1) + \\mathbf{x}\\\\)|0x1a0111ea397fe69a4b1ba7b6434bacd764774b84f3
8512bf6730d2a0f6b0f6241eabfffeb153ffffb9feffffffffaaab|381 bits, prime|\n|Subgroup size \\\\(r\\\\)|\\\\((\\mathbf{x}^4-\\mathbf{x}^2+1)\\\\)|0x73eda753299d7d483339d80809a1d80553
bda402fffe5bfeffffffff00000001|255 bits, prime|\n\n## Field extensions\nField extensions are fundamental to elliptic curve pairings. The “12” is BLS12-381 is not only the embedding degree, it is also (relatedly) the degree of field extension that we will need to use.\nFor example, The complex numbers are a quadratic extension of the real numbers (\\\\(x^2 = 1\\\\)). Complex numbers can’t be extended any further because there are [no irreducible polynomials over the complex numbers](https://en.wikipedia.org/wiki/Fundamental_theorem_of_algebra). But for finite fields, if we can find an irreducible \\\\(k\\\\)-degree polynomial in our field \\\\(F_q\\\\), and we often can, then we are able to extend the field to \\\\(F_{q^k}\\\\), , and represent the elements of the extended field as degree \\\\(k-1\\\\) polynomials, \\\\(a_0 + a_1 x + ... + a_{k-1}x^{k-1}\\\\). we can represent this compactly as (\\\\(a_0, a_1, ..., a_{k-1}\\\\))\nIn practice, large extension fields like \\\\(F_{q^{12}}\\\\)are implemented as towers of smaller extensions.\n\n## the curves\nBLS12-381 is really dealing with two curves. Both curves share more-or-less the same curve equation, but are defined over different fields.\nThe simpler one is over the finite field \\\\(F_q\\\\), which is just the integers mod \\\\(q\\\\). So the curve has points only where the equation \\\\(y^2 = x^3 +4\\\\) has solutions with \\\\(x\\\\) and \\\\(y\\\\) both integers less than \\\\(q\\\\). We shall call this curve \\\\(E(F_q)\\\\).\nThe other curve is defined over an extension of \\\\(F_q\\\\)to \\\\(F_{q^2}\\\\) (think complex numbers). In this case, the curve equation is slightly modified to be \\\\(y^2 = x^3+4(1+i)\\\\), and we call the curve \\\\(E'(F_{q^2})\\\\)\n\n## the subgroups\nA pairing is a bilinear map. This means that it takes as input two points, each from a group of the same order, \\\\(r\\\\). for rather technical reasons, these two groups need to be distinct. Let’s call them \n\\\\(G_1\\\\) and \\\\(G_2\\\\).\nUnfortunately, our simple curve \\\\(E(F_q)\\\\) **has only a single large subgroup of order \\\\(r\\\\)**, so we can’t define a pairing based solely on \\\\(E(F_q)\\\\). However, if we keep extending the field over which \n\\\\(E\\\\) is defined, it can be proved that we eventually find a curve that has more than one subgroup of order \\\\(r\\\\). that is, for some \\\\(k\\\\), **\\\\(E(F_{q^k})\\\\) contains other subgroups of order \\\\(r\\\\) that we can use**. One of these subgroups contains only points having a **trace of zero**[1], and we choose that subgroup to be \\\\(G_2\\\\)\n\n\nThis number \\\\(k\\\\), the amount that we need to extend the base field by to find the new group, is called the embedding degree of the curve, which in our case is the “12” in BLS12-381. For completeness, note that each of \\\\(G_1\\\\) and \\\\(G_2\\\\) shares with its containing curve the “point at infinity”. This is the identity element of the elliptic curve arithmetic group, often denoted \\\\(\\mathcal O\\\\)\n. For any point \\\\(P\\\\), \\\\( P + \\mathcal O = \\mathcal O + P = P\\\\). \n\n## Twists\nBut there’s another challenge. As discussed earlier, doing arithmetic in \\\\(F_{q^{12}}\\\\) is horribly complicated and inefficient. And curve operations need a lot of arithmetic. A [twist](http://indigo.ie/~mscott/twists.pdf) is something like a coordinate transformation. Rather wonderfully, this can be used to transform our \\\\(E(F_{q^{12}})\\\\) curve into a curve defined over a lower degree field that still has an order \\\\(r\\\\) subgroup. Moreover, this subgroup has a simple mapping to and from our \\\\(G_2\\\\) group\n\nBLS12-381 uses a “sextic twist”. This means that it reduces the degree of the extension field by a factor of six. So \\\\(G_2\\\\) on the twisted curve can be defined over \\\\(F_{q^2}\\\\) instead of \\\\(F_{q^{12}}\\\\)\nI haven’t seen this written down anywhere—but attempting to decode section 3 of [this](https://eprint.iacr.org/2005/133.pdf)—if we find a \\\\(u\\\\) such that \\\\(u^6 = (1+i)^{-1}\\\\), then we can define our twisting transformation as \\\\( (x,y) -> (x/u^2, y/u^3)\\\\). This transforms our original curve \\\\(E: y^2 = x^3 + 4\\\\) into the curve \\\\(E': y^2 + 4/u^6 = x^3 + 4(1+i)\\\\). **\\\\(E\\\\) and \\\\(E'\\\\) look different, but are actually the same object presented with respect to coefficinets in different base fields**.\n\nSo these are the two groups we will be using:\n- \\\\(G_1 \\subset E(F_q)\\\\), where \\\\(E: y^2 = x^3 + 4\\\\)\n- \\\\(G_2 \\subset E(F_{q^2})\\\\), where \\\\(E: y^2 = x^3 + 4(1+i)\\\\)\nNote that coordinates of points in the \\\\(G_1\\\\) group are pairs of integers, and coordinates of points in the \\\\(G_2\\\\) group are pairs of complex integers.\n\n## Parings\ns far as BLS12-381 is concerned, a pairing simply takes a point \\\\(P \\in G_1 \\subset E(F_q)\\\\), and a point \\\\(Q \\in G_2 \\subset E'(F_{q^2})\\\\) and outputs a point from a group \\\\(G_T \\subset F_{q^{12}}\\\\). That is, for a paring \\\\(e\\\\), \\\\(e: G_1 \\times G_2 \\rightarrow G_T\\\\)\n\nproperties of pairing\n\\\\( e(P, Q+R) = e(P,Q) \\cdot e(P,R) \\\\), \n**note** as a way of memory, can think it as \\\\(P^{Q+R} = P^Q \\cdot P^R\\\\)\n\\\\( e(P+S, R) = e(P,R) \\cdot e(S,R)\\\\)\nFrom this, we can deduce that all of the following identities hold:\n\\\\(e([a]P, [b]Q) = e(P, [b]Q)^a = e(P,Q)^{ab} = e(P, [a]Q)^b = e([b]P, [a]Q)\\\\)\n\n\n## Embedding degree\nThe embedding degree, \\\\(k\\\\), is calculated as the smallest positive integer such that \\\\(r\\\\) divides \\\\(q^k -1\\\\). So, in the case of BLS12-381, \\\\(r\\\\) is a factor of \\\\(q^{12} -1\\\\), but not of any lower power.\nThe choice of an embedding degree is a balance between security and efficiency (as ever). On the security front, the embedding degree is also known as the security multiplier: a higher embedding degree makes the discrete logarithm problem harder to solve in \n. However, a high embedding degree means we have to do field operations in high-degree extensions, which is clunky and inefficient. \n\n\n## cofactor\nA subgroup’s cofactor is the ratio of the size of the whole group to the size of the subgroup. \n| Group |Cofactor|Equation|value(hex)|\n|---|:---:|:---:|---:|\n|\\\\(G_1\\\\)|\\\\(h_1\\\\)|\\\\(\\frac{\\mathbf{x-1}^2}{3}\\\\)|0x396c8c005555e
1568c00aaab0000aaab|\n|\\\\(G_2\\\\)|\\\\(h_2\\\\)|\\\\(\\frac{\\mathbf{x}^8 -4\\mathbf{x}^7+5\\mathbf{x}^6-4\\mathbf{x}^4+6\\mathbf{x}^3-4\\mathbf{x}^2-4\\mathbf{x}+13}{9}\\\\)|0x5d543a95414e7f1091d50792876a202cd91de
4547085abaa68a205b2e5a7ddfa628f1cb4d9e82ef2
1537e293a6691ae1616ec6e786f0c70cf1c38e31c7238e5|\n\n**note** multiplying by the cofactor turns out to be a straightforward way to map any arbitrary point on the elliptic curve into the respective subgroup \\\\(G_1\\\\) or \\\\(G_2\\\\)\n\n## Generators\n\\\\(G_1\\\\) and \\\\(G_2\\\\) are cyclic groups of prime order, so any point (except the identity/point at infinity) is a generator. Thus, picking generators is just a matter of convention.\nGenerator points \\\\(G_1\\\\) and \\\\(G_2\\\\) are specified in decimal [here](https://github.com/zcash/librustzcash/blob/6e0364cd42a2b3d2b958a54771ef51a8db79dd29/pairing/src/bls12_381/README.md#generators)\n\nThese were chosen as follows:\n\n> The generators of \\\\(G_1\\\\) and \\\\(G_2\\\\) are computed by finding the lexicographically smallest valid x-coordinate, and its lexicographically smallest > y-coordinate and scaling it by the cofactor such that the result is not the point at infinity.\n\n\n## Foot notes\n[1] The “trace zero subgroup” qualifies as an obscure incantation. Basically, the trace of a point is \\\\(\\sum_{i=0}^{k-1}(x^{q^i}, y^{q^i})\\\\), where \\\\(k=12\\\\) in our case. Understanding this involves stuff like the Frobenius endomorphism.\n\n\n## references\n- https://hackmd.io/@benjaminion/bls12-381\n- [initial post of bls12-381](https://electriccoin.co/blog/new-snark-curve/)\n- [implementation: blst](https://github.com/supranational/blst)","source":"_posts/cryptography/elliptic_curve/bls12-381.md","raw":"---\ntitle: bls12_381\ndate: 2023-12-01 19:09:11\ntags: [cryptography, ec]\n---\n\n## introduction\nBLS12-381 is a pairing-friendly elliptic curve. Curve BLS12-381 was designed by [Sean Bowe](https://twitter.com/ebfull) in early 2017 as the foundation for an upgrade to the Zcash protocol. It is both pairing-friendly (making it efficient for digital signatures) and effective for constructing zkSnarks. BLS12-381 is part of a family of curves described by Barreto, Lynn, and Scott (BLS). The 12 is the embedding degree of the curve: neither too low, nor too high. The 381 is the number of bits needed to represent coordinates on the curve: the field modulus, \\\\(q\\\\). 381 is a fairly handy number as we can use 48 bytes per field element, with 3 bits left over for useful flags or arithmetic optimisations. This size of this number is guided both by security requirements and implementation efficiency.\n\n## curve equation and parameters\nThe basic equation of the BLS12-381 curve is \\\\( y^2 = x^3 + 4\\\\)\nThe key parameters for a BLS curve are set using a single parameter \\\\(\\mathbf{x}\\\\) (different from the in the curve equation!). BLS12-381 is derived from the \\\\( k \\equiv 0 \\mod 6\\\\) case of Construction 6.6 in the [taxonomy](https://eprint.iacr.org/2006/372.pdf).\n\nSpecific design goals for BLS12-381 are:\n- \\\\(\\mathbf{x}\\\\) has “low hamming weight”, meaning that it has very few bits set to 1. This is particularly important for the efficiency of the algorithm that calculates pairings (the Miller loop).\n- The field modulus \\\\(q\\\\) mentioned above is prime and has 383 bits or fewer, which makes 64-bit or 32-bit arithmetic on it more efficient.\n- The order \\\\(r\\\\) of the subgroups we use is prime and has 255 bits or fewer, which is good for the same reason as above.\n- To support zkSnark schemes, we want to have a large power of two root of unity in the field. This means we want **\\\\(2^n\\\\) to be a factor of \\\\(r-1\\\\)**, for some biggish \\\\(n\\\\). (Making \\\\(\\mathbf{x}\\\\) a multiple of \\\\(2^{n/2}\\\\) will achieve this.) This property is key to being able to use fast Fourier transforms for interesting things like polynomial multiplication.\n\nThe value \\\\(\\mathbf{x}\\\\) = -0xd201000000010000 (hexadecimal, note that it is negative) gives the largest and the lowest Hamming weight meeting these criteria. With this value we have,\n| parameters |equation|value(hex)|comments|\n|---|:---:|:---:|---:|\n|Field modulus \\\\(q\\\\)|\\\\(\\frac{1}{3}(\\mathbf{x}-1)^2(\\mathbf{x}^4-\\mathbf{x}^2+1) + \\mathbf{x}\\\\)|0x1a0111ea397fe69a4b1ba7b6434bacd764774b84f3
8512bf6730d2a0f6b0f6241eabfffeb153ffffb9feffffffffaaab|381 bits, prime|\n|Subgroup size \\\\(r\\\\)|\\\\((\\mathbf{x}^4-\\mathbf{x}^2+1)\\\\)|0x73eda753299d7d483339d80809a1d80553
bda402fffe5bfeffffffff00000001|255 bits, prime|\n\n## Field extensions\nField extensions are fundamental to elliptic curve pairings. The “12” is BLS12-381 is not only the embedding degree, it is also (relatedly) the degree of field extension that we will need to use.\nFor example, The complex numbers are a quadratic extension of the real numbers (\\\\(x^2 = 1\\\\)). Complex numbers can’t be extended any further because there are [no irreducible polynomials over the complex numbers](https://en.wikipedia.org/wiki/Fundamental_theorem_of_algebra). But for finite fields, if we can find an irreducible \\\\(k\\\\)-degree polynomial in our field \\\\(F_q\\\\), and we often can, then we are able to extend the field to \\\\(F_{q^k}\\\\), , and represent the elements of the extended field as degree \\\\(k-1\\\\) polynomials, \\\\(a_0 + a_1 x + ... + a_{k-1}x^{k-1}\\\\). we can represent this compactly as (\\\\(a_0, a_1, ..., a_{k-1}\\\\))\nIn practice, large extension fields like \\\\(F_{q^{12}}\\\\)are implemented as towers of smaller extensions.\n\n## the curves\nBLS12-381 is really dealing with two curves. Both curves share more-or-less the same curve equation, but are defined over different fields.\nThe simpler one is over the finite field \\\\(F_q\\\\), which is just the integers mod \\\\(q\\\\). So the curve has points only where the equation \\\\(y^2 = x^3 +4\\\\) has solutions with \\\\(x\\\\) and \\\\(y\\\\) both integers less than \\\\(q\\\\). We shall call this curve \\\\(E(F_q)\\\\).\nThe other curve is defined over an extension of \\\\(F_q\\\\)to \\\\(F_{q^2}\\\\) (think complex numbers). In this case, the curve equation is slightly modified to be \\\\(y^2 = x^3+4(1+i)\\\\), and we call the curve \\\\(E'(F_{q^2})\\\\)\n\n## the subgroups\nA pairing is a bilinear map. This means that it takes as input two points, each from a group of the same order, \\\\(r\\\\). for rather technical reasons, these two groups need to be distinct. Let’s call them \n\\\\(G_1\\\\) and \\\\(G_2\\\\).\nUnfortunately, our simple curve \\\\(E(F_q)\\\\) **has only a single large subgroup of order \\\\(r\\\\)**, so we can’t define a pairing based solely on \\\\(E(F_q)\\\\). However, if we keep extending the field over which \n\\\\(E\\\\) is defined, it can be proved that we eventually find a curve that has more than one subgroup of order \\\\(r\\\\). that is, for some \\\\(k\\\\), **\\\\(E(F_{q^k})\\\\) contains other subgroups of order \\\\(r\\\\) that we can use**. One of these subgroups contains only points having a **trace of zero**[1], and we choose that subgroup to be \\\\(G_2\\\\)\n\n\nThis number \\\\(k\\\\), the amount that we need to extend the base field by to find the new group, is called the embedding degree of the curve, which in our case is the “12” in BLS12-381. For completeness, note that each of \\\\(G_1\\\\) and \\\\(G_2\\\\) shares with its containing curve the “point at infinity”. This is the identity element of the elliptic curve arithmetic group, often denoted \\\\(\\mathcal O\\\\)\n. For any point \\\\(P\\\\), \\\\( P + \\mathcal O = \\mathcal O + P = P\\\\). \n\n## Twists\nBut there’s another challenge. As discussed earlier, doing arithmetic in \\\\(F_{q^{12}}\\\\) is horribly complicated and inefficient. And curve operations need a lot of arithmetic. A [twist](http://indigo.ie/~mscott/twists.pdf) is something like a coordinate transformation. Rather wonderfully, this can be used to transform our \\\\(E(F_{q^{12}})\\\\) curve into a curve defined over a lower degree field that still has an order \\\\(r\\\\) subgroup. Moreover, this subgroup has a simple mapping to and from our \\\\(G_2\\\\) group\n\nBLS12-381 uses a “sextic twist”. This means that it reduces the degree of the extension field by a factor of six. So \\\\(G_2\\\\) on the twisted curve can be defined over \\\\(F_{q^2}\\\\) instead of \\\\(F_{q^{12}}\\\\)\nI haven’t seen this written down anywhere—but attempting to decode section 3 of [this](https://eprint.iacr.org/2005/133.pdf)—if we find a \\\\(u\\\\) such that \\\\(u^6 = (1+i)^{-1}\\\\), then we can define our twisting transformation as \\\\( (x,y) -> (x/u^2, y/u^3)\\\\). This transforms our original curve \\\\(E: y^2 = x^3 + 4\\\\) into the curve \\\\(E': y^2 + 4/u^6 = x^3 + 4(1+i)\\\\). **\\\\(E\\\\) and \\\\(E'\\\\) look different, but are actually the same object presented with respect to coefficinets in different base fields**.\n\nSo these are the two groups we will be using:\n- \\\\(G_1 \\subset E(F_q)\\\\), where \\\\(E: y^2 = x^3 + 4\\\\)\n- \\\\(G_2 \\subset E(F_{q^2})\\\\), where \\\\(E: y^2 = x^3 + 4(1+i)\\\\)\nNote that coordinates of points in the \\\\(G_1\\\\) group are pairs of integers, and coordinates of points in the \\\\(G_2\\\\) group are pairs of complex integers.\n\n## Parings\ns far as BLS12-381 is concerned, a pairing simply takes a point \\\\(P \\in G_1 \\subset E(F_q)\\\\), and a point \\\\(Q \\in G_2 \\subset E'(F_{q^2})\\\\) and outputs a point from a group \\\\(G_T \\subset F_{q^{12}}\\\\). That is, for a paring \\\\(e\\\\), \\\\(e: G_1 \\times G_2 \\rightarrow G_T\\\\)\n\nproperties of pairing\n\\\\( e(P, Q+R) = e(P,Q) \\cdot e(P,R) \\\\), \n**note** as a way of memory, can think it as \\\\(P^{Q+R} = P^Q \\cdot P^R\\\\)\n\\\\( e(P+S, R) = e(P,R) \\cdot e(S,R)\\\\)\nFrom this, we can deduce that all of the following identities hold:\n\\\\(e([a]P, [b]Q) = e(P, [b]Q)^a = e(P,Q)^{ab} = e(P, [a]Q)^b = e([b]P, [a]Q)\\\\)\n\n\n## Embedding degree\nThe embedding degree, \\\\(k\\\\), is calculated as the smallest positive integer such that \\\\(r\\\\) divides \\\\(q^k -1\\\\). So, in the case of BLS12-381, \\\\(r\\\\) is a factor of \\\\(q^{12} -1\\\\), but not of any lower power.\nThe choice of an embedding degree is a balance between security and efficiency (as ever). On the security front, the embedding degree is also known as the security multiplier: a higher embedding degree makes the discrete logarithm problem harder to solve in \n. However, a high embedding degree means we have to do field operations in high-degree extensions, which is clunky and inefficient. \n\n\n## cofactor\nA subgroup’s cofactor is the ratio of the size of the whole group to the size of the subgroup. \n| Group |Cofactor|Equation|value(hex)|\n|---|:---:|:---:|---:|\n|\\\\(G_1\\\\)|\\\\(h_1\\\\)|\\\\(\\frac{\\mathbf{x-1}^2}{3}\\\\)|0x396c8c005555e
1568c00aaab0000aaab|\n|\\\\(G_2\\\\)|\\\\(h_2\\\\)|\\\\(\\frac{\\mathbf{x}^8 -4\\mathbf{x}^7+5\\mathbf{x}^6-4\\mathbf{x}^4+6\\mathbf{x}^3-4\\mathbf{x}^2-4\\mathbf{x}+13}{9}\\\\)|0x5d543a95414e7f1091d50792876a202cd91de
4547085abaa68a205b2e5a7ddfa628f1cb4d9e82ef2
1537e293a6691ae1616ec6e786f0c70cf1c38e31c7238e5|\n\n**note** multiplying by the cofactor turns out to be a straightforward way to map any arbitrary point on the elliptic curve into the respective subgroup \\\\(G_1\\\\) or \\\\(G_2\\\\)\n\n## Generators\n\\\\(G_1\\\\) and \\\\(G_2\\\\) are cyclic groups of prime order, so any point (except the identity/point at infinity) is a generator. Thus, picking generators is just a matter of convention.\nGenerator points \\\\(G_1\\\\) and \\\\(G_2\\\\) are specified in decimal [here](https://github.com/zcash/librustzcash/blob/6e0364cd42a2b3d2b958a54771ef51a8db79dd29/pairing/src/bls12_381/README.md#generators)\n\nThese were chosen as follows:\n\n> The generators of \\\\(G_1\\\\) and \\\\(G_2\\\\) are computed by finding the lexicographically smallest valid x-coordinate, and its lexicographically smallest > y-coordinate and scaling it by the cofactor such that the result is not the point at infinity.\n\n\n## Foot notes\n[1] The “trace zero subgroup” qualifies as an obscure incantation. Basically, the trace of a point is \\\\(\\sum_{i=0}^{k-1}(x^{q^i}, y^{q^i})\\\\), where \\\\(k=12\\\\) in our case. Understanding this involves stuff like the Frobenius endomorphism.\n\n\n## references\n- https://hackmd.io/@benjaminion/bls12-381\n- [initial post of bls12-381](https://electriccoin.co/blog/new-snark-curve/)\n- [implementation: blst](https://github.com/supranational/blst)","slug":"cryptography/elliptic_curve/bls12-381","published":1,"updated":"2023-12-03T03:53:57.439Z","_id":"clpm1qei10000hpsj4saxbd96","comments":1,"layout":"post","photos":[],"link":"","content":"

introduction

BLS12-381 is a pairing-friendly elliptic curve. Curve BLS12-381 was designed by Sean Bowe in early 2017 as the foundation for an upgrade to the Zcash protocol. It is both pairing-friendly (making it efficient for digital signatures) and effective for constructing zkSnarks. BLS12-381 is part of a family of curves described by Barreto, Lynn, and Scott (BLS). The 12 is the embedding degree of the curve: neither too low, nor too high. The 381 is the number of bits needed to represent coordinates on the curve: the field modulus, \\(q\\). 381 is a fairly handy number as we can use 48 bytes per field element, with 3 bits left over for useful flags or arithmetic optimisations. This size of this number is guided both by security requirements and implementation efficiency.

\n

curve equation and parameters

The basic equation of the BLS12-381 curve is \\( y^2 = x^3 + 4\\)
The key parameters for a BLS curve are set using a single parameter \\(\\mathbf{x}\\) (different from the in the curve equation!). BLS12-381 is derived from the \\( k \\equiv 0 \\mod 6\\) case of Construction 6.6 in the taxonomy.

\n

Specific design goals for BLS12-381 are:

\n
    \n
  • \\(\\mathbf{x}\\) has “low hamming weight”, meaning that it has very few bits set to 1. This is particularly important for the efficiency of the algorithm that calculates pairings (the Miller loop).
    • \n
  • The field modulus \\(q\\) mentioned above is prime and has 383 bits or fewer, which makes 64-bit or 32-bit arithmetic on it more efficient.
    • \n
  • The order \\(r\\) of the subgroups we use is prime and has 255 bits or fewer, which is good for the same reason as above.
    • \n
  • To support zkSnark schemes, we want to have a large power of two root of unity in the field. This means we want \\(2^n\\) to be a factor of \\(r-1\\), for some biggish \\(n\\). (Making \\(\\mathbf{x}\\) a multiple of \\(2^{n/2}\\) will achieve this.) This property is key to being able to use fast Fourier transforms for interesting things like polynomial multiplication.
    • \n
\n

The value \\(\\mathbf{x}\\) = -0xd201000000010000 (hexadecimal, note that it is negative) gives the largest and the lowest Hamming weight meeting these criteria. With this value we have,

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
parametersequationvalue(hex)comments
Field modulus \\(q\\)\\(\\frac{1}{3}(\\mathbf{x}-1)^2(\\mathbf{x}^4-\\mathbf{x}^2+1) + \\mathbf{x}\\)0x1a0111ea397fe69a4b1ba7b6434bacd764774b84f3
8512bf6730d2a0f6b0f6241eabfffeb153ffffb9feffffffffaaab		381 bits, prime
Subgroup size \\(r\\)\\((\\mathbf{x}^4-\\mathbf{x}^2+1)\\)0x73eda753299d7d483339d80809a1d80553
bda402fffe5bfeffffffff00000001		255 bits, prime
\n

Field extensions

Field extensions are fundamental to elliptic curve pairings. The “12” is BLS12-381 is not only the embedding degree, it is also (relatedly) the degree of field extension that we will need to use.
For example, The complex numbers are a quadratic extension of the real numbers (\\(x^2 = 1\\)). Complex numbers can’t be extended any further because there are no irreducible polynomials over the complex numbers. But for finite fields, if we can find an irreducible \\(k\\)-degree polynomial in our field \\(F_q\\), and we often can, then we are able to extend the field to \\(F_{q^k}\\), , and represent the elements of the extended field as degree \\(k-1\\) polynomials, \\(a_0 + a_1 x + … + a_{k-1}x^{k-1}\\). we can represent this compactly as (\\(a_0, a_1, …, a_{k-1}\\))
In practice, large extension fields like \\(F_{q^{12}}\\)are implemented as towers of smaller extensions.

\n

the curves

BLS12-381 is really dealing with two curves. Both curves share more-or-less the same curve equation, but are defined over different fields.
The simpler one is over the finite field \\(F_q\\), which is just the integers mod \\(q\\). So the curve has points only where the equation \\(y^2 = x^3 +4\\) has solutions with \\(x\\) and \\(y\\) both integers less than \\(q\\). We shall call this curve \\(E(F_q)\\).
The other curve is defined over an extension of \\(F_q\\)to \\(F_{q^2}\\) (think complex numbers). In this case, the curve equation is slightly modified to be \\(y^2 = x^3+4(1+i)\\), and we call the curve \\(E’(F_{q^2})\\)

\n

the subgroups

A pairing is a bilinear map. This means that it takes as input two points, each from a group of the same order, \\(r\\). for rather technical reasons, these two groups need to be distinct. Let’s call them
\\(G_1\\) and \\(G_2\\).
Unfortunately, our simple curve \\(E(F_q)\\) has only a single large subgroup of order \\(r\\), so we can’t define a pairing based solely on \\(E(F_q)\\). However, if we keep extending the field over which
\\(E\\) is defined, it can be proved that we eventually find a curve that has more than one subgroup of order \\(r\\). that is, for some \\(k\\), \\(E(F_{q^k})\\) contains other subgroups of order \\(r\\) that we can use. One of these subgroups contains only points having a trace of zero[1], and we choose that subgroup to be \\(G_2\\)

\n

This number \\(k\\), the amount that we need to extend the base field by to find the new group, is called the embedding degree of the curve, which in our case is the “12” in BLS12-381. For completeness, note that each of \\(G_1\\) and \\(G_2\\) shares with its containing curve the “point at infinity”. This is the identity element of the elliptic curve arithmetic group, often denoted \\(\\mathcal O\\)
. For any point \\(P\\), \\( P + \\mathcal O = \\mathcal O + P = P\\).

\n

Twists

But there’s another challenge. As discussed earlier, doing arithmetic in \\(F_{q^{12}}\\) is horribly complicated and inefficient. And curve operations need a lot of arithmetic. A twist is something like a coordinate transformation. Rather wonderfully, this can be used to transform our \\(E(F_{q^{12}})\\) curve into a curve defined over a lower degree field that still has an order \\(r\\) subgroup. Moreover, this subgroup has a simple mapping to and from our \\(G_2\\) group

\n

BLS12-381 uses a “sextic twist”. This means that it reduces the degree of the extension field by a factor of six. So \\(G_2\\) on the twisted curve can be defined over \\(F_{q^2}\\) instead of \\(F_{q^{12}}\\)
I haven’t seen this written down anywhere—but attempting to decode section 3 of this—if we find a \\(u\\) such that \\(u^6 = (1+i)^{-1}\\), then we can define our twisting transformation as \\( (x,y) -> (x/u^2, y/u^3)\\). This transforms our original curve \\(E: y^2 = x^3 + 4\\) into the curve \\(E’: y^2 + 4/u^6 = x^3 + 4(1+i)\\). \\(E\\) and \\(E’\\) look different, but are actually the same object presented with respect to coefficinets in different base fields.

\n

So these are the two groups we will be using:

\n
    \n
  • \\(G_1 \\subset E(F_q)\\), where \\(E: y^2 = x^3 + 4\\)
    • \n
  • \\(G_2 \\subset E(F_{q^2})\\), where \\(E: y^2 = x^3 + 4(1+i)\\)
    Note that coordinates of points in the \\(G_1\\) group are pairs of integers, and coordinates of points in the \\(G_2\\) group are pairs of complex integers.
    • \n
\n

Parings

s far as BLS12-381 is concerned, a pairing simply takes a point \\(P \\in G_1 \\subset E(F_q)\\), and a point \\(Q \\in G_2 \\subset E’(F_{q^2})\\) and outputs a point from a group \\(G_T \\subset F_{q^{12}}\\). That is, for a paring \\(e\\), \\(e: G_1 \\times G_2 \\rightarrow G_T\\)

\n

properties of pairing
\\( e(P, Q+R) = e(P,Q) \\cdot e(P,R) \\),
note as a way of memory, can think it as \\(P^{Q+R} = P^Q \\cdot P^R\\)
\\( e(P+S, R) = e(P,R) \\cdot e(S,R)\\)
From this, we can deduce that all of the following identities hold:
\\(e([a]P, [b]Q) = e(P, [b]Q)^a = e(P,Q)^{ab} = e(P, [a]Q)^b = e([b]P, [a]Q)\\)

\n

Embedding degree

The embedding degree, \\(k\\), is calculated as the smallest positive integer such that \\(r\\) divides \\(q^k -1\\). So, in the case of BLS12-381, \\(r\\) is a factor of \\(q^{12} -1\\), but not of any lower power.
The choice of an embedding degree is a balance between security and efficiency (as ever). On the security front, the embedding degree is also known as the security multiplier: a higher embedding degree makes the discrete logarithm problem harder to solve in
. However, a high embedding degree means we have to do field operations in high-degree extensions, which is clunky and inefficient.

\n

cofactor

A subgroup’s cofactor is the ratio of the size of the whole group to the size of the subgroup.

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
GroupCofactorEquationvalue(hex)
\\(G_1\\)\\(h_1\\)\\(\\frac{\\mathbf{x-1}^2}{3}\\)0x396c8c005555e
1568c00aaab0000aaab
\\(G_2\\)\\(h_2\\)\\(\\frac{\\mathbf{x}^8 -4\\mathbf{x}^7+5\\mathbf{x}^6-4\\mathbf{x}^4+6\\mathbf{x}^3-4\\mathbf{x}^2-4\\mathbf{x}+13}{9}\\)0x5d543a95414e7f1091d50792876a202cd91de
4547085abaa68a205b2e5a7ddfa628f1cb4d9e82ef2
1537e293a6691ae1616ec6e786f0c70cf1c38e31c7238e5
\n

note multiplying by the cofactor turns out to be a straightforward way to map any arbitrary point on the elliptic curve into the respective subgroup \\(G_1\\) or \\(G_2\\)

\n

Generators

\\(G_1\\) and \\(G_2\\) are cyclic groups of prime order, so any point (except the identity/point at infinity) is a generator. Thus, picking generators is just a matter of convention.
Generator points \\(G_1\\) and \\(G_2\\) are specified in decimal here

\n

These were chosen as follows:

\n
\n

The generators of \\(G_1\\) and \\(G_2\\) are computed by finding the lexicographically smallest valid x-coordinate, and its lexicographically smallest > y-coordinate and scaling it by the cofactor such that the result is not the point at infinity.

\n
\n

Foot notes

[1] The “trace zero subgroup” qualifies as an obscure incantation. Basically, the trace of a point is \\(\\sum_{i=0}^{k-1}(x^{q^i}, y^{q^i})\\), where \\(k=12\\) in our case. Understanding this involves stuff like the Frobenius endomorphism.

\n

references

\n","site":{"data":{}},"excerpt":"","more":"

introduction

BLS12-381 is a pairing-friendly elliptic curve. Curve BLS12-381 was designed by Sean Bowe in early 2017 as the foundation for an upgrade to the Zcash protocol. It is both pairing-friendly (making it efficient for digital signatures) and effective for constructing zkSnarks. BLS12-381 is part of a family of curves described by Barreto, Lynn, and Scott (BLS). The 12 is the embedding degree of the curve: neither too low, nor too high. The 381 is the number of bits needed to represent coordinates on the curve: the field modulus, \\(q\\). 381 is a fairly handy number as we can use 48 bytes per field element, with 3 bits left over for useful flags or arithmetic optimisations. This size of this number is guided both by security requirements and implementation efficiency.

\n

curve equation and parameters

The basic equation of the BLS12-381 curve is \\( y^2 = x^3 + 4\\)
The key parameters for a BLS curve are set using a single parameter \\(\\mathbf{x}\\) (different from the in the curve equation!). BLS12-381 is derived from the \\( k \\equiv 0 \\mod 6\\) case of Construction 6.6 in the taxonomy.

\n

Specific design goals for BLS12-381 are:

\n
    \n
  • \\(\\mathbf{x}\\) has “low hamming weight”, meaning that it has very few bits set to 1. This is particularly important for the efficiency of the algorithm that calculates pairings (the Miller loop).
    • \n
  • The field modulus \\(q\\) mentioned above is prime and has 383 bits or fewer, which makes 64-bit or 32-bit arithmetic on it more efficient.
    • \n
  • The order \\(r\\) of the subgroups we use is prime and has 255 bits or fewer, which is good for the same reason as above.
    • \n
  • To support zkSnark schemes, we want to have a large power of two root of unity in the field. This means we want \\(2^n\\) to be a factor of \\(r-1\\), for some biggish \\(n\\). (Making \\(\\mathbf{x}\\) a multiple of \\(2^{n/2}\\) will achieve this.) This property is key to being able to use fast Fourier transforms for interesting things like polynomial multiplication.
    • \n
\n

The value \\(\\mathbf{x}\\) = -0xd201000000010000 (hexadecimal, note that it is negative) gives the largest and the lowest Hamming weight meeting these criteria. With this value we have,

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
parametersequationvalue(hex)comments
Field modulus \\(q\\)\\(\\frac{1}{3}(\\mathbf{x}-1)^2(\\mathbf{x}^4-\\mathbf{x}^2+1) + \\mathbf{x}\\)0x1a0111ea397fe69a4b1ba7b6434bacd764774b84f3
8512bf6730d2a0f6b0f6241eabfffeb153ffffb9feffffffffaaab		381 bits, prime
Subgroup size \\(r\\)\\((\\mathbf{x}^4-\\mathbf{x}^2+1)\\)0x73eda753299d7d483339d80809a1d80553
bda402fffe5bfeffffffff00000001		255 bits, prime
\n

Field extensions

Field extensions are fundamental to elliptic curve pairings. The “12” is BLS12-381 is not only the embedding degree, it is also (relatedly) the degree of field extension that we will need to use.
For example, The complex numbers are a quadratic extension of the real numbers (\\(x^2 = 1\\)). Complex numbers can’t be extended any further because there are no irreducible polynomials over the complex numbers. But for finite fields, if we can find an irreducible \\(k\\)-degree polynomial in our field \\(F_q\\), and we often can, then we are able to extend the field to \\(F_{q^k}\\), , and represent the elements of the extended field as degree \\(k-1\\) polynomials, \\(a_0 + a_1 x + … + a_{k-1}x^{k-1}\\). we can represent this compactly as (\\(a_0, a_1, …, a_{k-1}\\))
In practice, large extension fields like \\(F_{q^{12}}\\)are implemented as towers of smaller extensions.

\n

the curves

BLS12-381 is really dealing with two curves. Both curves share more-or-less the same curve equation, but are defined over different fields.
The simpler one is over the finite field \\(F_q\\), which is just the integers mod \\(q\\). So the curve has points only where the equation \\(y^2 = x^3 +4\\) has solutions with \\(x\\) and \\(y\\) both integers less than \\(q\\). We shall call this curve \\(E(F_q)\\).
The other curve is defined over an extension of \\(F_q\\)to \\(F_{q^2}\\) (think complex numbers). In this case, the curve equation is slightly modified to be \\(y^2 = x^3+4(1+i)\\), and we call the curve \\(E’(F_{q^2})\\)

\n

the subgroups

A pairing is a bilinear map. This means that it takes as input two points, each from a group of the same order, \\(r\\). for rather technical reasons, these two groups need to be distinct. Let’s call them
\\(G_1\\) and \\(G_2\\).
Unfortunately, our simple curve \\(E(F_q)\\) has only a single large subgroup of order \\(r\\), so we can’t define a pairing based solely on \\(E(F_q)\\). However, if we keep extending the field over which
\\(E\\) is defined, it can be proved that we eventually find a curve that has more than one subgroup of order \\(r\\). that is, for some \\(k\\), \\(E(F_{q^k})\\) contains other subgroups of order \\(r\\) that we can use. One of these subgroups contains only points having a trace of zero[1], and we choose that subgroup to be \\(G_2\\)

\n

This number \\(k\\), the amount that we need to extend the base field by to find the new group, is called the embedding degree of the curve, which in our case is the “12” in BLS12-381. For completeness, note that each of \\(G_1\\) and \\(G_2\\) shares with its containing curve the “point at infinity”. This is the identity element of the elliptic curve arithmetic group, often denoted \\(\\mathcal O\\)
. For any point \\(P\\), \\( P + \\mathcal O = \\mathcal O + P = P\\).

\n

Twists

But there’s another challenge. As discussed earlier, doing arithmetic in \\(F_{q^{12}}\\) is horribly complicated and inefficient. And curve operations need a lot of arithmetic. A twist is something like a coordinate transformation. Rather wonderfully, this can be used to transform our \\(E(F_{q^{12}})\\) curve into a curve defined over a lower degree field that still has an order \\(r\\) subgroup. Moreover, this subgroup has a simple mapping to and from our \\(G_2\\) group

\n

BLS12-381 uses a “sextic twist”. This means that it reduces the degree of the extension field by a factor of six. So \\(G_2\\) on the twisted curve can be defined over \\(F_{q^2}\\) instead of \\(F_{q^{12}}\\)
I haven’t seen this written down anywhere—but attempting to decode section 3 of this—if we find a \\(u\\) such that \\(u^6 = (1+i)^{-1}\\), then we can define our twisting transformation as \\( (x,y) -> (x/u^2, y/u^3)\\). This transforms our original curve \\(E: y^2 = x^3 + 4\\) into the curve \\(E’: y^2 + 4/u^6 = x^3 + 4(1+i)\\). \\(E\\) and \\(E’\\) look different, but are actually the same object presented with respect to coefficinets in different base fields.

\n

So these are the two groups we will be using:

\n
    \n
  • \\(G_1 \\subset E(F_q)\\), where \\(E: y^2 = x^3 + 4\\)
    • \n
  • \\(G_2 \\subset E(F_{q^2})\\), where \\(E: y^2 = x^3 + 4(1+i)\\)
    Note that coordinates of points in the \\(G_1\\) group are pairs of integers, and coordinates of points in the \\(G_2\\) group are pairs of complex integers.
    • \n
\n

Parings

s far as BLS12-381 is concerned, a pairing simply takes a point \\(P \\in G_1 \\subset E(F_q)\\), and a point \\(Q \\in G_2 \\subset E’(F_{q^2})\\) and outputs a point from a group \\(G_T \\subset F_{q^{12}}\\). That is, for a paring \\(e\\), \\(e: G_1 \\times G_2 \\rightarrow G_T\\)

\n

properties of pairing
\\( e(P, Q+R) = e(P,Q) \\cdot e(P,R) \\),
note as a way of memory, can think it as \\(P^{Q+R} = P^Q \\cdot P^R\\)
\\( e(P+S, R) = e(P,R) \\cdot e(S,R)\\)
From this, we can deduce that all of the following identities hold:
\\(e([a]P, [b]Q) = e(P, [b]Q)^a = e(P,Q)^{ab} = e(P, [a]Q)^b = e([b]P, [a]Q)\\)

\n

Embedding degree

The embedding degree, \\(k\\), is calculated as the smallest positive integer such that \\(r\\) divides \\(q^k -1\\). So, in the case of BLS12-381, \\(r\\) is a factor of \\(q^{12} -1\\), but not of any lower power.
The choice of an embedding degree is a balance between security and efficiency (as ever). On the security front, the embedding degree is also known as the security multiplier: a higher embedding degree makes the discrete logarithm problem harder to solve in
. However, a high embedding degree means we have to do field operations in high-degree extensions, which is clunky and inefficient.

\n

cofactor

A subgroup’s cofactor is the ratio of the size of the whole group to the size of the subgroup.

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
GroupCofactorEquationvalue(hex)
\\(G_1\\)\\(h_1\\)\\(\\frac{\\mathbf{x-1}^2}{3}\\)0x396c8c005555e
1568c00aaab0000aaab
\\(G_2\\)\\(h_2\\)\\(\\frac{\\mathbf{x}^8 -4\\mathbf{x}^7+5\\mathbf{x}^6-4\\mathbf{x}^4+6\\mathbf{x}^3-4\\mathbf{x}^2-4\\mathbf{x}+13}{9}\\)0x5d543a95414e7f1091d50792876a202cd91de
4547085abaa68a205b2e5a7ddfa628f1cb4d9e82ef2
1537e293a6691ae1616ec6e786f0c70cf1c38e31c7238e5
\n

note multiplying by the cofactor turns out to be a straightforward way to map any arbitrary point on the elliptic curve into the respective subgroup \\(G_1\\) or \\(G_2\\)

\n

Generators

\\(G_1\\) and \\(G_2\\) are cyclic groups of prime order, so any point (except the identity/point at infinity) is a generator. Thus, picking generators is just a matter of convention.
Generator points \\(G_1\\) and \\(G_2\\) are specified in decimal here

\n

These were chosen as follows:

\n
\n

The generators of \\(G_1\\) and \\(G_2\\) are computed by finding the lexicographically smallest valid x-coordinate, and its lexicographically smallest > y-coordinate and scaling it by the cofactor such that the result is not the point at infinity.

\n
\n

Foot notes

[1] The “trace zero subgroup” qualifies as an obscure incantation. Basically, the trace of a point is \\(\\sum_{i=0}^{k-1}(x^{q^i}, y^{q^i})\\), where \\(k=12\\) in our case. Understanding this involves stuff like the Frobenius endomorphism.

\n

references

\n"},{"title":"kzg polynomial commitment","date":"2023-12-03T03:19:44.000Z","_content":"\n\n\n\n## introduction\nKZG polynomial commitments was introduced by Kate, Zaverucha and Goldberg. It is called a commitment, because having sent the commitment value (an elliptic curve point) to someone (the verifier), the prover cannot change the polynomial they are working with.\n\n## Comparison to Merkle trees\nA Merkle tree is what cryptographers call a vector commitment: Using a Merkle tree of depth \\\\(d\\\\), you can compute a commitment to a vector. A polynomial commitment can be achived by making a merkle tree commitment of all polynomials coefficients. however, it is not efficient.\n\n## parings\nLet \\\\(\\mathbb{G_1}\\\\) and \\\\(\\mathbb{G_2}\\\\) be two elliptic curves wit a paring \\\\(e: \\mathbb{G_1} \\times \\mathbb{G_2} \\rightarrow \\mathbb{G_T}\\\\). Let \\\\(p\\\\) be the order of \\\\(\\mathbb{G_1}\\\\) and \\\\(\\mathbb{G_2}\\\\), and \\\\(G\\\\) and \\\\(H\\\\) be generators of \\\\(\\mathbb{G_1}\\\\) and \\\\(\\mathbb{G_2}\\\\). We will use a very useful shorhand notation\n\\\\( [x]_1 = xG \\in \\mathbb{G_1}\\\\), and \\\\([x]_2 = xH \\in \\mathbb{G_2} \\\\), for any \\\\(x \\in \\mathbb{F_p}\\\\)\n\n## Trusted setup\nLet’s assume we have a trusted setup, so that for some secret \\\\(s\\\\), the elements \\\\([s^i]_1\\\\) and \\\\([s^i]_2\\\\) are available to both prover and verifier for \\\\(i=0, ..., n-1\\\\). \n\nLet \\\\(p(X) = \\sum_{i=0}^{n}p_i X^i \\\\) be a polynomial, the prover can compute\n\n\n\\\\( \\left[ p(s) \\right]_1 = p_0[s^0]_1 + p_1[s^1]_1 + ... + p_n[s^n]_1 \\\\)\n\n\n## Kate commitment\nIn the Kate commitment scheme, the eleemnt \\\\(C = [p(s)]_1\\\\) is the commitment to the polynomial \\\\(p(X)\\\\). Could the prover (without knowing \\\\(s\\\\)) find another polynomial \\\\(q(X) \\neq p(X)\\\\) that has the same commitment. Let's assume that this were the case. Then it would mean \\\\([p(s) - q(s)]_1 = 0\\\\), implying \\\\(p(s) - q(s) = 0\\\\).\n\nNow, \\\\(r(X) = p(X) - q(X)\\\\) is itself a polynomial. we know that it's not constant because \\\\(p(X) \\neq q(X) \\\\). It is a well-known fact that any non-constant polynomial of degree \\\\(n\\\\) can have at most \\\\(n\\\\) zeroes. \nSince the prover doesn’t know \\\\(s\\\\), the only way they could achieve that \\\\(p(s) - q(s) = 0\\\\)\n is by making \\\\( p(X) - q(X)=0\\\\) in as many places as possible. But since they can do that in at most \\\\(n\\\\)\n places, as we’ve just proved, they are very unlikely to succeed: since \\\\(n\\\\) is much smaller than the order of the curve, \\\\(p\\\\). the probability that \\\\(s\\\\) will be one of the points they chose to make \\\\(p(X) = q(X)\\\\) will be vanishingly tiny.\n\n## Multiplying polynomials\nWhile elliptic curves themselves don’t allow polynomial multiplication, luckily we can do it with pairings: We have that\n\\\\[ e([a]_1, [b]_2) = e(G,H)^{(ab)} = [ab]_T\\\\]\nwhile we unfortunately can’t just multiply two field elements inside an elliptic curve and get their product as an elliptic curve element, we can multiply two field elements if we commited to them in different curves (one in \\\\(G_1\\\\) and one in \\\\(G_2\\\\), and the output is a \n\\\\(G_T\\\\) element.\n\nNoe let's say we want to prove that \\\\(p(z)=y\\\\). we will use the polynomial \\\\(p(X) - y\\\\). this polynomial is clearly zero at \\\\(z\\\\). Let \\\\(q(X)\\\\) be the polynomial \\\\(p(X) - y\\\\) divided by the linear factor \\\\(X-z\\\\), i.e\n\\\\[ q(X) = \\frac{p(X) - y}{X-z} \\\\]\nwhich is equivalent to saying that \\\\(q(X)(X-z) = p(X) - y\\\\)\n\n## Kate proofs\nnow a kate proof for the evaluation \\\\(p(z) = y\\\\) is defined as \\\\(\\pi = [q(s)]_1\\\\). remember the commitment to the polynomial \\\\(p(X)\\\\) is defined as \\\\(C=[p(s)]_1\\\\)\nthe verifier checks this proof using the following equation\n\\\\( e(\\pi, [s-z]_2) = e(C-[y]_1, H) \\\\)\n\nNote that the verifier can compute \\\\([s-z]_2\\\\), because it is just a combination of the element \\\\([s]_2\\\\)\n from the trusted setup and \\\\(z\\\\) is the point at which the polynomial is evaluated. \n\n## multiproofs\nSo far we have shown how to prove an evaluation of a polynomial at a single point. Note that this is already pretty amazing: You can show that some polynomial that could be any degree – say \\\\(2^{28}\\\\) – at some point takes a certain value, by only sending a single group element (that could be 48 bytes, for example in BLS12_381). The toy example of using Merkle trees as polynomial commitments would have needed to send \\\\(2^{28}\\\\) elements – all the coefficients of the polynomial.\n\nLet’s say we have a list of \\\\(k\\\\) points \\\\((z_0, y_0),(z_1, y_1), ..., (z_{k-1}, y_{k-1})\\\\). Using lagrange interpolation, we can find polynomial \\\\(I(X)\\\\) goes through all of these points.\n\nNow let's assume that we know \\\\(p(X)\\\\) goes through all these points. then the polynomial \\\\(p(X) - I(X)\\\\) will clearly be zero at each \\\\(z_0, z_1, ..., z_{k-1}\\\\). the zero polynomial is\n\\\\(Z(X) = (X-z_0)\\cdot (X-z_1)...(X-z_{k-1})\\\\)\nNow, we can compute the quotient\n\\\\(q(X) = \\frac{p(X)-I(X)}{Z(X)}\\\\)\nWe can now define the Kate multiproof for the evaluations \\\\((z_0, y_0),(z_1, y_1),..., (z_{k-1}, y_{k-1})\\\\): \\\\(\\pi = [q(s)]_1\\\\). note that this is still only one group element.\n\nNow, to check this, the verifier will also have to compute the interpolation polynomial \\\\(I(X)\\\\) and the zero polynomial Z(X). Using this, they can compute \\\\([Z(s)]_2\\\\) and \\\\([I(s)]_1\\\\), and thus verify the paring equation\n\\\\[ e(\\pi, [Z(s)]_2) = e(C-[I(s)]_1, H) \\\\]\n\n## Kate as a vector commitment\nWhile the Kate commitment scheme is designed as a polynomial commitment, it actually also makes a really nice vector commitment. Remember that a vector commitment commits to a vector \\\\(a_0, ..., a_{n-1}\\\\) and lets you prove that you committed to \\\\(a_i\\\\) for some \\\\(i\\\\). We can reproduce this using the Kate commitment scheme: Let \\\\(p(X)\\\\) be the polynomial that for all \\\\(i\\\\) evaluates as \\\\(p(i)=a_i\\\\). \nNow using this polynomial, we can prove any number of elements in the vector using just a single group element! Note how much more efficient (in terms of proof size) this is compared to Merkle trees\n\n## references\n[Dankrad Feist Post](https://dankradfeist.de/ethereum/2020/06/16/kate-polynomial-commitments.html)\n","source":"_posts/cryptography/zkp/kzg-commitment.md","raw":"---\ntitle: kzg polynomial commitment\ndate: 2023-12-03 11:19:44\ntags: [cryptography,zkp]\n---\n\n\n\n\n## introduction\nKZG polynomial commitments was introduced by Kate, Zaverucha and Goldberg. It is called a commitment, because having sent the commitment value (an elliptic curve point) to someone (the verifier), the prover cannot change the polynomial they are working with.\n\n## Comparison to Merkle trees\nA Merkle tree is what cryptographers call a vector commitment: Using a Merkle tree of depth \\\\(d\\\\), you can compute a commitment to a vector. A polynomial commitment can be achived by making a merkle tree commitment of all polynomials coefficients. however, it is not efficient.\n\n## parings\nLet \\\\(\\mathbb{G_1}\\\\) and \\\\(\\mathbb{G_2}\\\\) be two elliptic curves wit a paring \\\\(e: \\mathbb{G_1} \\times \\mathbb{G_2} \\rightarrow \\mathbb{G_T}\\\\). Let \\\\(p\\\\) be the order of \\\\(\\mathbb{G_1}\\\\) and \\\\(\\mathbb{G_2}\\\\), and \\\\(G\\\\) and \\\\(H\\\\) be generators of \\\\(\\mathbb{G_1}\\\\) and \\\\(\\mathbb{G_2}\\\\). We will use a very useful shorhand notation\n\\\\( [x]_1 = xG \\in \\mathbb{G_1}\\\\), and \\\\([x]_2 = xH \\in \\mathbb{G_2} \\\\), for any \\\\(x \\in \\mathbb{F_p}\\\\)\n\n## Trusted setup\nLet’s assume we have a trusted setup, so that for some secret \\\\(s\\\\), the elements \\\\([s^i]_1\\\\) and \\\\([s^i]_2\\\\) are available to both prover and verifier for \\\\(i=0, ..., n-1\\\\). \n\nLet \\\\(p(X) = \\sum_{i=0}^{n}p_i X^i \\\\) be a polynomial, the prover can compute\n\n\n\\\\( \\left[ p(s) \\right]_1 = p_0[s^0]_1 + p_1[s^1]_1 + ... + p_n[s^n]_1 \\\\)\n\n\n## Kate commitment\nIn the Kate commitment scheme, the eleemnt \\\\(C = [p(s)]_1\\\\) is the commitment to the polynomial \\\\(p(X)\\\\). Could the prover (without knowing \\\\(s\\\\)) find another polynomial \\\\(q(X) \\neq p(X)\\\\) that has the same commitment. Let's assume that this were the case. Then it would mean \\\\([p(s) - q(s)]_1 = 0\\\\), implying \\\\(p(s) - q(s) = 0\\\\).\n\nNow, \\\\(r(X) = p(X) - q(X)\\\\) is itself a polynomial. we know that it's not constant because \\\\(p(X) \\neq q(X) \\\\). It is a well-known fact that any non-constant polynomial of degree \\\\(n\\\\) can have at most \\\\(n\\\\) zeroes. \nSince the prover doesn’t know \\\\(s\\\\), the only way they could achieve that \\\\(p(s) - q(s) = 0\\\\)\n is by making \\\\( p(X) - q(X)=0\\\\) in as many places as possible. But since they can do that in at most \\\\(n\\\\)\n places, as we’ve just proved, they are very unlikely to succeed: since \\\\(n\\\\) is much smaller than the order of the curve, \\\\(p\\\\). the probability that \\\\(s\\\\) will be one of the points they chose to make \\\\(p(X) = q(X)\\\\) will be vanishingly tiny.\n\n## Multiplying polynomials\nWhile elliptic curves themselves don’t allow polynomial multiplication, luckily we can do it with pairings: We have that\n\\\\[ e([a]_1, [b]_2) = e(G,H)^{(ab)} = [ab]_T\\\\]\nwhile we unfortunately can’t just multiply two field elements inside an elliptic curve and get their product as an elliptic curve element, we can multiply two field elements if we commited to them in different curves (one in \\\\(G_1\\\\) and one in \\\\(G_2\\\\), and the output is a \n\\\\(G_T\\\\) element.\n\nNoe let's say we want to prove that \\\\(p(z)=y\\\\). we will use the polynomial \\\\(p(X) - y\\\\). this polynomial is clearly zero at \\\\(z\\\\). Let \\\\(q(X)\\\\) be the polynomial \\\\(p(X) - y\\\\) divided by the linear factor \\\\(X-z\\\\), i.e\n\\\\[ q(X) = \\frac{p(X) - y}{X-z} \\\\]\nwhich is equivalent to saying that \\\\(q(X)(X-z) = p(X) - y\\\\)\n\n## Kate proofs\nnow a kate proof for the evaluation \\\\(p(z) = y\\\\) is defined as \\\\(\\pi = [q(s)]_1\\\\). remember the commitment to the polynomial \\\\(p(X)\\\\) is defined as \\\\(C=[p(s)]_1\\\\)\nthe verifier checks this proof using the following equation\n\\\\( e(\\pi, [s-z]_2) = e(C-[y]_1, H) \\\\)\n\nNote that the verifier can compute \\\\([s-z]_2\\\\), because it is just a combination of the element \\\\([s]_2\\\\)\n from the trusted setup and \\\\(z\\\\) is the point at which the polynomial is evaluated. \n\n## multiproofs\nSo far we have shown how to prove an evaluation of a polynomial at a single point. Note that this is already pretty amazing: You can show that some polynomial that could be any degree – say \\\\(2^{28}\\\\) – at some point takes a certain value, by only sending a single group element (that could be 48 bytes, for example in BLS12_381). The toy example of using Merkle trees as polynomial commitments would have needed to send \\\\(2^{28}\\\\) elements – all the coefficients of the polynomial.\n\nLet’s say we have a list of \\\\(k\\\\) points \\\\((z_0, y_0),(z_1, y_1), ..., (z_{k-1}, y_{k-1})\\\\). Using lagrange interpolation, we can find polynomial \\\\(I(X)\\\\) goes through all of these points.\n\nNow let's assume that we know \\\\(p(X)\\\\) goes through all these points. then the polynomial \\\\(p(X) - I(X)\\\\) will clearly be zero at each \\\\(z_0, z_1, ..., z_{k-1}\\\\). the zero polynomial is\n\\\\(Z(X) = (X-z_0)\\cdot (X-z_1)...(X-z_{k-1})\\\\)\nNow, we can compute the quotient\n\\\\(q(X) = \\frac{p(X)-I(X)}{Z(X)}\\\\)\nWe can now define the Kate multiproof for the evaluations \\\\((z_0, y_0),(z_1, y_1),..., (z_{k-1}, y_{k-1})\\\\): \\\\(\\pi = [q(s)]_1\\\\). note that this is still only one group element.\n\nNow, to check this, the verifier will also have to compute the interpolation polynomial \\\\(I(X)\\\\) and the zero polynomial Z(X). Using this, they can compute \\\\([Z(s)]_2\\\\) and \\\\([I(s)]_1\\\\), and thus verify the paring equation\n\\\\[ e(\\pi, [Z(s)]_2) = e(C-[I(s)]_1, H) \\\\]\n\n## Kate as a vector commitment\nWhile the Kate commitment scheme is designed as a polynomial commitment, it actually also makes a really nice vector commitment. Remember that a vector commitment commits to a vector \\\\(a_0, ..., a_{n-1}\\\\) and lets you prove that you committed to \\\\(a_i\\\\) for some \\\\(i\\\\). We can reproduce this using the Kate commitment scheme: Let \\\\(p(X)\\\\) be the polynomial that for all \\\\(i\\\\) evaluates as \\\\(p(i)=a_i\\\\). \nNow using this polynomial, we can prove any number of elements in the vector using just a single group element! Note how much more efficient (in terms of proof size) this is compared to Merkle trees\n\n## references\n[Dankrad Feist Post](https://dankradfeist.de/ethereum/2020/06/16/kate-polynomial-commitments.html)\n","slug":"cryptography/zkp/kzg-commitment","published":1,"updated":"2023-12-11T10:07:27.075Z","_id":"clpoya0gr0000pksj12ah1gzj","comments":1,"layout":"post","photos":[],"link":"","content":"\n\n\n

introduction

KZG polynomial commitments was introduced by Kate, Zaverucha and Goldberg. It is called a commitment, because having sent the commitment value (an elliptic curve point) to someone (the verifier), the prover cannot change the polynomial they are working with.

\n

Comparison to Merkle trees

A Merkle tree is what cryptographers call a vector commitment: Using a Merkle tree of depth \\(d\\), you can compute a commitment to a vector. A polynomial commitment can be achived by making a merkle tree commitment of all polynomials coefficients. however, it is not efficient.

\n

parings

Let \\(\\mathbb{G_1}\\) and \\(\\mathbb{G_2}\\) be two elliptic curves wit a paring \\(e: \\mathbb{G_1} \\times \\mathbb{G_2} \\rightarrow \\mathbb{G_T}\\). Let \\(p\\) be the order of \\(\\mathbb{G_1}\\) and \\(\\mathbb{G_2}\\), and \\(G\\) and \\(H\\) be generators of \\(\\mathbb{G_1}\\) and \\(\\mathbb{G_2}\\). We will use a very useful shorhand notation
\\( [x]_1 = xG \\in \\mathbb{G_1}\\), and \\([x]_2 = xH \\in \\mathbb{G_2} \\), for any \\(x \\in \\mathbb{F_p}\\)

\n

Trusted setup

Let’s assume we have a trusted setup, so that for some secret \\(s\\), the elements \\([s^i]_1\\) and \\([s^i]_2\\) are available to both prover and verifier for \\(i=0, …, n-1\\).

\n

Let \\(p(X) = \\sum_{i=0}^{n}p_i X^i \\) be a polynomial, the prover can compute

\n

\\( \\left[ p(s) \\right]_1 = p_0[s^0]_1 + p_1[s^1]_1 + … + p_n[s^n]_1 \\)

\n

Kate commitment

In the Kate commitment scheme, the eleemnt \\(C = [p(s)]_1\\) is the commitment to the polynomial \\(p(X)\\). Could the prover (without knowing \\(s\\)) find another polynomial \\(q(X) \\neq p(X)\\) that has the same commitment. Let’s assume that this were the case. Then it would mean \\([p(s) - q(s)]_1 = 0\\), implying \\(p(s) - q(s) = 0\\).

\n

Now, \\(r(X) = p(X) - q(X)\\) is itself a polynomial. we know that it’s not constant because \\(p(X) \\neq q(X) \\). It is a well-known fact that any non-constant polynomial of degree \\(n\\) can have at most \\(n\\) zeroes.
Since the prover doesn’t know \\(s\\), the only way they could achieve that \\(p(s) - q(s) = 0\\)
is by making \\( p(X) - q(X)=0\\) in as many places as possible. But since they can do that in at most \\(n\\)
places, as we’ve just proved, they are very unlikely to succeed: since \\(n\\) is much smaller than the order of the curve, \\(p\\). the probability that \\(s\\) will be one of the points they chose to make \\(p(X) = q(X)\\) will be vanishingly tiny.

\n

Multiplying polynomials

While elliptic curves themselves don’t allow polynomial multiplication, luckily we can do it with pairings: We have that
\\[ e([a]_1, [b]_2) = e(G,H)^{(ab)} = [ab]_T\\]
while we unfortunately can’t just multiply two field elements inside an elliptic curve and get their product as an elliptic curve element, we can multiply two field elements if we commited to them in different curves (one in \\(G_1\\) and one in \\(G_2\\), and the output is a
\\(G_T\\) element.

\n

Noe let’s say we want to prove that \\(p(z)=y\\). we will use the polynomial \\(p(X) - y\\). this polynomial is clearly zero at \\(z\\). Let \\(q(X)\\) be the polynomial \\(p(X) - y\\) divided by the linear factor \\(X-z\\), i.e
\\[ q(X) = \\frac{p(X) - y}{X-z} \\]
which is equivalent to saying that \\(q(X)(X-z) = p(X) - y\\)

\n

Kate proofs

now a kate proof for the evaluation \\(p(z) = y\\) is defined as \\(\\pi = [q(s)]_1\\). remember the commitment to the polynomial \\(p(X)\\) is defined as \\(C=[p(s)]_1\\)
the verifier checks this proof using the following equation
\\( e(\\pi, [s-z]_2) = e(C-[y]_1, H) \\)

\n

Note that the verifier can compute \\([s-z]_2\\), because it is just a combination of the element \\([s]_2\\)
from the trusted setup and \\(z\\) is the point at which the polynomial is evaluated.

\n

multiproofs

So far we have shown how to prove an evaluation of a polynomial at a single point. Note that this is already pretty amazing: You can show that some polynomial that could be any degree – say \\(2^{28}\\) – at some point takes a certain value, by only sending a single group element (that could be 48 bytes, for example in BLS12_381). The toy example of using Merkle trees as polynomial commitments would have needed to send \\(2^{28}\\) elements – all the coefficients of the polynomial.

\n

Let’s say we have a list of \\(k\\) points \\((z_0, y_0),(z_1, y_1), …, (z_{k-1}, y_{k-1})\\). Using lagrange interpolation, we can find polynomial \\(I(X)\\) goes through all of these points.

\n

Now let’s assume that we know \\(p(X)\\) goes through all these points. then the polynomial \\(p(X) - I(X)\\) will clearly be zero at each \\(z_0, z_1, …, z_{k-1}\\). the zero polynomial is
\\(Z(X) = (X-z_0)\\cdot (X-z_1)…(X-z_{k-1})\\)
Now, we can compute the quotient
\\(q(X) = \\frac{p(X)-I(X)}{Z(X)}\\)
We can now define the Kate multiproof for the evaluations \\((z_0, y_0),(z_1, y_1),…, (z_{k-1}, y_{k-1})\\): \\(\\pi = [q(s)]_1\\). note that this is still only one group element.

\n

Now, to check this, the verifier will also have to compute the interpolation polynomial \\(I(X)\\) and the zero polynomial Z(X). Using this, they can compute \\([Z(s)]_2\\) and \\([I(s)]_1\\), and thus verify the paring equation
\\[ e(\\pi, [Z(s)]_2) = e(C-[I(s)]_1, H) \\]

\n

Kate as a vector commitment

While the Kate commitment scheme is designed as a polynomial commitment, it actually also makes a really nice vector commitment. Remember that a vector commitment commits to a vector \\(a_0, …, a_{n-1}\\) and lets you prove that you committed to \\(a_i\\) for some \\(i\\). We can reproduce this using the Kate commitment scheme: Let \\(p(X)\\) be the polynomial that for all \\(i\\) evaluates as \\(p(i)=a_i\\).
Now using this polynomial, we can prove any number of elements in the vector using just a single group element! Note how much more efficient (in terms of proof size) this is compared to Merkle trees

\n

references

Dankrad Feist Post

\n","site":{"data":{}},"excerpt":"","more":"\n\n\n

introduction

KZG polynomial commitments was introduced by Kate, Zaverucha and Goldberg. It is called a commitment, because having sent the commitment value (an elliptic curve point) to someone (the verifier), the prover cannot change the polynomial they are working with.

\n

Comparison to Merkle trees

A Merkle tree is what cryptographers call a vector commitment: Using a Merkle tree of depth \\(d\\), you can compute a commitment to a vector. A polynomial commitment can be achived by making a merkle tree commitment of all polynomials coefficients. however, it is not efficient.

\n

parings

Let \\(\\mathbb{G_1}\\) and \\(\\mathbb{G_2}\\) be two elliptic curves wit a paring \\(e: \\mathbb{G_1} \\times \\mathbb{G_2} \\rightarrow \\mathbb{G_T}\\). Let \\(p\\) be the order of \\(\\mathbb{G_1}\\) and \\(\\mathbb{G_2}\\), and \\(G\\) and \\(H\\) be generators of \\(\\mathbb{G_1}\\) and \\(\\mathbb{G_2}\\). We will use a very useful shorhand notation
\\( [x]_1 = xG \\in \\mathbb{G_1}\\), and \\([x]_2 = xH \\in \\mathbb{G_2} \\), for any \\(x \\in \\mathbb{F_p}\\)

\n

Trusted setup

Let’s assume we have a trusted setup, so that for some secret \\(s\\), the elements \\([s^i]_1\\) and \\([s^i]_2\\) are available to both prover and verifier for \\(i=0, …, n-1\\).

\n

Let \\(p(X) = \\sum_{i=0}^{n}p_i X^i \\) be a polynomial, the prover can compute

\n

\\( \\left[ p(s) \\right]_1 = p_0[s^0]_1 + p_1[s^1]_1 + … + p_n[s^n]_1 \\)

\n

Kate commitment

In the Kate commitment scheme, the eleemnt \\(C = [p(s)]_1\\) is the commitment to the polynomial \\(p(X)\\). Could the prover (without knowing \\(s\\)) find another polynomial \\(q(X) \\neq p(X)\\) that has the same commitment. Let’s assume that this were the case. Then it would mean \\([p(s) - q(s)]_1 = 0\\), implying \\(p(s) - q(s) = 0\\).

\n

Now, \\(r(X) = p(X) - q(X)\\) is itself a polynomial. we know that it’s not constant because \\(p(X) \\neq q(X) \\). It is a well-known fact that any non-constant polynomial of degree \\(n\\) can have at most \\(n\\) zeroes.
Since the prover doesn’t know \\(s\\), the only way they could achieve that \\(p(s) - q(s) = 0\\)
is by making \\( p(X) - q(X)=0\\) in as many places as possible. But since they can do that in at most \\(n\\)
places, as we’ve just proved, they are very unlikely to succeed: since \\(n\\) is much smaller than the order of the curve, \\(p\\). the probability that \\(s\\) will be one of the points they chose to make \\(p(X) = q(X)\\) will be vanishingly tiny.

\n

Multiplying polynomials

While elliptic curves themselves don’t allow polynomial multiplication, luckily we can do it with pairings: We have that
\\[ e([a]_1, [b]_2) = e(G,H)^{(ab)} = [ab]_T\\]
while we unfortunately can’t just multiply two field elements inside an elliptic curve and get their product as an elliptic curve element, we can multiply two field elements if we commited to them in different curves (one in \\(G_1\\) and one in \\(G_2\\), and the output is a
\\(G_T\\) element.

\n

Noe let’s say we want to prove that \\(p(z)=y\\). we will use the polynomial \\(p(X) - y\\). this polynomial is clearly zero at \\(z\\). Let \\(q(X)\\) be the polynomial \\(p(X) - y\\) divided by the linear factor \\(X-z\\), i.e
\\[ q(X) = \\frac{p(X) - y}{X-z} \\]
which is equivalent to saying that \\(q(X)(X-z) = p(X) - y\\)

\n

Kate proofs

now a kate proof for the evaluation \\(p(z) = y\\) is defined as \\(\\pi = [q(s)]_1\\). remember the commitment to the polynomial \\(p(X)\\) is defined as \\(C=[p(s)]_1\\)
the verifier checks this proof using the following equation
\\( e(\\pi, [s-z]_2) = e(C-[y]_1, H) \\)

\n

Note that the verifier can compute \\([s-z]_2\\), because it is just a combination of the element \\([s]_2\\)
from the trusted setup and \\(z\\) is the point at which the polynomial is evaluated.

\n

multiproofs

So far we have shown how to prove an evaluation of a polynomial at a single point. Note that this is already pretty amazing: You can show that some polynomial that could be any degree – say \\(2^{28}\\) – at some point takes a certain value, by only sending a single group element (that could be 48 bytes, for example in BLS12_381). The toy example of using Merkle trees as polynomial commitments would have needed to send \\(2^{28}\\) elements – all the coefficients of the polynomial.

\n

Let’s say we have a list of \\(k\\) points \\((z_0, y_0),(z_1, y_1), …, (z_{k-1}, y_{k-1})\\). Using lagrange interpolation, we can find polynomial \\(I(X)\\) goes through all of these points.

\n

Now let’s assume that we know \\(p(X)\\) goes through all these points. then the polynomial \\(p(X) - I(X)\\) will clearly be zero at each \\(z_0, z_1, …, z_{k-1}\\). the zero polynomial is
\\(Z(X) = (X-z_0)\\cdot (X-z_1)…(X-z_{k-1})\\)
Now, we can compute the quotient
\\(q(X) = \\frac{p(X)-I(X)}{Z(X)}\\)
We can now define the Kate multiproof for the evaluations \\((z_0, y_0),(z_1, y_1),…, (z_{k-1}, y_{k-1})\\): \\(\\pi = [q(s)]_1\\). note that this is still only one group element.

\n

Now, to check this, the verifier will also have to compute the interpolation polynomial \\(I(X)\\) and the zero polynomial Z(X). Using this, they can compute \\([Z(s)]_2\\) and \\([I(s)]_1\\), and thus verify the paring equation
\\[ e(\\pi, [Z(s)]_2) = e(C-[I(s)]_1, H) \\]

\n

Kate as a vector commitment

While the Kate commitment scheme is designed as a polynomial commitment, it actually also makes a really nice vector commitment. Remember that a vector commitment commits to a vector \\(a_0, …, a_{n-1}\\) and lets you prove that you committed to \\(a_i\\) for some \\(i\\). We can reproduce this using the Kate commitment scheme: Let \\(p(X)\\) be the polynomial that for all \\(i\\) evaluates as \\(p(i)=a_i\\).
Now using this polynomial, we can prove any number of elements in the vector using just a single group element! Note how much more efficient (in terms of proof size) this is compared to Merkle trees

\n

references

Dankrad Feist Post

\n"},{"title":"plonky2 code analysis","date":"2023-11-06T05:24:03.000Z","_content":"\n\n\n\n## introduction\nthis post is a source code analysis of https://github.com/0xPolygonZero/plonky2\n\n## 1. Config\n### 1.1 CircuitConfig\nthe default config is `Self::standard_recursion_config()` with value as below\n```rust\nSelf {\n num_wires: 135,\n num_routed_wires: 80, \n num_constants: 2, // used in ConstantGate, which hold the consant value for the circuit\n /// Whether to use a dedicated gate for base field arithmetic, rather than using a single gate\n /// for both base field and extension field arithmetic.\n use_base_arithmetic_gate: true,\n security_bits: 100,\n /// The number of challenge points to generate, for IOPs that have soundness errors of (roughly)\n /// `degree / |F|`.\n num_challenges: 2,\n zero_knowledge: false,\n /// A cap on the quotient polynomial's degree factor. The actual degree factor is derived\n /// systematically, but will never exceed this value.\n max_quotient_degree_factor: 8,\n fri_config: FriConfig {\n rate_bits: 3, // used in lde\n cap_height: 4,\n proof_of_work_bits: 16,\n reduction_strategy: FriReductionStrategy::ConstantArityBits(4, 5),\n num_query_rounds: 28,\n },\n}\n```\n\n**notes**\n- *routable wire* It's a wire that can be connected to other gates' wires, rather than an \"advice\" wire which is for internal state. For example if we had a gate for computing x^8, we'd have two routable wires for the input and output, x and x^8. But if we wanted lower-degree constraints, we could add advice wires for the intermediate results x^2 and x^4\n\n- *num_routed_wires*, the default value is 80. For eample, for a arithmetic gate, it computes `const_0 * multiplicand_0 * multiplicand_1 + const_1 * addend`. i.e 2 wires for multiplicands and 1 wire for the addend and 1 for output. total is 4. therefore, the gate can have `num_routed_wires/4` operations, in this case, it is 20. the circuit builder will add gate when number of operations exceeds 20.\n\n### 1.2 FriConfig\n```rust\npub struct FriConfig {\n/// `rate = 2^{-rate_bits}`. default is 3\npub rate_bits: usize, \n\n/// Height of Merkle tree caps. default is 4\npub cap_height: usize,\n/// default is 16\npub proof_of_work_bits: u32,\n\npub reduction_strategy: FriReductionStrategy,\n\n/// Number of query rounds to perform. default is 28\npub num_query_rounds: usize,\n}\n```\n\n## 2. Gates\n```rust\n/// A custom gate.\npub trait Gate, const D: usize>: 'static + Send + Sync {\n fn id(&self) -> String;\n /// The maximum degree among this gate's constraint polynomials. it is 3 for ArithmeticGate (Left, Right, Output); 7 for PoseidonGate, 1 for ConstantGate, PublicInputGate;\n fn degree(&self) -> usize;\n\n /// The generators used to populate the witness.\n /// Note: This should return exactly 1 generator per operation in the gate.\n fn generators(&self, row: usize, local_constants: &[F]) -> Vec>;\n\n /// Enables gates to store some \"routed constants\", if they have both unused constants and\n /// unused routed wires.\n ///\n /// Each entry in the returned `Vec` has the form `(constant_index, wire_index)`. `wire_index`\n /// must correspond to a *routed* wire.\n /// for ConstantGate, the size of vector is equal to num_of_consts, it's empty for ArithmeticGate, PublicInputGate, and PoseidonGate\n fn extra_constant_wires(&self) -> Vec<(usize, usize)> {\n vec![]\n }\n}\n```\n### ArithmeticGate\n```rust\n/// A gate which can perform a weighted multiply-add, i.e. `result = c0 x y + c1 z`. If the config\n/// supports enough routed wires, it can support several such operations in one gate.\npub struct ArithmeticGate {\n /// Number of arithmetic operations performed by an arithmetic gate.\n pub num_ops: usize,\n}\n\n/// A gate which takes a single constant parameter and outputs that value.\npub struct ConstantGate {\n pub(crate) num_consts: usize,\n}\n/// for constant gate, the evaluation is just input - wire output, the result should be 0\nfn eval_unfiltered(&self, vars: EvaluationVars) -> Vec {\n (0..self.num_consts)\n .map(|i| {\n vars.local_constants[self.const_input(i)] - vars.local_wires[self.wire_output(i)]\n })\n .collect()\n}\n\n/// A gate whose first four wires will be equal to a hash of public inputs. the value 4 is hadcoded as the hash\n/// out inputs is 4 Targets. PublicInputGate is aliased as pi gate\npub struct PublicInputGate;\nimpl PublicInputGate {\n pub fn wires_public_inputs_hash() -> Range {\n 0..4\n }\n}\n```\n\n## 3. Circuit Builder\n```rust\npub struct CircuitBuilder, const D: usize> {\n pub config: CircuitConfig,\n\n /// A domain separator, which is included in the initial Fiat-Shamir seed. This is generally not\n /// needed, but can be used to ensure that proofs for one application are not valid for another.\n /// Defaults to the empty vector.\n domain_separator: Option>,\n\n /// The types of gates used in this circuit.\n gates: HashSet>,\n\n /// The concrete placement of each gate.\n pub(crate) gate_instances: Vec>,\n\n /// Targets to be made public.\n public_inputs: Vec,\n\n /// The next available index for a `VirtualTarget`.\n virtual_target_index: usize,\n\n copy_constraints: Vec,\n\n /// A tree of named scopes, used for debugging.\n context_log: ContextTree,\n\n /// Generators used to generate the witness.\n generators: Vec>,\n\n /// maps the constant value -> ConstantTarget(a virtual target)\n constants_to_targets: HashMap,\n /// an inverse map of the constants_to_targets\n targets_to_constants: HashMap,\n\n /// Memoized results of `arithmetic` calls.\n pub(crate) base_arithmetic_results: HashMap, Target>,\n\n /// Memoized results of `arithmetic_extension` calls.\n pub(crate) arithmetic_results: HashMap, ExtensionTarget>,\n\n /// Map between gate type and the current gate of this type with available slots.\n current_slots: HashMap, CurrentSlot>,\n\n /// List of constant generators used to fill the constant wires.\n constant_generators: Vec>,\n\n /// Rows for each LUT: LookupWire contains: first `LookupGate`, first `LookupTableGate`, last `LookupTableGate`.\n lookup_rows: Vec,\n\n /// For each LUT index, vector of `(looking_in, looking_out)` pairs.\n lut_to_lookups: Vec,\n\n // Lookup tables in the form of `Vec<(input_value, output_value)>`.\n luts: Vec,\n\n /// Optional common data. When it is `Some(goal_data)`, the `build` function panics if the resulting\n /// common data doesn't equal `goal_data`.\n /// This is used in cyclic recursion.\n pub(crate) goal_common_data: Option>,\n\n /// Optional verifier data that is registered as public inputs.\n /// This is used in cyclic recursion to hold the circuit's own verifier key.\n pub(crate) verifier_data_public_input: Option,\n}\n```\n- **virtual_target**: This is not an actual wire in the witness, but just a target that help facilitate witness generation. In particular, a generator can assign a values to a virtual target, which can then be copied to other (virtual or concrete) targets. When we generate the final witness (a grid of wire values), these virtual targets will go away.\n\n### 3.1 methods\n```rust\n/// Adds a gate to the circuit, and returns its index.\npub fn add_gate>(&mut self, gate_type: G, mut constants: Vec) -> usize {}\n\n/// Returns a routable target with the given constant value.\n/// for example, for ArithmeticGate, at least constant of `ONE` and `ZERO` is allocated\n/// those constant target are all virtual target\npub fn constant(&mut self, c: F) -> Target {\n if let Some(&target) = self.constants_to_targets.get(&c) {\n // We already have a wire for this constant.\n return target;\n }\n\n let target = self.add_virtual_target();\n self.constants_to_targets.insert(c, target);\n self.targets_to_constants.insert(target, c);\n\n target\n}\n\n/// get polynomial to constants for each gate\n/// the size of the vector is the number of constants (a value for the whole circuit, gate selectors + constants for gate inputs)\n/// each polynomial interpolates the actual constant value at each gate, \nfn constant_polys(&self) -> Vec> {}\n\n/// if zero_knwoledge is set, it will blind the circuit\n/// if the number of circuits is not a power of 2, pad NoopGate, and make the total number of gates to be power of 2\nfn blind_and_pad(&mut self) {}\n/**\n * hash the circuit inputs into m Target, m is a constant value of 4. During this process, a `PoseidonGate` is added as the hashing of inputs is conducted\n * @param inputs: public inputs and private inputs\n */\npub fn hash_n_to_hash_no_pad>(\n &mut self,\n inputs: Vec,\n) -> HashOutTarget {\n HashOutTarget::from_vec(self.hash_n_to_m_no_pad::(inputs, NUM_HASH_OUT_ELTS))\n}\n\n\n/**\n * @param k_is, the coset shift, g^{j}, j \\in [0,num_shifts - 1], num_shifts is equal to num_routed_wires (columns); e.q 100\n * @subgroup: the \\Omega domain. \\Omega^{i}, i \\in [0, N-1], N is number of gates (rows); e.q 8 (2^3)\n * @return(0): vector of the sigma polynomial (sigma polynomial is the copy constraint permutation); the size is equal to the number of cols\n * @return(1): the forest is a data structure representing all wires copy constraint. the collected wires forms a unique partition\n */\nfn sigma_vecs(&self, k_is: &[F], subgroup: &[F]) -> (Vec>, Forest) {}\n```\n\n### 3.2 functions\n```rust\n/// Finds a set of shifts that result in unique cosets for the multiplicative subgroup of size `2^subgroup_bits`.\n/// Let `g` be a generator of the entire multiplicative group. the result is just g^0, ..., g^(num_shifts - 1)\npub fn get_unique_coset_shifts(subgroup_size: usize, num_shifts: usize) -> Vec {}\n```\n\n## 4. selectors\n```rust\n/// Selector polynomials are computed as follows:\n/// Partition the gates into (the smallest amount of) groups `{ G_i }`, such that for each group `G`\n/// `|G| + max_{g in G} g.degree() <= max_degree`. These groups are constructed greedily from\n/// the list of gates sorted by degree.\n/// We build a selector polynomial `S_i` for each group `G_i`, with\n/// S_i\\[j\\] =\n/// if j-th row gate=g_k in G_i\n/// k (k \\in [0, NUM_OF_GATE_TYPES)\n/// else\n/// UNUSED_SELECTOR\n/// @param gates: the types of gates , gate type is differentiated by gate.id()\n/// @param instances: all instances of gates ( a same type of gate could be used multiple times)\n/// @param max_degree: max_degree of each group\n/// @return(0): a vector of selector polynomials, the size of the vector is number of groups, in each group, the polynomial indicate which gate type it is\n/// @return(1) SelectorsInfo {\n/// selector_indices, a vector of gate group information. the size of vector is same to the number of gate types. the value represent which group the gate type belongs to\n/// groups: a vector of groups information, the size is the number of groups. in each group, it is a range of the gate types (ordered as a method described above)\n/// }\npub(crate) fn selector_polynomials, const D: usize>(\n gates: &[GateRef],\n instances: &[GateInstance],\n max_degree: usize,\n) -> (Vec>, SelectorsInfo) {}\n```\n\n## 5. permutation\n```rust\npub struct WirePartition {\n // each element of the vector is a vector of copy constraint\n partition: Vec>,\n}\n\n/// Disjoint Set Forest data-structure following .\npub struct Forest {\n /// A map of parent pointers, stored as indices. the parent value is the partition it belongs to\n /// those node with same parent value, means they are in the copy constraint \n pub(crate) parents: Vec,\n\n num_wires: usize,\n num_routed_wires: usize, // num_routed_wires is used to construct all wires\n degree: usize,\n}\n```\n\n## 6. hash\ndefined in `CircuitBuilder`\n```rust\n/// Hash function used for building Merkle trees.\ntype Hasher: Hasher;\n/// Algebraic hash function used for the challenger and hashing public inputs.\ntype InnerHasher: AlgebraicHasher;\n\n/// Trait for algebraic hash functions, built from a permutation using the sponge construction.\npub trait AlgebraicHasher: Hasher> {\n type AlgebraicPermutation: PlonkyPermutation;\n\n /// Circuit to conditionally swap two chunks of the inputs (useful in verifying Merkle proofs),\n /// then apply the permutation.\n fn permute_swapped(\n inputs: Self::AlgebraicPermutation,\n swap: BoolTarget,\n builder: &mut CircuitBuilder,\n ) -> Self::AlgebraicPermutation\n where\n F: RichField + Extendable;\n}\n```\n\n\n## 7. IOP\n### 7.1 generator\n```rust\n/// Generator used to fill an extra constant.\n#[derive(Debug, Clone, Default)]\npub struct ConstantGenerator {\n pub row: usize, // specifying the gate row number\n pub constant_index: usize, // index of constant input. for example, for a constant gate, num_consts is specified; num_consts will specify the total number of constants\n pub wire_index: usize, // index of constant wire output\n pub constant: F, // the constant used to fill a target\n}\n\n\n/// A generator which runs once after a list of dependencies is present in the witness.\npub trait SimpleGenerator, const D: usize>:\n 'static + Send + Sync + Debug\n{\n fn id(&self) -> String;\n\n fn dependencies(&self) -> Vec;\n\n fn run_once(&self, witness: &PartitionWitness, out_buffer: &mut GeneratedValues);\n\n fn adapter(self) -> SimpleGeneratorAdapter\n where\n Self: Sized,\n {\n SimpleGeneratorAdapter {\n inner: self,\n _phantom: PhantomData,\n }\n }\n\n fn serialize(&self, dst: &mut Vec, common_data: &CommonCircuitData) -> IoResult<()>;\n\n fn deserialize(src: &mut Buffer, common_data: &CommonCircuitData) -> IoResult\n where\n Self: Sized;\n}\n```\n\n## 8. FRI\n\n\n\n## Questions\n","source":"_posts/cryptography/zkp/plonky2-code-analysis.md","raw":"---\ntitle: plonky2 code analysis\ndate: 2023-11-06 13:24:03\ntags: [cryptography,zkp]\n---\n\n\n\n\n## introduction\nthis post is a source code analysis of https://github.com/0xPolygonZero/plonky2\n\n## 1. Config\n### 1.1 CircuitConfig\nthe default config is `Self::standard_recursion_config()` with value as below\n```rust\nSelf {\n num_wires: 135,\n num_routed_wires: 80, \n num_constants: 2, // used in ConstantGate, which hold the consant value for the circuit\n /// Whether to use a dedicated gate for base field arithmetic, rather than using a single gate\n /// for both base field and extension field arithmetic.\n use_base_arithmetic_gate: true,\n security_bits: 100,\n /// The number of challenge points to generate, for IOPs that have soundness errors of (roughly)\n /// `degree / |F|`.\n num_challenges: 2,\n zero_knowledge: false,\n /// A cap on the quotient polynomial's degree factor. The actual degree factor is derived\n /// systematically, but will never exceed this value.\n max_quotient_degree_factor: 8,\n fri_config: FriConfig {\n rate_bits: 3, // used in lde\n cap_height: 4,\n proof_of_work_bits: 16,\n reduction_strategy: FriReductionStrategy::ConstantArityBits(4, 5),\n num_query_rounds: 28,\n },\n}\n```\n\n**notes**\n- *routable wire* It's a wire that can be connected to other gates' wires, rather than an \"advice\" wire which is for internal state. For example if we had a gate for computing x^8, we'd have two routable wires for the input and output, x and x^8. But if we wanted lower-degree constraints, we could add advice wires for the intermediate results x^2 and x^4\n\n- *num_routed_wires*, the default value is 80. For eample, for a arithmetic gate, it computes `const_0 * multiplicand_0 * multiplicand_1 + const_1 * addend`. i.e 2 wires for multiplicands and 1 wire for the addend and 1 for output. total is 4. therefore, the gate can have `num_routed_wires/4` operations, in this case, it is 20. the circuit builder will add gate when number of operations exceeds 20.\n\n### 1.2 FriConfig\n```rust\npub struct FriConfig {\n/// `rate = 2^{-rate_bits}`. default is 3\npub rate_bits: usize, \n\n/// Height of Merkle tree caps. default is 4\npub cap_height: usize,\n/// default is 16\npub proof_of_work_bits: u32,\n\npub reduction_strategy: FriReductionStrategy,\n\n/// Number of query rounds to perform. default is 28\npub num_query_rounds: usize,\n}\n```\n\n## 2. Gates\n```rust\n/// A custom gate.\npub trait Gate, const D: usize>: 'static + Send + Sync {\n fn id(&self) -> String;\n /// The maximum degree among this gate's constraint polynomials. it is 3 for ArithmeticGate (Left, Right, Output); 7 for PoseidonGate, 1 for ConstantGate, PublicInputGate;\n fn degree(&self) -> usize;\n\n /// The generators used to populate the witness.\n /// Note: This should return exactly 1 generator per operation in the gate.\n fn generators(&self, row: usize, local_constants: &[F]) -> Vec>;\n\n /// Enables gates to store some \"routed constants\", if they have both unused constants and\n /// unused routed wires.\n ///\n /// Each entry in the returned `Vec` has the form `(constant_index, wire_index)`. `wire_index`\n /// must correspond to a *routed* wire.\n /// for ConstantGate, the size of vector is equal to num_of_consts, it's empty for ArithmeticGate, PublicInputGate, and PoseidonGate\n fn extra_constant_wires(&self) -> Vec<(usize, usize)> {\n vec![]\n }\n}\n```\n### ArithmeticGate\n```rust\n/// A gate which can perform a weighted multiply-add, i.e. `result = c0 x y + c1 z`. If the config\n/// supports enough routed wires, it can support several such operations in one gate.\npub struct ArithmeticGate {\n /// Number of arithmetic operations performed by an arithmetic gate.\n pub num_ops: usize,\n}\n\n/// A gate which takes a single constant parameter and outputs that value.\npub struct ConstantGate {\n pub(crate) num_consts: usize,\n}\n/// for constant gate, the evaluation is just input - wire output, the result should be 0\nfn eval_unfiltered(&self, vars: EvaluationVars) -> Vec {\n (0..self.num_consts)\n .map(|i| {\n vars.local_constants[self.const_input(i)] - vars.local_wires[self.wire_output(i)]\n })\n .collect()\n}\n\n/// A gate whose first four wires will be equal to a hash of public inputs. the value 4 is hadcoded as the hash\n/// out inputs is 4 Targets. PublicInputGate is aliased as pi gate\npub struct PublicInputGate;\nimpl PublicInputGate {\n pub fn wires_public_inputs_hash() -> Range {\n 0..4\n }\n}\n```\n\n## 3. Circuit Builder\n```rust\npub struct CircuitBuilder, const D: usize> {\n pub config: CircuitConfig,\n\n /// A domain separator, which is included in the initial Fiat-Shamir seed. This is generally not\n /// needed, but can be used to ensure that proofs for one application are not valid for another.\n /// Defaults to the empty vector.\n domain_separator: Option>,\n\n /// The types of gates used in this circuit.\n gates: HashSet>,\n\n /// The concrete placement of each gate.\n pub(crate) gate_instances: Vec>,\n\n /// Targets to be made public.\n public_inputs: Vec,\n\n /// The next available index for a `VirtualTarget`.\n virtual_target_index: usize,\n\n copy_constraints: Vec,\n\n /// A tree of named scopes, used for debugging.\n context_log: ContextTree,\n\n /// Generators used to generate the witness.\n generators: Vec>,\n\n /// maps the constant value -> ConstantTarget(a virtual target)\n constants_to_targets: HashMap,\n /// an inverse map of the constants_to_targets\n targets_to_constants: HashMap,\n\n /// Memoized results of `arithmetic` calls.\n pub(crate) base_arithmetic_results: HashMap, Target>,\n\n /// Memoized results of `arithmetic_extension` calls.\n pub(crate) arithmetic_results: HashMap, ExtensionTarget>,\n\n /// Map between gate type and the current gate of this type with available slots.\n current_slots: HashMap, CurrentSlot>,\n\n /// List of constant generators used to fill the constant wires.\n constant_generators: Vec>,\n\n /// Rows for each LUT: LookupWire contains: first `LookupGate`, first `LookupTableGate`, last `LookupTableGate`.\n lookup_rows: Vec,\n\n /// For each LUT index, vector of `(looking_in, looking_out)` pairs.\n lut_to_lookups: Vec,\n\n // Lookup tables in the form of `Vec<(input_value, output_value)>`.\n luts: Vec,\n\n /// Optional common data. When it is `Some(goal_data)`, the `build` function panics if the resulting\n /// common data doesn't equal `goal_data`.\n /// This is used in cyclic recursion.\n pub(crate) goal_common_data: Option>,\n\n /// Optional verifier data that is registered as public inputs.\n /// This is used in cyclic recursion to hold the circuit's own verifier key.\n pub(crate) verifier_data_public_input: Option,\n}\n```\n- **virtual_target**: This is not an actual wire in the witness, but just a target that help facilitate witness generation. In particular, a generator can assign a values to a virtual target, which can then be copied to other (virtual or concrete) targets. When we generate the final witness (a grid of wire values), these virtual targets will go away.\n\n### 3.1 methods\n```rust\n/// Adds a gate to the circuit, and returns its index.\npub fn add_gate>(&mut self, gate_type: G, mut constants: Vec) -> usize {}\n\n/// Returns a routable target with the given constant value.\n/// for example, for ArithmeticGate, at least constant of `ONE` and `ZERO` is allocated\n/// those constant target are all virtual target\npub fn constant(&mut self, c: F) -> Target {\n if let Some(&target) = self.constants_to_targets.get(&c) {\n // We already have a wire for this constant.\n return target;\n }\n\n let target = self.add_virtual_target();\n self.constants_to_targets.insert(c, target);\n self.targets_to_constants.insert(target, c);\n\n target\n}\n\n/// get polynomial to constants for each gate\n/// the size of the vector is the number of constants (a value for the whole circuit, gate selectors + constants for gate inputs)\n/// each polynomial interpolates the actual constant value at each gate, \nfn constant_polys(&self) -> Vec> {}\n\n/// if zero_knwoledge is set, it will blind the circuit\n/// if the number of circuits is not a power of 2, pad NoopGate, and make the total number of gates to be power of 2\nfn blind_and_pad(&mut self) {}\n/**\n * hash the circuit inputs into m Target, m is a constant value of 4. During this process, a `PoseidonGate` is added as the hashing of inputs is conducted\n * @param inputs: public inputs and private inputs\n */\npub fn hash_n_to_hash_no_pad>(\n &mut self,\n inputs: Vec,\n) -> HashOutTarget {\n HashOutTarget::from_vec(self.hash_n_to_m_no_pad::(inputs, NUM_HASH_OUT_ELTS))\n}\n\n\n/**\n * @param k_is, the coset shift, g^{j}, j \\in [0,num_shifts - 1], num_shifts is equal to num_routed_wires (columns); e.q 100\n * @subgroup: the \\Omega domain. \\Omega^{i}, i \\in [0, N-1], N is number of gates (rows); e.q 8 (2^3)\n * @return(0): vector of the sigma polynomial (sigma polynomial is the copy constraint permutation); the size is equal to the number of cols\n * @return(1): the forest is a data structure representing all wires copy constraint. the collected wires forms a unique partition\n */\nfn sigma_vecs(&self, k_is: &[F], subgroup: &[F]) -> (Vec>, Forest) {}\n```\n\n### 3.2 functions\n```rust\n/// Finds a set of shifts that result in unique cosets for the multiplicative subgroup of size `2^subgroup_bits`.\n/// Let `g` be a generator of the entire multiplicative group. the result is just g^0, ..., g^(num_shifts - 1)\npub fn get_unique_coset_shifts(subgroup_size: usize, num_shifts: usize) -> Vec {}\n```\n\n## 4. selectors\n```rust\n/// Selector polynomials are computed as follows:\n/// Partition the gates into (the smallest amount of) groups `{ G_i }`, such that for each group `G`\n/// `|G| + max_{g in G} g.degree() <= max_degree`. These groups are constructed greedily from\n/// the list of gates sorted by degree.\n/// We build a selector polynomial `S_i` for each group `G_i`, with\n/// S_i\\[j\\] =\n/// if j-th row gate=g_k in G_i\n/// k (k \\in [0, NUM_OF_GATE_TYPES)\n/// else\n/// UNUSED_SELECTOR\n/// @param gates: the types of gates , gate type is differentiated by gate.id()\n/// @param instances: all instances of gates ( a same type of gate could be used multiple times)\n/// @param max_degree: max_degree of each group\n/// @return(0): a vector of selector polynomials, the size of the vector is number of groups, in each group, the polynomial indicate which gate type it is\n/// @return(1) SelectorsInfo {\n/// selector_indices, a vector of gate group information. the size of vector is same to the number of gate types. the value represent which group the gate type belongs to\n/// groups: a vector of groups information, the size is the number of groups. in each group, it is a range of the gate types (ordered as a method described above)\n/// }\npub(crate) fn selector_polynomials, const D: usize>(\n gates: &[GateRef],\n instances: &[GateInstance],\n max_degree: usize,\n) -> (Vec>, SelectorsInfo) {}\n```\n\n## 5. permutation\n```rust\npub struct WirePartition {\n // each element of the vector is a vector of copy constraint\n partition: Vec>,\n}\n\n/// Disjoint Set Forest data-structure following .\npub struct Forest {\n /// A map of parent pointers, stored as indices. the parent value is the partition it belongs to\n /// those node with same parent value, means they are in the copy constraint \n pub(crate) parents: Vec,\n\n num_wires: usize,\n num_routed_wires: usize, // num_routed_wires is used to construct all wires\n degree: usize,\n}\n```\n\n## 6. hash\ndefined in `CircuitBuilder`\n```rust\n/// Hash function used for building Merkle trees.\ntype Hasher: Hasher;\n/// Algebraic hash function used for the challenger and hashing public inputs.\ntype InnerHasher: AlgebraicHasher;\n\n/// Trait for algebraic hash functions, built from a permutation using the sponge construction.\npub trait AlgebraicHasher: Hasher> {\n type AlgebraicPermutation: PlonkyPermutation;\n\n /// Circuit to conditionally swap two chunks of the inputs (useful in verifying Merkle proofs),\n /// then apply the permutation.\n fn permute_swapped(\n inputs: Self::AlgebraicPermutation,\n swap: BoolTarget,\n builder: &mut CircuitBuilder,\n ) -> Self::AlgebraicPermutation\n where\n F: RichField + Extendable;\n}\n```\n\n\n## 7. IOP\n### 7.1 generator\n```rust\n/// Generator used to fill an extra constant.\n#[derive(Debug, Clone, Default)]\npub struct ConstantGenerator {\n pub row: usize, // specifying the gate row number\n pub constant_index: usize, // index of constant input. for example, for a constant gate, num_consts is specified; num_consts will specify the total number of constants\n pub wire_index: usize, // index of constant wire output\n pub constant: F, // the constant used to fill a target\n}\n\n\n/// A generator which runs once after a list of dependencies is present in the witness.\npub trait SimpleGenerator, const D: usize>:\n 'static + Send + Sync + Debug\n{\n fn id(&self) -> String;\n\n fn dependencies(&self) -> Vec;\n\n fn run_once(&self, witness: &PartitionWitness, out_buffer: &mut GeneratedValues);\n\n fn adapter(self) -> SimpleGeneratorAdapter\n where\n Self: Sized,\n {\n SimpleGeneratorAdapter {\n inner: self,\n _phantom: PhantomData,\n }\n }\n\n fn serialize(&self, dst: &mut Vec, common_data: &CommonCircuitData) -> IoResult<()>;\n\n fn deserialize(src: &mut Buffer, common_data: &CommonCircuitData) -> IoResult\n where\n Self: Sized;\n}\n```\n\n## 8. FRI\n\n\n\n## Questions\n","slug":"cryptography/zkp/plonky2-code-analysis","published":1,"updated":"2023-12-12T11:45:37.866Z","_id":"clq0nd00w00004z7u3a6ubwbd","comments":1,"layout":"post","photos":[],"link":"","content":"\n\n\n

introduction

this post is a source code analysis of https://github.com/0xPolygonZero/plonky2

\n

1. Config

1.1 CircuitConfig

the default config is Self::standard_recursion_config() with value as below

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
Self {
  num_wires: 135,
  num_routed_wires: 80, 
  num_constants: 2,  // used in ConstantGate, which hold the consant value for the circuit
  /// Whether to use a dedicated gate for base field arithmetic, rather than using a single gate
  /// for both base field and extension field arithmetic.
  use_base_arithmetic_gate: true,
  security_bits: 100,
  /// The number of challenge points to generate, for IOPs that have soundness errors of (roughly)
  /// `degree / |F|`.
  num_challenges: 2,
  zero_knowledge: false,
  /// A cap on the quotient polynomial's degree factor. The actual degree factor is derived
  /// systematically, but will never exceed this value.
  max_quotient_degree_factor: 8,
  fri_config: FriConfig {
      rate_bits: 3,  // used in lde
      cap_height: 4,
      proof_of_work_bits: 16,
      reduction_strategy: FriReductionStrategy::ConstantArityBits(4, 5),
      num_query_rounds: 28,
  },
}
\n\n

notes

\n
    \n

  • routable wire It’s a wire that can be connected to other gates’ wires, rather than an “advice” wire which is for internal state. For example if we had a gate for computing x^8, we’d have two routable wires for the input and output, x and x^8. But if we wanted lower-degree constraints, we could add advice wires for the intermediate results x^2 and x^4

    \n
    • \n

  • num_routed_wires, the default value is 80. For eample, for a arithmetic gate, it computes const_0 * multiplicand_0 * multiplicand_1 + const_1 * addend. i.e 2 wires for multiplicands and 1 wire for the addend and 1 for output. total is 4. therefore, the gate can have num_routed_wires/4 operations, in this case, it is 20. the circuit builder will add gate when number of operations exceeds 20.

    \n
    • \n
\n

1.2 FriConfig

1
2
3
4
5
6
7
8
9
10
11
12
13
14
pub struct FriConfig {
/// `rate = 2^{-rate_bits}`. default is 3
pub rate_bits: usize,  

/// Height of Merkle tree caps. default is 4
pub cap_height: usize,
/// default is 16
pub proof_of_work_bits: u32,

pub reduction_strategy: FriReductionStrategy,

/// Number of query rounds to perform. default is 28
pub num_query_rounds: usize,
}
\n\n

2. Gates

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
/// A custom gate.
pub trait Gate<F: RichField + Extendable<D>, const D: usize>: 'static + Send + Sync {
    fn id(&self) -> String;
    /// The maximum degree among this gate's constraint polynomials. it is 3 for ArithmeticGate (Left, Right, Output); 7 for PoseidonGate, 1 for ConstantGate, PublicInputGate;
    fn degree(&self) -> usize;

    /// The generators used to populate the witness.
    /// Note: This should return exactly 1 generator per operation in the gate.
    fn generators(&self, row: usize, local_constants: &[F]) -> Vec<WitnessGeneratorRef<F, D>>;

    /// Enables gates to store some "routed constants", if they have both unused constants and
    /// unused routed wires.
    ///
    /// Each entry in the returned `Vec` has the form `(constant_index, wire_index)`. `wire_index`
    /// must correspond to a *routed* wire.
    /// for ConstantGate, the size of vector is equal to num_of_consts, it's empty for ArithmeticGate, PublicInputGate, and PoseidonGate
    fn extra_constant_wires(&self) -> Vec<(usize, usize)> {
        vec![]
    }
}
\n

ArithmeticGate

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
/// A gate which can perform a weighted multiply-add, i.e. `result = c0 x y + c1 z`. If the config
/// supports enough routed wires, it can support several such operations in one gate.
pub struct ArithmeticGate {
    /// Number of arithmetic operations performed by an arithmetic gate.
    pub num_ops: usize,
}

/// A gate which takes a single constant parameter and outputs that value.
pub struct ConstantGate {
    pub(crate) num_consts: usize,
}
/// for constant gate, the evaluation is just input - wire output, the result should be 0
fn eval_unfiltered(&self, vars: EvaluationVars<F, D>) -> Vec<F::Extension> {
    (0..self.num_consts)
        .map(|i| {
            vars.local_constants[self.const_input(i)] - vars.local_wires[self.wire_output(i)]
        })
        .collect()
}

/// A gate whose first four wires will be equal to a hash of public inputs. the value 4 is hadcoded as the hash
/// out inputs is 4 Targets. PublicInputGate is aliased as pi gate
pub struct PublicInputGate;
impl PublicInputGate {
    pub fn wires_public_inputs_hash() -> Range<usize> {
        0..4
    }
}
\n\n

3. Circuit Builder

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
pub struct CircuitBuilder<F: RichField + Extendable<D>, const D: usize> {
    pub config: CircuitConfig,

    /// A domain separator, which is included in the initial Fiat-Shamir seed. This is generally not
    /// needed, but can be used to ensure that proofs for one application are not valid for another.
    /// Defaults to the empty vector.
    domain_separator: Option<Vec<F>>,

    /// The types of gates used in this circuit.
    gates: HashSet<GateRef<F, D>>,

    /// The concrete placement of each gate.
    pub(crate) gate_instances: Vec<GateInstance<F, D>>,

    /// Targets to be made public.
    public_inputs: Vec<Target>,

    /// The next available index for a `VirtualTarget`.
    virtual_target_index: usize,

    copy_constraints: Vec<CopyConstraint>,

    /// A tree of named scopes, used for debugging.
    context_log: ContextTree,

    /// Generators used to generate the witness.
    generators: Vec<WitnessGeneratorRef<F, D>>,

    /// maps the constant value -> ConstantTarget(a virtual target)
    constants_to_targets: HashMap<F, Target>,
    /// an inverse map of the constants_to_targets
    targets_to_constants: HashMap<Target, F>,

    /// Memoized results of `arithmetic` calls.
    pub(crate) base_arithmetic_results: HashMap<BaseArithmeticOperation<F>, Target>,

    /// Memoized results of `arithmetic_extension` calls.
    pub(crate) arithmetic_results: HashMap<ExtensionArithmeticOperation<F, D>, ExtensionTarget<D>>,

    /// Map between gate type and the current gate of this type with available slots.
    current_slots: HashMap<GateRef<F, D>, CurrentSlot<F, D>>,

    /// List of constant generators used to fill the constant wires.
    constant_generators: Vec<ConstantGenerator<F>>,

    /// Rows for each LUT: LookupWire contains: first `LookupGate`, first `LookupTableGate`, last `LookupTableGate`.
    lookup_rows: Vec<LookupWire>,

    /// For each LUT index, vector of `(looking_in, looking_out)` pairs.
    lut_to_lookups: Vec<Lookup>,

    // Lookup tables in the form of `Vec<(input_value, output_value)>`.
    luts: Vec<LookupTable>,

    /// Optional common data. When it is `Some(goal_data)`, the `build` function panics if the resulting
    /// common data doesn't equal `goal_data`.
    /// This is used in cyclic recursion.
    pub(crate) goal_common_data: Option<CommonCircuitData<F, D>>,

    /// Optional verifier data that is registered as public inputs.
    /// This is used in cyclic recursion to hold the circuit's own verifier key.
    pub(crate) verifier_data_public_input: Option<VerifierCircuitTarget>,
}
\n
    \n
  • virtual_target: This is not an actual wire in the witness, but just a target that help facilitate witness generation. In particular, a generator can assign a values to a virtual target, which can then be copied to other (virtual or concrete) targets. When we generate the final witness (a grid of wire values), these virtual targets will go away.
    • \n
\n

3.1 methods

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
/// Adds a gate to the circuit, and returns its index.
pub fn add_gate<G: Gate<F, D>>(&mut self, gate_type: G, mut constants: Vec<F>) -> usize {}

/// Returns a routable target with the given constant value.
/// for example, for ArithmeticGate, at least constant of `ONE` and `ZERO` is allocated
/// those constant target are all virtual target
pub fn constant(&mut self, c: F) -> Target {
    if let Some(&target) = self.constants_to_targets.get(&c) {
        // We already have a wire for this constant.
        return target;
    }

    let target = self.add_virtual_target();
    self.constants_to_targets.insert(c, target);
    self.targets_to_constants.insert(target, c);

    target
}

/// get polynomial to constants for each gate
/// the size of the vector is the number of constants (a value for the whole circuit, gate selectors + constants for gate inputs)
/// each polynomial interpolates the actual constant value at each gate, 
fn constant_polys(&self) -> Vec<PolynomialValues<F>> {}

/// if zero_knwoledge is set, it will blind the circuit
/// if the number of circuits is not a power of 2, pad NoopGate, and make the total number of gates to be power of 2
fn blind_and_pad(&mut self) {}
/**
  * hash the circuit inputs into m Target, m is a constant value of 4. During this process, a `PoseidonGate` is added as the hashing of inputs is conducted
  * @param inputs: public inputs and private inputs
  */
pub fn hash_n_to_hash_no_pad<H: AlgebraicHasher<F>>(
    &mut self,
    inputs: Vec<Target>,
) -> HashOutTarget {
    HashOutTarget::from_vec(self.hash_n_to_m_no_pad::<H>(inputs, NUM_HASH_OUT_ELTS))
}


/**
  * @param k_is, the coset shift, g^{j}, j \\in [0,num_shifts - 1], num_shifts is equal to num_routed_wires (columns); e.q 100
  * @subgroup: the \\Omega domain. \\Omega^{i}, i \\in [0, N-1], N is number of gates (rows); e.q 8 (2^3)
  * @return(0): vector of the sigma polynomial (sigma polynomial is the copy constraint permutation); the size is equal to the number of cols
  * @return(1): the forest is a data structure representing all wires copy constraint. the collected wires forms a unique partition
  */
fn sigma_vecs(&self, k_is: &[F], subgroup: &[F]) -> (Vec<PolynomialValues<F>>, Forest) {}
\n\n

3.2 functions

1
2
3
/// Finds a set of shifts that result in unique cosets for the multiplicative subgroup of size `2^subgroup_bits`.
/// Let `g` be a generator of the entire multiplicative group. the result is just g^0, ..., g^(num_shifts - 1)
pub fn get_unique_coset_shifts<F: Field>(subgroup_size: usize, num_shifts: usize) -> Vec<F> {}
\n\n

4. selectors

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
/// Selector polynomials are computed as follows:
/// Partition the gates into (the smallest amount of) groups `{ G_i }`, such that for each group `G`
/// `|G| + max_{g in G} g.degree() <= max_degree`. These groups are constructed greedily from
/// the list of gates sorted by degree.
/// We build a selector polynomial `S_i` for each group `G_i`, with
/// S_i\\[j\\] =
///     if j-th row gate=g_k in G_i
///         k  (k \\in [0, NUM_OF_GATE_TYPES)
///     else
///         UNUSED_SELECTOR
/// @param gates: the types of gates , gate type is differentiated by gate.id()
/// @param instances: all instances of gates ( a same type of gate could be used multiple times)
/// @param max_degree: max_degree of each group
/// @return(0): a vector of selector polynomials, the size of the vector is number of groups, in each group, the polynomial indicate which gate type it is
/// @return(1) SelectorsInfo {
///     selector_indices, a vector of gate group information. the size of vector is same to the number of gate types. the value represent which group the gate type belongs to
///     groups: a vector of groups information, the size is the number of groups. in each group, it is a range of the gate types (ordered as a method described above)
/// }
pub(crate) fn selector_polynomials<F: RichField + Extendable<D>, const D: usize>(
    gates: &[GateRef<F, D>],
    instances: &[GateInstance<F, D>],
    max_degree: usize,
) -> (Vec<PolynomialValues<F>>, SelectorsInfo) {}
\n\n

5. permutation

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
pub struct WirePartition {
    // each element of the vector is a vector of copy constraint
    partition: Vec<Vec<Wire>>,
}

/// Disjoint Set Forest data-structure following <https://en.wikipedia.org/wiki/Disjoint-set_data_structure>.
pub struct Forest {
    /// A map of parent pointers, stored as indices. the parent value is the partition it belongs to
    /// those node with same parent value, means they are in the copy constraint 
    pub(crate) parents: Vec<usize>,

    num_wires: usize,
    num_routed_wires: usize, // num_routed_wires is used to construct all wires
    degree: usize,
}
\n\n

6. hash

defined in CircuitBuilder

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
/// Hash function used for building Merkle trees.
type Hasher: Hasher<Self::F>;
/// Algebraic hash function used for the challenger and hashing public inputs.
type InnerHasher: AlgebraicHasher<Self::F>;

/// Trait for algebraic hash functions, built from a permutation using the sponge construction.
pub trait AlgebraicHasher<F: RichField>: Hasher<F, Hash = HashOut<F>> {
    type AlgebraicPermutation: PlonkyPermutation<Target>;

    /// Circuit to conditionally swap two chunks of the inputs (useful in verifying Merkle proofs),
    /// then apply the permutation.
    fn permute_swapped<const D: usize>(
        inputs: Self::AlgebraicPermutation,
        swap: BoolTarget,
        builder: &mut CircuitBuilder<F, D>,
    ) -> Self::AlgebraicPermutation
    where
        F: RichField + Extendable<D>;
}
\n\n\n

7. IOP

7.1 generator

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
/// Generator used to fill an extra constant.
#[derive(Debug, Clone, Default)]
pub struct ConstantGenerator<F: Field> {
    pub row: usize,  //  specifying the gate row number
    pub constant_index: usize, // index of constant input. for example, for a constant gate, num_consts is specified; num_consts will specify the total number of constants
    pub wire_index: usize,     // index of constant wire output
    pub constant: F, // the constant used to fill a target
}


/// A generator which runs once after a list of dependencies is present in the witness.
pub trait SimpleGenerator<F: RichField + Extendable<D>, const D: usize>:
    'static + Send + Sync + Debug
{
    fn id(&self) -> String;

    fn dependencies(&self) -> Vec<Target>;

    fn run_once(&self, witness: &PartitionWitness<F>, out_buffer: &mut GeneratedValues<F>);

    fn adapter(self) -> SimpleGeneratorAdapter<F, Self, D>
    where
        Self: Sized,
    {
        SimpleGeneratorAdapter {
            inner: self,
            _phantom: PhantomData,
        }
    }

    fn serialize(&self, dst: &mut Vec<u8>, common_data: &CommonCircuitData<F, D>) -> IoResult<()>;

    fn deserialize(src: &mut Buffer, common_data: &CommonCircuitData<F, D>) -> IoResult<Self>
    where
        Self: Sized;
}
\n\n

8. FRI

Questions

","site":{"data":{}},"excerpt":"","more":"\n\n\n

introduction

this post is a source code analysis of https://github.com/0xPolygonZero/plonky2

\n

1. Config

1.1 CircuitConfig

the default config is Self::standard_recursion_config() with value as below

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
Self {
  num_wires: 135,
  num_routed_wires: 80, 
  num_constants: 2,  // used in ConstantGate, which hold the consant value for the circuit
  /// Whether to use a dedicated gate for base field arithmetic, rather than using a single gate
  /// for both base field and extension field arithmetic.
  use_base_arithmetic_gate: true,
  security_bits: 100,
  /// The number of challenge points to generate, for IOPs that have soundness errors of (roughly)
  /// `degree / |F|`.
  num_challenges: 2,
  zero_knowledge: false,
  /// A cap on the quotient polynomial's degree factor. The actual degree factor is derived
  /// systematically, but will never exceed this value.
  max_quotient_degree_factor: 8,
  fri_config: FriConfig {
      rate_bits: 3,  // used in lde
      cap_height: 4,
      proof_of_work_bits: 16,
      reduction_strategy: FriReductionStrategy::ConstantArityBits(4, 5),
      num_query_rounds: 28,
  },
}
\n\n

notes

\n
    \n

  • routable wire It’s a wire that can be connected to other gates’ wires, rather than an “advice” wire which is for internal state. For example if we had a gate for computing x^8, we’d have two routable wires for the input and output, x and x^8. But if we wanted lower-degree constraints, we could add advice wires for the intermediate results x^2 and x^4

    \n
    • \n

  • num_routed_wires, the default value is 80. For eample, for a arithmetic gate, it computes const_0 * multiplicand_0 * multiplicand_1 + const_1 * addend. i.e 2 wires for multiplicands and 1 wire for the addend and 1 for output. total is 4. therefore, the gate can have num_routed_wires/4 operations, in this case, it is 20. the circuit builder will add gate when number of operations exceeds 20.

    \n
    • \n
\n

1.2 FriConfig

1
2
3
4
5
6
7
8
9
10
11
12
13
14
pub struct FriConfig {
/// `rate = 2^{-rate_bits}`. default is 3
pub rate_bits: usize,  

/// Height of Merkle tree caps. default is 4
pub cap_height: usize,
/// default is 16
pub proof_of_work_bits: u32,

pub reduction_strategy: FriReductionStrategy,

/// Number of query rounds to perform. default is 28
pub num_query_rounds: usize,
}
\n\n

2. Gates

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
/// A custom gate.
pub trait Gate<F: RichField + Extendable<D>, const D: usize>: 'static + Send + Sync {
    fn id(&self) -> String;
    /// The maximum degree among this gate's constraint polynomials. it is 3 for ArithmeticGate (Left, Right, Output); 7 for PoseidonGate, 1 for ConstantGate, PublicInputGate;
    fn degree(&self) -> usize;

    /// The generators used to populate the witness.
    /// Note: This should return exactly 1 generator per operation in the gate.
    fn generators(&self, row: usize, local_constants: &[F]) -> Vec<WitnessGeneratorRef<F, D>>;

    /// Enables gates to store some "routed constants", if they have both unused constants and
    /// unused routed wires.
    ///
    /// Each entry in the returned `Vec` has the form `(constant_index, wire_index)`. `wire_index`
    /// must correspond to a *routed* wire.
    /// for ConstantGate, the size of vector is equal to num_of_consts, it's empty for ArithmeticGate, PublicInputGate, and PoseidonGate
    fn extra_constant_wires(&self) -> Vec<(usize, usize)> {
        vec![]
    }
}
\n

ArithmeticGate

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
/// A gate which can perform a weighted multiply-add, i.e. `result = c0 x y + c1 z`. If the config
/// supports enough routed wires, it can support several such operations in one gate.
pub struct ArithmeticGate {
    /// Number of arithmetic operations performed by an arithmetic gate.
    pub num_ops: usize,
}

/// A gate which takes a single constant parameter and outputs that value.
pub struct ConstantGate {
    pub(crate) num_consts: usize,
}
/// for constant gate, the evaluation is just input - wire output, the result should be 0
fn eval_unfiltered(&self, vars: EvaluationVars<F, D>) -> Vec<F::Extension> {
    (0..self.num_consts)
        .map(|i| {
            vars.local_constants[self.const_input(i)] - vars.local_wires[self.wire_output(i)]
        })
        .collect()
}

/// A gate whose first four wires will be equal to a hash of public inputs. the value 4 is hadcoded as the hash
/// out inputs is 4 Targets. PublicInputGate is aliased as pi gate
pub struct PublicInputGate;
impl PublicInputGate {
    pub fn wires_public_inputs_hash() -> Range<usize> {
        0..4
    }
}
\n\n

3. Circuit Builder

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
pub struct CircuitBuilder<F: RichField + Extendable<D>, const D: usize> {
    pub config: CircuitConfig,

    /// A domain separator, which is included in the initial Fiat-Shamir seed. This is generally not
    /// needed, but can be used to ensure that proofs for one application are not valid for another.
    /// Defaults to the empty vector.
    domain_separator: Option<Vec<F>>,

    /// The types of gates used in this circuit.
    gates: HashSet<GateRef<F, D>>,

    /// The concrete placement of each gate.
    pub(crate) gate_instances: Vec<GateInstance<F, D>>,

    /// Targets to be made public.
    public_inputs: Vec<Target>,

    /// The next available index for a `VirtualTarget`.
    virtual_target_index: usize,

    copy_constraints: Vec<CopyConstraint>,

    /// A tree of named scopes, used for debugging.
    context_log: ContextTree,

    /// Generators used to generate the witness.
    generators: Vec<WitnessGeneratorRef<F, D>>,

    /// maps the constant value -> ConstantTarget(a virtual target)
    constants_to_targets: HashMap<F, Target>,
    /// an inverse map of the constants_to_targets
    targets_to_constants: HashMap<Target, F>,

    /// Memoized results of `arithmetic` calls.
    pub(crate) base_arithmetic_results: HashMap<BaseArithmeticOperation<F>, Target>,

    /// Memoized results of `arithmetic_extension` calls.
    pub(crate) arithmetic_results: HashMap<ExtensionArithmeticOperation<F, D>, ExtensionTarget<D>>,

    /// Map between gate type and the current gate of this type with available slots.
    current_slots: HashMap<GateRef<F, D>, CurrentSlot<F, D>>,

    /// List of constant generators used to fill the constant wires.
    constant_generators: Vec<ConstantGenerator<F>>,

    /// Rows for each LUT: LookupWire contains: first `LookupGate`, first `LookupTableGate`, last `LookupTableGate`.
    lookup_rows: Vec<LookupWire>,

    /// For each LUT index, vector of `(looking_in, looking_out)` pairs.
    lut_to_lookups: Vec<Lookup>,

    // Lookup tables in the form of `Vec<(input_value, output_value)>`.
    luts: Vec<LookupTable>,

    /// Optional common data. When it is `Some(goal_data)`, the `build` function panics if the resulting
    /// common data doesn't equal `goal_data`.
    /// This is used in cyclic recursion.
    pub(crate) goal_common_data: Option<CommonCircuitData<F, D>>,

    /// Optional verifier data that is registered as public inputs.
    /// This is used in cyclic recursion to hold the circuit's own verifier key.
    pub(crate) verifier_data_public_input: Option<VerifierCircuitTarget>,
}
\n
    \n
  • virtual_target: This is not an actual wire in the witness, but just a target that help facilitate witness generation. In particular, a generator can assign a values to a virtual target, which can then be copied to other (virtual or concrete) targets. When we generate the final witness (a grid of wire values), these virtual targets will go away.
    • \n
\n

3.1 methods

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
/// Adds a gate to the circuit, and returns its index.
pub fn add_gate<G: Gate<F, D>>(&mut self, gate_type: G, mut constants: Vec<F>) -> usize {}

/// Returns a routable target with the given constant value.
/// for example, for ArithmeticGate, at least constant of `ONE` and `ZERO` is allocated
/// those constant target are all virtual target
pub fn constant(&mut self, c: F) -> Target {
    if let Some(&target) = self.constants_to_targets.get(&c) {
        // We already have a wire for this constant.
        return target;
    }

    let target = self.add_virtual_target();
    self.constants_to_targets.insert(c, target);
    self.targets_to_constants.insert(target, c);

    target
}

/// get polynomial to constants for each gate
/// the size of the vector is the number of constants (a value for the whole circuit, gate selectors + constants for gate inputs)
/// each polynomial interpolates the actual constant value at each gate, 
fn constant_polys(&self) -> Vec<PolynomialValues<F>> {}

/// if zero_knwoledge is set, it will blind the circuit
/// if the number of circuits is not a power of 2, pad NoopGate, and make the total number of gates to be power of 2
fn blind_and_pad(&mut self) {}
/**
  * hash the circuit inputs into m Target, m is a constant value of 4. During this process, a `PoseidonGate` is added as the hashing of inputs is conducted
  * @param inputs: public inputs and private inputs
  */
pub fn hash_n_to_hash_no_pad<H: AlgebraicHasher<F>>(
    &mut self,
    inputs: Vec<Target>,
) -> HashOutTarget {
    HashOutTarget::from_vec(self.hash_n_to_m_no_pad::<H>(inputs, NUM_HASH_OUT_ELTS))
}


/**
  * @param k_is, the coset shift, g^{j}, j \\in [0,num_shifts - 1], num_shifts is equal to num_routed_wires (columns); e.q 100
  * @subgroup: the \\Omega domain. \\Omega^{i}, i \\in [0, N-1], N is number of gates (rows); e.q 8 (2^3)
  * @return(0): vector of the sigma polynomial (sigma polynomial is the copy constraint permutation); the size is equal to the number of cols
  * @return(1): the forest is a data structure representing all wires copy constraint. the collected wires forms a unique partition
  */
fn sigma_vecs(&self, k_is: &[F], subgroup: &[F]) -> (Vec<PolynomialValues<F>>, Forest) {}
\n\n

3.2 functions

1
2
3
/// Finds a set of shifts that result in unique cosets for the multiplicative subgroup of size `2^subgroup_bits`.
/// Let `g` be a generator of the entire multiplicative group. the result is just g^0, ..., g^(num_shifts - 1)
pub fn get_unique_coset_shifts<F: Field>(subgroup_size: usize, num_shifts: usize) -> Vec<F> {}
\n\n

4. selectors

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
/// Selector polynomials are computed as follows:
/// Partition the gates into (the smallest amount of) groups `{ G_i }`, such that for each group `G`
/// `|G| + max_{g in G} g.degree() <= max_degree`. These groups are constructed greedily from
/// the list of gates sorted by degree.
/// We build a selector polynomial `S_i` for each group `G_i`, with
/// S_i\\[j\\] =
///     if j-th row gate=g_k in G_i
///         k  (k \\in [0, NUM_OF_GATE_TYPES)
///     else
///         UNUSED_SELECTOR
/// @param gates: the types of gates , gate type is differentiated by gate.id()
/// @param instances: all instances of gates ( a same type of gate could be used multiple times)
/// @param max_degree: max_degree of each group
/// @return(0): a vector of selector polynomials, the size of the vector is number of groups, in each group, the polynomial indicate which gate type it is
/// @return(1) SelectorsInfo {
///     selector_indices, a vector of gate group information. the size of vector is same to the number of gate types. the value represent which group the gate type belongs to
///     groups: a vector of groups information, the size is the number of groups. in each group, it is a range of the gate types (ordered as a method described above)
/// }
pub(crate) fn selector_polynomials<F: RichField + Extendable<D>, const D: usize>(
    gates: &[GateRef<F, D>],
    instances: &[GateInstance<F, D>],
    max_degree: usize,
) -> (Vec<PolynomialValues<F>>, SelectorsInfo) {}
\n\n

5. permutation

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
pub struct WirePartition {
    // each element of the vector is a vector of copy constraint
    partition: Vec<Vec<Wire>>,
}

/// Disjoint Set Forest data-structure following <https://en.wikipedia.org/wiki/Disjoint-set_data_structure>.
pub struct Forest {
    /// A map of parent pointers, stored as indices. the parent value is the partition it belongs to
    /// those node with same parent value, means they are in the copy constraint 
    pub(crate) parents: Vec<usize>,

    num_wires: usize,
    num_routed_wires: usize, // num_routed_wires is used to construct all wires
    degree: usize,
}
\n\n

6. hash

defined in CircuitBuilder

\n
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
/// Hash function used for building Merkle trees.
type Hasher: Hasher<Self::F>;
/// Algebraic hash function used for the challenger and hashing public inputs.
type InnerHasher: AlgebraicHasher<Self::F>;

/// Trait for algebraic hash functions, built from a permutation using the sponge construction.
pub trait AlgebraicHasher<F: RichField>: Hasher<F, Hash = HashOut<F>> {
    type AlgebraicPermutation: PlonkyPermutation<Target>;

    /// Circuit to conditionally swap two chunks of the inputs (useful in verifying Merkle proofs),
    /// then apply the permutation.
    fn permute_swapped<const D: usize>(
        inputs: Self::AlgebraicPermutation,
        swap: BoolTarget,
        builder: &mut CircuitBuilder<F, D>,
    ) -> Self::AlgebraicPermutation
    where
        F: RichField + Extendable<D>;
}
\n\n\n

7. IOP

7.1 generator

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
/// Generator used to fill an extra constant.
#[derive(Debug, Clone, Default)]
pub struct ConstantGenerator<F: Field> {
    pub row: usize,  //  specifying the gate row number
    pub constant_index: usize, // index of constant input. for example, for a constant gate, num_consts is specified; num_consts will specify the total number of constants
    pub wire_index: usize,     // index of constant wire output
    pub constant: F, // the constant used to fill a target
}


/// A generator which runs once after a list of dependencies is present in the witness.
pub trait SimpleGenerator<F: RichField + Extendable<D>, const D: usize>:
    'static + Send + Sync + Debug
{
    fn id(&self) -> String;

    fn dependencies(&self) -> Vec<Target>;

    fn run_once(&self, witness: &PartitionWitness<F>, out_buffer: &mut GeneratedValues<F>);

    fn adapter(self) -> SimpleGeneratorAdapter<F, Self, D>
    where
        Self: Sized,
    {
        SimpleGeneratorAdapter {
            inner: self,
            _phantom: PhantomData,
        }
    }

    fn serialize(&self, dst: &mut Vec<u8>, common_data: &CommonCircuitData<F, D>) -> IoResult<()>;

    fn deserialize(src: &mut Buffer, common_data: &CommonCircuitData<F, D>) -> IoResult<Self>
    where
        Self: Sized;
}
\n\n

8. FRI

Questions

"},{"title":"polygon zkEVM","date":"2023-12-11T05:24:03.000Z","_content":"\n\n\n\n## zkProver\n### State Machines\nThe zkProver follows modularity of design to the extent that, except for a few components, it is mainly a cluster of State Machines. It has a total of thirteen (13) State Machines;\n- The Main State Machine\n- Secondary State Machines → Binary SM, Storage SM, Memory SM, Arithmetic SM, Keccak Function SM, PoseidonG SM,\n- Auxiliary State Machines → Padding-PG SM, Padding-KK SM, Bits2Field SM, Memory Align SM, Byte4 SM, ROM SM.\nDue to the modular design of zkProver, the Main State Machine can delegate as many of tasks as possible to other specialist State Machines. This heavily improves the efficiency of Main SM.\n![](/images/zkp/zkevm/polygon/state_machines.png)\n\n### Two Novel Languages For zkProver\n#### Zero-Knowledge Assembly(zkASM)\nAs an Assembly language, the Zero-Knowledge Assembly (or zkASM) language is specially designed to map instructions from the zkProver's Main State Machine to other State Machines. In case of the State Machines with firmware, zkASM is the Interpreter for the firmware.\n\n#### Polynomial Identity Language(PIL)\nThe Polynomial Identity Language (or PIL) is especially designed for the zkProver. Almost all state machines express computations in terms of polynomials. Therefore, state transitions in State Machines must satisfy computation-specific polynomial identities.\n\n### Microprocessor State Machines\nThere are two microprocessor-type state machines; the Main SM and the Storage SM. These two State Machines have the Firmware and the Hardware part. It is worth noting that each of these Microprocessor SM has its own ROM.\n\nThe Firmware part runs the zkASM language to set up the logic and rules, which are expressed in JSON format and stored in a ROM. The JSON-file is then parsed to the specific SM Executor, which executes Storage Actions in compliance with the rules and logic in the JSON-file.\n\nThe Hardware component, which uses the Polynomial Identity Language (PIL), defines constraints (or polynomial identities), expresses them in JSON format, and stores them in the accompanying JSON-file. These constraints are parsed to the specific SM Executor, because all computations must be executed in conformance to the polynomial identities.\n\n![](/images/zkp/zkevm/polygon/micro_processor.png)\n","source":"_posts/cryptography/zkp/polygon_zkevm.md","raw":"---\ntitle: polygon zkEVM\ndate: 2023-12-11 13:24:03\ntags: [cryptography,zkp]\n---\n\n\n\n\n## zkProver\n### State Machines\nThe zkProver follows modularity of design to the extent that, except for a few components, it is mainly a cluster of State Machines. It has a total of thirteen (13) State Machines;\n- The Main State Machine\n- Secondary State Machines → Binary SM, Storage SM, Memory SM, Arithmetic SM, Keccak Function SM, PoseidonG SM,\n- Auxiliary State Machines → Padding-PG SM, Padding-KK SM, Bits2Field SM, Memory Align SM, Byte4 SM, ROM SM.\nDue to the modular design of zkProver, the Main State Machine can delegate as many of tasks as possible to other specialist State Machines. This heavily improves the efficiency of Main SM.\n![](/images/zkp/zkevm/polygon/state_machines.png)\n\n### Two Novel Languages For zkProver\n#### Zero-Knowledge Assembly(zkASM)\nAs an Assembly language, the Zero-Knowledge Assembly (or zkASM) language is specially designed to map instructions from the zkProver's Main State Machine to other State Machines. In case of the State Machines with firmware, zkASM is the Interpreter for the firmware.\n\n#### Polynomial Identity Language(PIL)\nThe Polynomial Identity Language (or PIL) is especially designed for the zkProver. Almost all state machines express computations in terms of polynomials. Therefore, state transitions in State Machines must satisfy computation-specific polynomial identities.\n\n### Microprocessor State Machines\nThere are two microprocessor-type state machines; the Main SM and the Storage SM. These two State Machines have the Firmware and the Hardware part. It is worth noting that each of these Microprocessor SM has its own ROM.\n\nThe Firmware part runs the zkASM language to set up the logic and rules, which are expressed in JSON format and stored in a ROM. The JSON-file is then parsed to the specific SM Executor, which executes Storage Actions in compliance with the rules and logic in the JSON-file.\n\nThe Hardware component, which uses the Polynomial Identity Language (PIL), defines constraints (or polynomial identities), expresses them in JSON format, and stores them in the accompanying JSON-file. These constraints are parsed to the specific SM Executor, because all computations must be executed in conformance to the polynomial identities.\n\n![](/images/zkp/zkevm/polygon/micro_processor.png)\n","slug":"cryptography/zkp/polygon_zkevm","published":1,"updated":"2023-12-11T08:34:59.530Z","_id":"clq0nd00w00024z7u843t2waw","comments":1,"layout":"post","photos":[],"link":"","content":"\n\n\n

zkProver

State Machines

The zkProver follows modularity of design to the extent that, except for a few components, it is mainly a cluster of State Machines. It has a total of thirteen (13) State Machines;

\n
    \n
  • The Main State Machine
    • \n
  • Secondary State Machines → Binary SM, Storage SM, Memory SM, Arithmetic SM, Keccak Function SM, PoseidonG SM,
    • \n
  • Auxiliary State Machines → Padding-PG SM, Padding-KK SM, Bits2Field SM, Memory Align SM, Byte4 SM, ROM SM.
    Due to the modular design of zkProver, the Main State Machine can delegate as many of tasks as possible to other specialist State Machines. This heavily improves the efficiency of Main SM.
    • \n
\n

Two Novel Languages For zkProver

Zero-Knowledge Assembly(zkASM)

As an Assembly language, the Zero-Knowledge Assembly (or zkASM) language is specially designed to map instructions from the zkProver’s Main State Machine to other State Machines. In case of the State Machines with firmware, zkASM is the Interpreter for the firmware.

\n

Polynomial Identity Language(PIL)

The Polynomial Identity Language (or PIL) is especially designed for the zkProver. Almost all state machines express computations in terms of polynomials. Therefore, state transitions in State Machines must satisfy computation-specific polynomial identities.

\n

Microprocessor State Machines

There are two microprocessor-type state machines; the Main SM and the Storage SM. These two State Machines have the Firmware and the Hardware part. It is worth noting that each of these Microprocessor SM has its own ROM.

\n

The Firmware part runs the zkASM language to set up the logic and rules, which are expressed in JSON format and stored in a ROM. The JSON-file is then parsed to the specific SM Executor, which executes Storage Actions in compliance with the rules and logic in the JSON-file.

\n

The Hardware component, which uses the Polynomial Identity Language (PIL), defines constraints (or polynomial identities), expresses them in JSON format, and stores them in the accompanying JSON-file. These constraints are parsed to the specific SM Executor, because all computations must be executed in conformance to the polynomial identities.

\n

\n","site":{"data":{}},"excerpt":"","more":"\n\n\n

zkProver

State Machines

The zkProver follows modularity of design to the extent that, except for a few components, it is mainly a cluster of State Machines. It has a total of thirteen (13) State Machines;

\n
    \n
  • The Main State Machine
    • \n
  • Secondary State Machines → Binary SM, Storage SM, Memory SM, Arithmetic SM, Keccak Function SM, PoseidonG SM,
    • \n
  • Auxiliary State Machines → Padding-PG SM, Padding-KK SM, Bits2Field SM, Memory Align SM, Byte4 SM, ROM SM.
    Due to the modular design of zkProver, the Main State Machine can delegate as many of tasks as possible to other specialist State Machines. This heavily improves the efficiency of Main SM.
    • \n
\n

Two Novel Languages For zkProver

Zero-Knowledge Assembly(zkASM)

As an Assembly language, the Zero-Knowledge Assembly (or zkASM) language is specially designed to map instructions from the zkProver’s Main State Machine to other State Machines. In case of the State Machines with firmware, zkASM is the Interpreter for the firmware.

\n

Polynomial Identity Language(PIL)

The Polynomial Identity Language (or PIL) is especially designed for the zkProver. Almost all state machines express computations in terms of polynomials. Therefore, state transitions in State Machines must satisfy computation-specific polynomial identities.

\n

Microprocessor State Machines

There are two microprocessor-type state machines; the Main SM and the Storage SM. These two State Machines have the Firmware and the Hardware part. It is worth noting that each of these Microprocessor SM has its own ROM.

\n

The Firmware part runs the zkASM language to set up the logic and rules, which are expressed in JSON format and stored in a ROM. The JSON-file is then parsed to the specific SM Executor, which executes Storage Actions in compliance with the rules and logic in the JSON-file.

\n

The Hardware component, which uses the Polynomial Identity Language (PIL), defines constraints (or polynomial identities), expresses them in JSON format, and stores them in the accompanying JSON-file. These constraints are parsed to the specific SM Executor, because all computations must be executed in conformance to the polynomial identities.

\n

\n"}],"PostAsset":[],"PostCategory":[],"PostTag":[{"post_id":"clokyy8dm0005qwsja95zaawe","tag_id":"clokyy8dk0002qwsj0l5me3u1","_id":"clokyy8dm0008qwsj3znq2qrc"},{"post_id":"clokyy8dh0000qwsj6ebuelv1","tag_id":"clokyy8dk0002qwsj0l5me3u1","_id":"clokyy8dn000cqwsja7jo4in6"},{"post_id":"clokyy8dh0000qwsj6ebuelv1","tag_id":"clokyy8dm0006qwsjcajbbgh0","_id":"clokyy8do000eqwsj32y8gju9"},{"post_id":"clokyy8dj0001qwsj9waw412a","tag_id":"clokyy8dk0002qwsj0l5me3u1","_id":"clokyy8dp000hqwsj8nvnb73i"},{"post_id":"clokyy8do000gqwsj13yvh382","tag_id":"clokyy8do000fqwsjasxpecdm","_id":"clokyy8dp000kqwsj0nhw3uvb"},{"post_id":"clokyy8dm0009qwsj4fr9fx90","tag_id":"clokyy8do000fqwsjasxpecdm","_id":"clokyy8dq000qqwsjecokb6pk"},{"post_id":"clokyy8dp000nqwsjgbxocb3i","tag_id":"clokyy8do000fqwsjasxpecdm","_id":"clokyy8dq000sqwsjeqa36czr"},{"post_id":"clokyy8dn000bqwsj9gt33o5h","tag_id":"clokyy8do000fqwsjasxpecdm","_id":"clokyy8dr000vqwsj01jb05f6"},{"post_id":"clokyy8do000dqwsj2vh07e7k","tag_id":"clokyy8do000fqwsjasxpecdm","_id":"clokyy8dr000zqwsj0ef9b9hi"},{"post_id":"clokyy8dq000pqwsjfar5aa4q","tag_id":"clokyy8do000fqwsjasxpecdm","_id":"clokyy8dt001bqwsjgw34gzwh"},{"post_id":"clokyy8dq000pqwsjfar5aa4q","tag_id":"clokyy8dr0011qwsjdywmbc62","_id":"clokyy8dt001dqwsj92lifgwe"},{"post_id":"clokyy8dq000pqwsjfar5aa4q","tag_id":"clokyy8ds0016qwsjb4c37d4g","_id":"clokyy8dt001gqwsje8zvg1fo"},{"post_id":"clokyy8dq000rqwsj5r3jepey","tag_id":"clokyy8ds0019qwsjevwpamn0","_id":"clokyy8dt001iqwsjfpxe9uvo"},{"post_id":"clokyy8dq000uqwsj46jo8rlt","tag_id":"clokyy8ds0019qwsjevwpamn0","_id":"clokyy8dt001lqwsjfky4bmth"},{"post_id":"clokyy8dt001kqwsj843kh526","tag_id":"clokyy8dt001jqwsjbfai4oxz","_id":"clokyy8du001nqwsjfqy76zi3"},{"post_id":"clokyy8dr000wqwsjaarwgrmn","tag_id":"clokyy8dt001jqwsjbfai4oxz","_id":"clokyy8du001qqwsj91e9gqqo"},{"post_id":"clokyy8du001pqwsj1k4uhzw0","tag_id":"clokyy8dt001jqwsjbfai4oxz","_id":"clokyy8du001sqwsj12wcgz9b"},{"post_id":"clokyy8dr000yqwsj41fu3ebh","tag_id":"clokyy8dt001jqwsjbfai4oxz","_id":"clokyy8du001vqwsjf63p54qk"},{"post_id":"clokyy8du001rqwsj7syrcry3","tag_id":"clokyy8dt001jqwsjbfai4oxz","_id":"clokyy8dv001xqwsj30s8eqen"},{"post_id":"clokyy8dr0010qwsj4oml40f0","tag_id":"clokyy8dt001jqwsjbfai4oxz","_id":"clokyy8dv0020qwsj99twcuuo"},{"post_id":"clokyy8ds0012qwsj7v2de3x4","tag_id":"clokyy8dt001jqwsjbfai4oxz","_id":"clokyy8dv0024qwsj21muefxx"},{"post_id":"clokyy8dv0023qwsjbe4aeis9","tag_id":"clokyy8dk0002qwsj0l5me3u1","_id":"clokyy8dw0026qwsj1o9f4y0e"},{"post_id":"clokyy8dv0023qwsjbe4aeis9","tag_id":"clokyy8dm0006qwsjcajbbgh0","_id":"clokyy8dx0029qwsj74s5gr22"},{"post_id":"clokyy8ds0014qwsjeg0uevgt","tag_id":"clokyy8dt001jqwsjbfai4oxz","_id":"clokyy8dx002bqwsj5ynk8d2b"},{"post_id":"clokyy8dv0025qwsjg5vzbwcu","tag_id":"clokyy8dk0002qwsj0l5me3u1","_id":"clokyy8dx002eqwsjhwtt76q2"},{"post_id":"clokyy8dv0025qwsjg5vzbwcu","tag_id":"clokyy8dm0006qwsjcajbbgh0","_id":"clokyy8dx002fqwsj0pbe66d4"},{"post_id":"clokyy8dw0028qwsj2vul70i7","tag_id":"clokyy8dk0002qwsj0l5me3u1","_id":"clokyy8dx002hqwsjbsur18r9"},{"post_id":"clokyy8dw0028qwsj2vul70i7","tag_id":"clokyy8dm0006qwsjcajbbgh0","_id":"clokyy8dx002iqwsj6j5wcl99"},{"post_id":"clokyy8ds0015qwsjh6j437e0","tag_id":"clokyy8dt001jqwsjbfai4oxz","_id":"clokyy8dx002kqwsj3cl16h9h"},{"post_id":"clokyy8dx002aqwsjacvpcivw","tag_id":"clokyy8dm0006qwsjcajbbgh0","_id":"clokyy8dx002lqwsjc2g1h6d9"},{"post_id":"clokyy8dx002dqwsj8w7zb4o4","tag_id":"clokyy8dm0006qwsjcajbbgh0","_id":"clokyy8dy002nqwsj5zeu2uni"},{"post_id":"clokyy8ds0017qwsj9lqm1w2r","tag_id":"clokyy8dt001jqwsjbfai4oxz","_id":"clokyy8dy002oqwsj4nxi1rgg"},{"post_id":"clokyy8ds0018qwsj3eer6n3l","tag_id":"clokyy8dt001jqwsjbfai4oxz","_id":"clokyy8dy002qqwsjerqu7lkg"},{"post_id":"clokyy8ds001aqwsjdxc70zfv","tag_id":"clokyy8dt001jqwsjbfai4oxz","_id":"clokyy8dy002rqwsjcssn2d2r"},{"post_id":"clokyy8dt001cqwsj0grhcnus","tag_id":"clokyy8dt001jqwsjbfai4oxz","_id":"clokyy8dy002tqwsj7u0qesqk"},{"post_id":"clokyy8dt001fqwsjbi7t4e7s","tag_id":"clokyy8dt001jqwsjbfai4oxz","_id":"clokyy8dy002uqwsj8czk072q"},{"post_id":"clokyy8dt001hqwsj4zoehfes","tag_id":"clokyy8dt001jqwsjbfai4oxz","_id":"clokyy8dy002wqwsjbvlcg7am"},{"post_id":"clokyy8dt001mqwsjfk9x8s37","tag_id":"clokyy8dt001jqwsjbfai4oxz","_id":"clokyy8dy002yqwsj3edrfhml"},{"post_id":"clokyy8dt001mqwsjfk9x8s37","tag_id":"clokyy8dy002vqwsjeapz6z25","_id":"clokyy8dy002zqwsjbq0bbhwl"},{"post_id":"clokyy8dv001zqwsj9hdfghdd","tag_id":"clokyy8dy0033qwsjexf1205y","_id":"clokyy8dy0037qwsj1sde9p8c"},{"post_id":"clokyy8dv0021qwsjhghcad7b","tag_id":"clokyy8dy0036qwsjbbfo2mww","_id":"clokyy8dy0038qwsj3umr87jx"},{"post_id":"clokyy8e00039qwsjfxurcxan","tag_id":"clokyy8do000fqwsjasxpecdm","_id":"clokyy8e1003bqwsj7twkfcni"},{"post_id":"clokyy8e00039qwsjfxurcxan","tag_id":"clokyy8dy002xqwsjftg07glq","_id":"clokyy8e1003dqwsj4hncbhbd"},{"post_id":"clokyy8e1003aqwsj8bnledgy","tag_id":"clokyy8do000fqwsjasxpecdm","_id":"clokyy8e1003fqwsjc7vv6zug"},{"post_id":"clokyy8e1003aqwsj8bnledgy","tag_id":"clokyy8dy002xqwsjftg07glq","_id":"clokyy8e2003hqwsj7lka58og"},{"post_id":"clokyy8e1003cqwsjc3p416e4","tag_id":"clokyy8do000fqwsjasxpecdm","_id":"clokyy8e2003jqwsjcff88w6t"},{"post_id":"clokyy8e1003cqwsjc3p416e4","tag_id":"clokyy8dy002xqwsjftg07glq","_id":"clokyy8e2003lqwsjc5zwhw8w"},{"post_id":"clokyy8e1003eqwsj3oda50v5","tag_id":"clokyy8dm0006qwsjcajbbgh0","_id":"clokyy8e3003nqwsj8mk3ej5y"},{"post_id":"clokyy8e1003gqwsj9p1scg9k","tag_id":"clokyy8do000fqwsjasxpecdm","_id":"clokyy8e3003qqwsj322l2852"},{"post_id":"clokyy8e2003iqwsj23zw9035","tag_id":"clokyy8do000fqwsjasxpecdm","_id":"clokyy8e4003uqwsjgm41brc0"},{"post_id":"clokyy8e2003iqwsj23zw9035","tag_id":"clokyy8dy002xqwsjftg07glq","_id":"clokyy8e4003xqwsj9r2t2d31"},{"post_id":"clokyy8e2003kqwsjg4hf5noa","tag_id":"clokyy8dt001jqwsjbfai4oxz","_id":"clokyy8e4003yqwsj367c01en"},{"post_id":"clokyy8e2003mqwsj9ztifql4","tag_id":"clokyy8e3003pqwsj13nx4h92","_id":"clokyy8e40040qwsj6dqray98"},{"post_id":"clokyy8e3003oqwsj76cl15kp","tag_id":"clokyy8e4003wqwsj7uh112pz","_id":"clokyy8e40041qwsj93t1f6vk"},{"post_id":"clokyy8e3003rqwsje2592e0r","tag_id":"clokyy8e4003wqwsj7uh112pz","_id":"clokyy8e40043qwsj27jq1jfb"},{"post_id":"clokyy8e3003tqwsj3w7e71hn","tag_id":"clokyy8e4003wqwsj7uh112pz","_id":"clokyy8e40045qwsj471w5z55"},{"post_id":"clokyy8e4003vqwsj77aj2fxl","tag_id":"clokyy8e4003wqwsj7uh112pz","_id":"clokyy8e40046qwsjbv0r0qdg"},{"post_id":"clp8cm0vx0000l57u7eqtbi6s","tag_id":"clp8cm0vz0001l57uazgmgnu5","_id":"clp8cm0w40002l57u85wb4ln4"},{"post_id":"clp8cm0w50003l57ubl7e5d6y","tag_id":"clp8cm0vz0001l57uazgmgnu5","_id":"clp8cm0w60005l57ubyymd5zb"},{"post_id":"clp8cm0w60004l57uckd04dmz","tag_id":"clp8cm0vz0001l57uazgmgnu5","_id":"clp8cm0w70007l57ugr0oce1c"},{"post_id":"clp8cm0w60006l57u0fvme5s2","tag_id":"clp8cm0vz0001l57uazgmgnu5","_id":"clp8cm0w70008l57u46esdf2b"},{"post_id":"clp95py9p0000e37u3ivq1as7","tag_id":"clp95py9p0001e37ubed0fc6x","_id":"clp95py9u0002e37u04o36rzx"},{"post_id":"clp95py9w0003e37uhwx0baye","tag_id":"clp95py9p0001e37ubed0fc6x","_id":"clp95py9x0004e37uhe5ggxyr"},{"post_id":"clp3h29tl0000p97ug63eajx3","tag_id":"clp95py9p0001e37ubed0fc6x","_id":"clp95py9x0005e37uao1y6l14"},{"post_id":"clp9d5a6z0007e37u1cxf6c30","tag_id":"clp9d6e5q0008e37uhxcaa8dh","_id":"clp9d6e5r0009e37ub41xctyw"},{"post_id":"clphpvdof0000aj7u9gaz5qfy","tag_id":"clokyy8dr000xqwsj9q3g4o5b","_id":"clphpvdoi0001aj7u0ov372gu"},{"post_id":"clphpvdoj0003aj7ugctear78","tag_id":"clokyy8do000fqwsjasxpecdm","_id":"clphpvdoj0007aj7u7unw5mx1"},{"post_id":"clphpvdoj0003aj7ugctear78","tag_id":"clokyy8dy002xqwsjftg07glq","_id":"clphpvdoj0008aj7u0ltm0gbj"},{"post_id":"clphpvdoj0005aj7ug6vbhr5l","tag_id":"clokyy8do000fqwsjasxpecdm","_id":"clphpvdoj0009aj7u3ktze2r0"},{"post_id":"clphpvdoj0005aj7ug6vbhr5l","tag_id":"clokyy8dy002xqwsjftg07glq","_id":"clphpvdoj000aaj7u7n244zgr"},{"post_id":"clphpvdok000baj7u3i3te2rg","tag_id":"clokyy8do000fqwsjasxpecdm","_id":"clphpvdol000daj7ubp814h2h"},{"post_id":"clphpvdok000baj7u3i3te2rg","tag_id":"clokyy8dy002xqwsjftg07glq","_id":"clphpvdol000eaj7ucw8nbcww"},{"post_id":"clphpvdok000caj7ubjq53ldn","tag_id":"clokyy8do000fqwsjasxpecdm","_id":"clphpvdol000faj7ub7tte5bg"},{"post_id":"clphpvdok000caj7ubjq53ldn","tag_id":"clokyy8dy002xqwsjftg07glq","_id":"clphpvdol000gaj7u3wqldktg"},{"post_id":"clpi1wlcr0000067ufpzn63zv","tag_id":"clp9d6e5q0008e37uhxcaa8dh","_id":"clpi1x45e00001o7u7c8k8j51"},{"post_id":"clokyy8e1003gqwsj9p1scg9k","tag_id":"clpm1qei20001hpsj09vdgxjm","_id":"clpm1qei80003hpsj7yjgb3rc"},{"post_id":"clpm1qei10000hpsj4saxbd96","tag_id":"clokyy8do000fqwsjasxpecdm","_id":"clpm1qei80004hpsjbxmcfbmj"},{"post_id":"clpm1qei10000hpsj4saxbd96","tag_id":"clpm1qei20001hpsj09vdgxjm","_id":"clpm1qei80005hpsj2mk298gs"},{"post_id":"clpoya0gr0000pksj12ah1gzj","tag_id":"clokyy8do000fqwsjasxpecdm","_id":"clpoya0gu0001pksj5tpz36bl"},{"post_id":"clpoya0gr0000pksj12ah1gzj","tag_id":"clokyy8dy002xqwsjftg07glq","_id":"clpoya0gu0002pksjh66nabe3"},{"post_id":"clp96plbb0006e37uei4tcdkw","tag_id":"clp9d6e5q0008e37uhxcaa8dh","_id":"clq0nd00w00014z7ugvhxfsgu"},{"post_id":"clq0nd00w00004z7u3a6ubwbd","tag_id":"clokyy8do000fqwsjasxpecdm","_id":"clq0nd00x00034z7uba2abgjb"},{"post_id":"clq0nd00w00004z7u3a6ubwbd","tag_id":"clokyy8dy002xqwsjftg07glq","_id":"clq0nd00x00044z7u86wqcako"},{"post_id":"clq0nd00w00024z7u843t2waw","tag_id":"clokyy8do000fqwsjasxpecdm","_id":"clq0nd00x00054z7u1coxge4a"},{"post_id":"clq0nd00w00024z7u843t2waw","tag_id":"clokyy8dy002xqwsjftg07glq","_id":"clq0nd00x00064z7ueqeb329q"}],"Tag":[{"name":"blockchain","_id":"clokyy8dk0002qwsj0l5me3u1"},{"name":"geth","_id":"clokyy8dm0006qwsjcajbbgh0"},{"name":"cryptography","_id":"clokyy8do000fqwsjasxpecdm"},{"name":"number_theory","_id":"clokyy8dr000xqwsj9q3g4o5b"},{"name":"mpc","_id":"clokyy8dr0011qwsjdywmbc62"},{"name":"ecdsa","_id":"clokyy8ds0016qwsjb4c37d4g"},{"name":"golang","_id":"clokyy8ds0019qwsjevwpamn0"},{"name":"rust","_id":"clokyy8dt001jqwsjbfai4oxz"},{"name":"cargo","_id":"clokyy8dy002vqwsjeapz6z25"},{"name":"zkp","_id":"clokyy8dy002xqwsjftg07glq"},{"name":"tool","_id":"clokyy8dy0033qwsjexf1205y"},{"name":"math","_id":"clokyy8dy0036qwsjbbfo2mww"},{"name":"rust-crate","_id":"clokyy8e3003pqwsj13nx4h92"},{"name":"rust-std","_id":"clokyy8e4003wqwsj7uh112pz"},{"name":"cpp","_id":"clp8cm0vz0001l57uazgmgnu5"},{"name":"cuda","_id":"clp95py9p0001e37ubed0fc6x"},{"name":"arithmatic","_id":"clp9d6e5q0008e37uhxcaa8dh"},{"name":"ec","_id":"clpm1qei20001hpsj09vdgxjm"}]}} \ No newline at end of file diff --git a/public/2023/11/06/cryptography/zkp/plonky2-code-analysis/index.html b/public/2023/11/06/cryptography/zkp/plonky2-code-analysis/index.html index 421f7372..82440264 100644 --- a/public/2023/11/06/cryptography/zkp/plonky2-code-analysis/index.html +++ b/public/2023/11/06/cryptography/zkp/plonky2-code-analysis/index.html @@ -6,15 +6,15 @@ plonky2 code analysis | cliff's personal blog - + - + - + @@ -102,8 +102,9 @@

type="text/javascript"> -

Config

CircuitConfig

the default config is Self::standard_recursion_config() with value as below

-
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
Self {
  num_wires: 135,
  num_routed_wires: 80, 
  num_constants: 2,
  use_base_arithmetic_gate: true,
  security_bits: 100,
  num_challenges: 2,
  zero_knowledge: false,
  max_quotient_degree_factor: 8,
  fri_config: FriConfig {
      rate_bits: 3,
      cap_height: 4,
      proof_of_work_bits: 16,
      reduction_strategy: FriReductionStrategy::ConstantArityBits(4, 5),
      num_query_rounds: 28,
  },
}
+

introduction

this post is a source code analysis of https://github.com/0xPolygonZero/plonky2

+

1. Config

1.1 CircuitConfig

the default config is Self::standard_recursion_config() with value as below

+
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
Self {
  num_wires: 135,
  num_routed_wires: 80, 
  num_constants: 2,  // used in ConstantGate, which hold the consant value for the circuit
  /// Whether to use a dedicated gate for base field arithmetic, rather than using a single gate
  /// for both base field and extension field arithmetic.
  use_base_arithmetic_gate: true,
  security_bits: 100,
  /// The number of challenge points to generate, for IOPs that have soundness errors of (roughly)
  /// `degree / |F|`.
  num_challenges: 2,
  zero_knowledge: false,
  /// A cap on the quotient polynomial's degree factor. The actual degree factor is derived
  /// systematically, but will never exceed this value.
  max_quotient_degree_factor: 8,
  fri_config: FriConfig {
      rate_bits: 3,  // used in lde
      cap_height: 4,
      proof_of_work_bits: 16,
      reduction_strategy: FriReductionStrategy::ConstantArityBits(4, 5),
      num_query_rounds: 28,
  },
}

notes

    @@ -112,34 +113,30 @@

    Config

    num_routed_wires, the default value is 80. For eample, for a arithmetic gate, it computes const_0 * multiplicand_0 * multiplicand_1 + const_1 * addend. i.e 2 wires for multiplicands and 1 wire for the addend and 1 for output. total is 4. therefore, the gate can have num_routed_wires/4 operations, in this case, it is 20. the circuit builder will add gate when number of operations exceeds 20.

-

FriConfig

1
2
3
4
5
6
7
8
9
10
11
12
13
14
pub struct FriConfig {
/// `rate = 2^{-rate_bits}`. default is 3
pub rate_bits: usize,  

/// Height of Merkle tree caps. default is 4
pub cap_height: usize,
/// default is 16
pub proof_of_work_bits: u32,

pub reduction_strategy: FriReductionStrategy,

/// Number of query rounds to perform. default is 28
pub num_query_rounds: usize,
}
+

1.2 FriConfig

1
2
3
4
5
6
7
8
9
10
11
12
13
14
pub struct FriConfig {
/// `rate = 2^{-rate_bits}`. default is 3
pub rate_bits: usize,  

/// Height of Merkle tree caps. default is 4
pub cap_height: usize,
/// default is 16
pub proof_of_work_bits: u32,

pub reduction_strategy: FriReductionStrategy,

/// Number of query rounds to perform. default is 28
pub num_query_rounds: usize,
}
-

Gates

ArithmeticGate

1
2
3
4
5
6
7
8
9
10
11
/// A gate which can perform a weighted multiply-add, i.e. `result = c0 x y + c1 z`. If the config
/// supports enough routed wires, it can support several such operations in one gate.
pub struct ArithmeticGate {
    /// Number of arithmetic operations performed by an arithmetic gate.
    pub num_ops: usize,
}

/// A gate which takes a single constant parameter and outputs that value.
pub struct ConstantGate {
    pub(crate) num_consts: usize,
}
+

2. Gates

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
/// A custom gate.
pub trait Gate<F: RichField + Extendable<D>, const D: usize>: 'static + Send + Sync {
    fn id(&self) -> String;
    /// The maximum degree among this gate's constraint polynomials. it is 3 for ArithmeticGate (Left, Right, Output); 7 for PoseidonGate, 1 for ConstantGate, PublicInputGate;
    fn degree(&self) -> usize;

    /// The generators used to populate the witness.
    /// Note: This should return exactly 1 generator per operation in the gate.
    fn generators(&self, row: usize, local_constants: &[F]) -> Vec<WitnessGeneratorRef<F, D>>;

    /// Enables gates to store some "routed constants", if they have both unused constants and
    /// unused routed wires.
    ///
    /// Each entry in the returned `Vec` has the form `(constant_index, wire_index)`. `wire_index`
    /// must correspond to a *routed* wire.
    /// for ConstantGate, the size of vector is equal to num_of_consts, it's empty for ArithmeticGate, PublicInputGate, and PoseidonGate
    fn extra_constant_wires(&self) -> Vec<(usize, usize)> {
        vec![]
    }
}
+

ArithmeticGate

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
/// A gate which can perform a weighted multiply-add, i.e. `result = c0 x y + c1 z`. If the config
/// supports enough routed wires, it can support several such operations in one gate.
pub struct ArithmeticGate {
    /// Number of arithmetic operations performed by an arithmetic gate.
    pub num_ops: usize,
}

/// A gate which takes a single constant parameter and outputs that value.
pub struct ConstantGate {
    pub(crate) num_consts: usize,
}
/// for constant gate, the evaluation is just input - wire output, the result should be 0
fn eval_unfiltered(&self, vars: EvaluationVars<F, D>) -> Vec<F::Extension> {
    (0..self.num_consts)
        .map(|i| {
            vars.local_constants[self.const_input(i)] - vars.local_wires[self.wire_output(i)]
        })
        .collect()
}

/// A gate whose first four wires will be equal to a hash of public inputs. the value 4 is hadcoded as the hash
/// out inputs is 4 Targets. PublicInputGate is aliased as pi gate
pub struct PublicInputGate;
impl PublicInputGate {
    pub fn wires_public_inputs_hash() -> Range<usize> {
        0..4
    }
}
-

Circuit Builder

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
pub struct CircuitBuilder<F: RichField + Extendable<D>, const D: usize> {
    pub config: CircuitConfig,

    /// A domain separator, which is included in the initial Fiat-Shamir seed. This is generally not
    /// needed, but can be used to ensure that proofs for one application are not valid for another.
    /// Defaults to the empty vector.
    domain_separator: Option<Vec<F>>,

    /// The types of gates used in this circuit.
    gates: HashSet<GateRef<F, D>>,

    /// The concrete placement of each gate.
    pub(crate) gate_instances: Vec<GateInstance<F, D>>,

    /// Targets to be made public.
    public_inputs: Vec<Target>,

    /// The next available index for a `VirtualTarget`.
    virtual_target_index: usize,

    copy_constraints: Vec<CopyConstraint>,

    /// A tree of named scopes, used for debugging.
    context_log: ContextTree,

    /// Generators used to generate the witness.
    generators: Vec<WitnessGeneratorRef<F, D>>,

    /// maps constant value -> ConstantTarget(a virtual target)
    constants_to_targets: HashMap<F, Target>,
    targets_to_constants: HashMap<Target, F>,

    /// Memoized results of `arithmetic` calls.
    pub(crate) base_arithmetic_results: HashMap<BaseArithmeticOperation<F>, Target>,

    /// Memoized results of `arithmetic_extension` calls.
    pub(crate) arithmetic_results: HashMap<ExtensionArithmeticOperation<F, D>, ExtensionTarget<D>>,

    /// Map between gate type and the current gate of this type with available slots.
    current_slots: HashMap<GateRef<F, D>, CurrentSlot<F, D>>,

    /// List of constant generators used to fill the constant wires.
    constant_generators: Vec<ConstantGenerator<F>>,

    /// Rows for each LUT: LookupWire contains: first `LookupGate`, first `LookupTableGate`, last `LookupTableGate`.
    lookup_rows: Vec<LookupWire>,

    /// For each LUT index, vector of `(looking_in, looking_out)` pairs.
    lut_to_lookups: Vec<Lookup>,

    // Lookup tables in the form of `Vec<(input_value, output_value)>`.
    luts: Vec<LookupTable>,

    /// Optional common data. When it is `Some(goal_data)`, the `build` function panics if the resulting
    /// common data doesn't equal `goal_data`.
    /// This is used in cyclic recursion.
    pub(crate) goal_common_data: Option<CommonCircuitData<F, D>>,

    /// Optional verifier data that is registered as public inputs.
    /// This is used in cyclic recursion to hold the circuit's own verifier key.
    pub(crate) verifier_data_public_input: Option<VerifierCircuitTarget>,
}
+

3. Circuit Builder

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
pub struct CircuitBuilder<F: RichField + Extendable<D>, const D: usize> {
    pub config: CircuitConfig,

    /// A domain separator, which is included in the initial Fiat-Shamir seed. This is generally not
    /// needed, but can be used to ensure that proofs for one application are not valid for another.
    /// Defaults to the empty vector.
    domain_separator: Option<Vec<F>>,

    /// The types of gates used in this circuit.
    gates: HashSet<GateRef<F, D>>,

    /// The concrete placement of each gate.
    pub(crate) gate_instances: Vec<GateInstance<F, D>>,

    /// Targets to be made public.
    public_inputs: Vec<Target>,

    /// The next available index for a `VirtualTarget`.
    virtual_target_index: usize,

    copy_constraints: Vec<CopyConstraint>,

    /// A tree of named scopes, used for debugging.
    context_log: ContextTree,

    /// Generators used to generate the witness.
    generators: Vec<WitnessGeneratorRef<F, D>>,

    /// maps the constant value -> ConstantTarget(a virtual target)
    constants_to_targets: HashMap<F, Target>,
    /// an inverse map of the constants_to_targets
    targets_to_constants: HashMap<Target, F>,

    /// Memoized results of `arithmetic` calls.
    pub(crate) base_arithmetic_results: HashMap<BaseArithmeticOperation<F>, Target>,

    /// Memoized results of `arithmetic_extension` calls.
    pub(crate) arithmetic_results: HashMap<ExtensionArithmeticOperation<F, D>, ExtensionTarget<D>>,

    /// Map between gate type and the current gate of this type with available slots.
    current_slots: HashMap<GateRef<F, D>, CurrentSlot<F, D>>,

    /// List of constant generators used to fill the constant wires.
    constant_generators: Vec<ConstantGenerator<F>>,

    /// Rows for each LUT: LookupWire contains: first `LookupGate`, first `LookupTableGate`, last `LookupTableGate`.
    lookup_rows: Vec<LookupWire>,

    /// For each LUT index, vector of `(looking_in, looking_out)` pairs.
    lut_to_lookups: Vec<Lookup>,

    // Lookup tables in the form of `Vec<(input_value, output_value)>`.
    luts: Vec<LookupTable>,

    /// Optional common data. When it is `Some(goal_data)`, the `build` function panics if the resulting
    /// common data doesn't equal `goal_data`.
    /// This is used in cyclic recursion.
    pub(crate) goal_common_data: Option<CommonCircuitData<F, D>>,

    /// Optional verifier data that is registered as public inputs.
    /// This is used in cyclic recursion to hold the circuit's own verifier key.
    pub(crate) verifier_data_public_input: Option<VerifierCircuitTarget>,
}
  • virtual_target: This is not an actual wire in the witness, but just a target that help facilitate witness generation. In particular, a generator can assign a values to a virtual target, which can then be copied to other (virtual or concrete) targets. When we generate the final witness (a grid of wire values), these virtual targets will go away.
-

methods

1
2
3
4
5
6
7
8

/**
  * @param k_is, the coset shift, g^{j}, j \in [0,num_shifts - 1], num_shifts is equal to num_routed_wires (columns); e.q 100
  * @subgroup: the \Omega domain. \Omega^{i}, i \in [0, N-1], N is number of gates (rows); e.q 8 (2^3)
  * @return(0): vector of the sigma polynomial (sigma polynomial is the copy constraint permutation); the size is equal to the number of cols
  * @return(1): the forest is a data structure representing all wires copy constraint. the collected wires forms a unique partition
  */
fn sigma_vecs(&self, k_is: &[F], subgroup: &[F]) -> (Vec<PolynomialValues<F>>, Forest) {}
- -

functions

1
2
3
/// Finds a set of shifts that result in unique cosets for the multiplicative subgroup of size `2^subgroup_bits`.
/// Let `g` be a generator of the entire multiplicative group. the result is just g^0, ..., g^(num_shifts - 1)
pub fn get_unique_coset_shifts<F: Field>(subgroup_size: usize, num_shifts: usize) -> Vec<F> {}
- -

permutation

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
pub struct WirePartition {
    // each element of the vector is a vector of copy constraint
    partition: Vec<Vec<Wire>>,
}

/// Disjoint Set Forest data-structure following <https://en.wikipedia.org/wiki/Disjoint-set_data_structure>.
pub struct Forest {
    /// A map of parent pointers, stored as indices. the parent value is the partition it belongs to
    /// those node with same parent value, means they are in the copy constraint 
    pub(crate) parents: Vec<usize>,

    num_wires: usize,
    num_routed_wires: usize, // num_routed_wires is used to construct all wires
    degree: usize,
}
- -

hash

defined in CircuitBuilder

-
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
/// Hash function used for building Merkle trees.
type Hasher: Hasher<Self::F>;
/// Algebraic hash function used for the challenger and hashing public inputs.
type InnerHasher: AlgebraicHasher<Self::F>;

/// Trait for algebraic hash functions, built from a permutation using the sponge construction.
pub trait AlgebraicHasher<F: RichField>: Hasher<F, Hash = HashOut<F>> {
    type AlgebraicPermutation: PlonkyPermutation<Target>;

    /// Circuit to conditionally swap two chunks of the inputs (useful in verifying Merkle proofs),
    /// then apply the permutation.
    fn permute_swapped<const D: usize>(
        inputs: Self::AlgebraicPermutation,
        swap: BoolTarget,
        builder: &mut CircuitBuilder<F, D>,
    ) -> Self::AlgebraicPermutation
    where
        F: RichField + Extendable<D>;
}
+

3.1 methods

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
/// Adds a gate to the circuit, and returns its index.
pub fn add_gate<G: Gate<F, D>>(&mut self, gate_type: G, mut constants: Vec<F>) -> usize {}

/// Returns a routable target with the given constant value.
/// for example, for ArithmeticGate, at least constant of `ONE` and `ZERO` is allocated
/// those constant target are all virtual target
pub fn constant(&mut self, c: F) -> Target {
    if let Some(&target) = self.constants_to_targets.get(&c) {
        // We already have a wire for this constant.
        return target;
    }

    let target = self.add_virtual_target();
    self.constants_to_targets.insert(c, target);
    self.targets_to_constants.insert(target, c);

    target
}

/// get polynomial to constants for each gate
/// the size of the vector is the number of constants (a value for the whole circuit, gate selectors + constants for gate inputs)
/// each polynomial interpolates the actual constant value at each gate, 
fn constant_polys(&self) -> Vec<PolynomialValues<F>> {}

/// if zero_knwoledge is set, it will blind the circuit
/// if the number of circuits is not a power of 2, pad NoopGate, and make the total number of gates to be power of 2
fn blind_and_pad(&mut self) {}
/**
  * hash the circuit inputs into m Target, m is a constant value of 4. During this process, a `PoseidonGate` is added as the hashing of inputs is conducted
  * @param inputs: public inputs and private inputs
  */
pub fn hash_n_to_hash_no_pad<H: AlgebraicHasher<F>>(
    &mut self,
    inputs: Vec<Target>,
) -> HashOutTarget {
    HashOutTarget::from_vec(self.hash_n_to_m_no_pad::<H>(inputs, NUM_HASH_OUT_ELTS))
}


/**
  * @param k_is, the coset shift, g^{j}, j \in [0,num_shifts - 1], num_shifts is equal to num_routed_wires (columns); e.q 100
  * @subgroup: the \Omega domain. \Omega^{i}, i \in [0, N-1], N is number of gates (rows); e.q 8 (2^3)
  * @return(0): vector of the sigma polynomial (sigma polynomial is the copy constraint permutation); the size is equal to the number of cols
  * @return(1): the forest is a data structure representing all wires copy constraint. the collected wires forms a unique partition
  */
fn sigma_vecs(&self, k_is: &[F], subgroup: &[F]) -> (Vec<PolynomialValues<F>>, Forest) {}
+

3.2 functions

1
2
3
/// Finds a set of shifts that result in unique cosets for the multiplicative subgroup of size `2^subgroup_bits`.
/// Let `g` be a generator of the entire multiplicative group. the result is just g^0, ..., g^(num_shifts - 1)
pub fn get_unique_coset_shifts<F: Field>(subgroup_size: usize, num_shifts: usize) -> Vec<F> {}
+

4. selectors

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
/// Selector polynomials are computed as follows:
/// Partition the gates into (the smallest amount of) groups `{ G_i }`, such that for each group `G`
/// `|G| + max_{g in G} g.degree() <= max_degree`. These groups are constructed greedily from
/// the list of gates sorted by degree.
/// We build a selector polynomial `S_i` for each group `G_i`, with
/// S_i\[j\] =
///     if j-th row gate=g_k in G_i
///         k  (k \in [0, NUM_OF_GATE_TYPES)
///     else
///         UNUSED_SELECTOR
/// @param gates: the types of gates , gate type is differentiated by gate.id()
/// @param instances: all instances of gates ( a same type of gate could be used multiple times)
/// @param max_degree: max_degree of each group
/// @return(0): a vector of selector polynomials, the size of the vector is number of groups, in each group, the polynomial indicate which gate type it is
/// @return(1) SelectorsInfo {
///     selector_indices, a vector of gate group information. the size of vector is same to the number of gate types. the value represent which group the gate type belongs to
///     groups: a vector of groups information, the size is the number of groups. in each group, it is a range of the gate types (ordered as a method described above)
/// }
pub(crate) fn selector_polynomials<F: RichField + Extendable<D>, const D: usize>(
    gates: &[GateRef<F, D>],
    instances: &[GateInstance<F, D>],
    max_degree: usize,
) -> (Vec<PolynomialValues<F>>, SelectorsInfo) {}
+

5. permutation

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
pub struct WirePartition {
    // each element of the vector is a vector of copy constraint
    partition: Vec<Vec<Wire>>,
}

/// Disjoint Set Forest data-structure following <https://en.wikipedia.org/wiki/Disjoint-set_data_structure>.
pub struct Forest {
    /// A map of parent pointers, stored as indices. the parent value is the partition it belongs to
    /// those node with same parent value, means they are in the copy constraint 
    pub(crate) parents: Vec<usize>,

    num_wires: usize,
    num_routed_wires: usize, // num_routed_wires is used to construct all wires
    degree: usize,
}
+

6. hash

defined in CircuitBuilder

+
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
/// Hash function used for building Merkle trees.
type Hasher: Hasher<Self::F>;
/// Algebraic hash function used for the challenger and hashing public inputs.
type InnerHasher: AlgebraicHasher<Self::F>;

/// Trait for algebraic hash functions, built from a permutation using the sponge construction.
pub trait AlgebraicHasher<F: RichField>: Hasher<F, Hash = HashOut<F>> {
    type AlgebraicPermutation: PlonkyPermutation<Target>;

    /// Circuit to conditionally swap two chunks of the inputs (useful in verifying Merkle proofs),
    /// then apply the permutation.
    fn permute_swapped<const D: usize>(
        inputs: Self::AlgebraicPermutation,
        swap: BoolTarget,
        builder: &mut CircuitBuilder<F, D>,
    ) -> Self::AlgebraicPermutation
    where
        F: RichField + Extendable<D>;
}
-

Questions

    -
  • CircuitBuilder: self.constant_generators
    • -
  • Gate: extra_constant_wires
    • -
  • Constant Gate
    1
    2
    3
    4
    5
    6
    7
    fn eval_unfiltered(&self, vars: EvaluationVars<F, D>) -> Vec<F::Extension> {
        (0..self.num_consts)
            .map(|i| {
                vars.local_constants[self.const_input(i)] - vars.local_wires[self.wire_output(i)]
            })
            .collect()
    }
    • -
+

7. IOP

7.1 generator

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
/// Generator used to fill an extra constant.
#[derive(Debug, Clone, Default)]
pub struct ConstantGenerator<F: Field> {
    pub row: usize,  //  specifying the gate row number
    pub constant_index: usize, // index of constant input. for example, for a constant gate, num_consts is specified; num_consts will specify the total number of constants
    pub wire_index: usize,     // index of constant wire output
    pub constant: F, // the constant used to fill a target
}


/// A generator which runs once after a list of dependencies is present in the witness.
pub trait SimpleGenerator<F: RichField + Extendable<D>, const D: usize>:
    'static + Send + Sync + Debug
{
    fn id(&self) -> String;

    fn dependencies(&self) -> Vec<Target>;

    fn run_once(&self, witness: &PartitionWitness<F>, out_buffer: &mut GeneratedValues<F>);

    fn adapter(self) -> SimpleGeneratorAdapter<F, Self, D>
    where
        Self: Sized,
    {
        SimpleGeneratorAdapter {
            inner: self,
            _phantom: PhantomData,
        }
    }

    fn serialize(&self, dst: &mut Vec<u8>, common_data: &CommonCircuitData<F, D>) -> IoResult<()>;

    fn deserialize(src: &mut Buffer, common_data: &CommonCircuitData<F, D>) -> IoResult<Self>
    where
        Self: Sized;
}
+

8. FRI

Questions